From cf95735543688cd01a2c15b9cc2280044f993834 Mon Sep 17 00:00:00 2001 From: shisan Date: Tue, 1 Sep 2026 21:55:33 +0800 Subject: [PATCH] [core] Fix streaming compaction recovery from expired snapshots --- .../operation/FileSystemWriteRestore.java | 143 ++++++- .../apache/paimon/table/source/DataSplit.java | 38 +- .../table/source/DataTableStreamScan.java | 188 ++++++++- .../table/system/CompactBucketsTable.java | 32 +- .../apache/paimon/utils/SnapshotManager.java | 33 ++ .../operation/FileSystemWriteRestoreTest.java | 173 +++++++++ .../table/source/DataSplitCompatibleTest.java | 20 + .../table/source/StreamTableScanTest.java | 52 +++ .../paimon/utils/SnapshotManagerTest.java | 18 + .../flink/action/CompactDatabaseAction.java | 22 +- .../flink/sink/BucketsRowChannelComputer.java | 4 + .../flink/sink/CompactorSinkBuilder.java | 12 +- .../flink/sink/FlinkStreamPartitioner.java | 37 ++ .../sink/GlobalFullCompactionSinkWrite.java | 12 + .../paimon/flink/sink/LookupSinkWrite.java | 3 + .../sink/MultiTablesStoreCompactOperator.java | 273 ++++++++++++- .../flink/sink/StoreCompactOperator.java | 181 ++++++++- .../paimon/flink/sink/StoreSinkWrite.java | 12 + .../paimon/flink/sink/StoreSinkWriteImpl.java | 12 + .../coordinator/CoordinatedWriteRestore.java | 27 +- .../coordinator/ScanCoordinationRequest.java | 33 ++ .../coordinator/TableWriteCoordinator.java | 118 +++++- .../flink/source/CompactorSourceBuilder.java | 10 + .../source/ContinuousFileSplitEnumerator.java | 85 ++++- .../source/ContinuousFileStoreSource.java | 34 ++ .../AlignedContinuousFileSplitEnumerator.java | 108 +++++- .../assigners/AlignedSplitAssigner.java | 24 +- .../DynamicPartitionPruningAssigner.java | 17 +- .../source/assigners/FIFOSplitAssigner.java | 17 +- .../assigners/PreAssignSplitAssigner.java | 18 +- .../flink/source/assigners/SplitAssigner.java | 3 + .../action/CompactDatabaseActionITCase.java | 358 +++++++++++++++++- .../sink/FlinkStreamPartitionerTest.java | 101 +++++ .../MultiTablesStoreCompactOperatorTest.java | 146 +++++++ .../flink/sink/StoreCompactOperatorTest.java | 313 ++++++++++++++- .../CoordinatedWriteRestoreTest.java | 120 ++++++ .../TableWriteCoordinatorTest.java | 34 ++ .../ContinuousFileSplitEnumeratorTest.java | 32 ++ .../source/ContinuousFileStoreSourceTest.java | 96 +++++ ...gnedContinuousFileSplitEnumeratorTest.java | 82 +++- 40 files changed, 2907 insertions(+), 134 deletions(-) create mode 100644 paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/FlinkStreamPartitionerTest.java create mode 100644 paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/MultiTablesStoreCompactOperatorTest.java create mode 100644 paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/coordinator/CoordinatedWriteRestoreTest.java create mode 100644 paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/ContinuousFileStoreSourceTest.java diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileSystemWriteRestore.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileSystemWriteRestore.java index 740e7399f087..6f3836341d2c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/FileSystemWriteRestore.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileSystemWriteRestore.java @@ -25,10 +25,12 @@ import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.manifest.ManifestEntry; +import org.apache.paimon.table.source.OutOfRangeException; import org.apache.paimon.utils.SnapshotManager; import javax.annotation.Nullable; +import java.io.FileNotFoundException; import java.util.ArrayList; import java.util.List; @@ -41,13 +43,16 @@ public class FileSystemWriteRestore implements WriteRestore { private final FileStoreScan scan; private final IndexFileHandler indexFileHandler; private final @Nullable Long snapshotId; + private final boolean fallbackToLatest; + /** Snapshot selected after an expiration fallback; shared by all buckets of this restore. */ + private @Nullable Snapshot fallbackSnapshot; public FileSystemWriteRestore( CoreOptions options, SnapshotManager snapshotManager, FileStoreScan scan, IndexFileHandler indexFileHandler) { - this(options, snapshotManager, scan, indexFileHandler, null); + this(options, snapshotManager, scan, indexFileHandler, null, false); } public FileSystemWriteRestore( @@ -56,7 +61,27 @@ public FileSystemWriteRestore( FileStoreScan scan, IndexFileHandler indexFileHandler, long snapshotId) { - this(options, snapshotManager, scan, indexFileHandler, Long.valueOf(snapshotId)); + this(options, snapshotManager, scan, indexFileHandler, Long.valueOf(snapshotId), false); + } + + /** + * Creates a restore which prefers {@code snapshotId}, but falls back to the latest filesystem + * snapshot when expiration races with writer initialization. + */ + public FileSystemWriteRestore( + CoreOptions options, + SnapshotManager snapshotManager, + FileStoreScan scan, + IndexFileHandler indexFileHandler, + long snapshotId, + boolean fallbackToLatest) { + this( + options, + snapshotManager, + scan, + indexFileHandler, + Long.valueOf(snapshotId), + fallbackToLatest); } private FileSystemWriteRestore( @@ -64,11 +89,13 @@ private FileSystemWriteRestore( SnapshotManager snapshotManager, FileStoreScan scan, IndexFileHandler indexFileHandler, - @Nullable Long snapshotId) { + @Nullable Long snapshotId, + boolean fallbackToLatest) { this.snapshotManager = snapshotManager; this.scan = scan; this.indexFileHandler = indexFileHandler; this.snapshotId = snapshotId; + this.fallbackToLatest = fallbackToLatest; if (options.manifestDeleteFileDropStats()) { if (this.scan != null) { this.scan.dropStats(); @@ -91,16 +118,75 @@ public RestoreFiles restoreFiles( boolean scanDynamicBucketIndex, boolean scanDeleteVectorsIndex, boolean scanSourceIndexPayloads) { - // NOTE: don't use snapshotManager.latestSnapshot() here, - // because we don't want to flood the catalog with high concurrency - Snapshot snapshot = - snapshotId == null - ? snapshotManager.latestSnapshotFromFileSystem() - : snapshotManager.snapshot(snapshotId); + Snapshot snapshot = snapshot(); if (snapshot == null) { return RestoreFiles.empty(); } + // Expiration can remove the manifest of the selected snapshot and then remove the + // replacement snapshot while it is being read. When fallback is explicitly enabled for + // a rebase, advance monotonically through a small bounded number of newer snapshots. + // Ordinary restores retain their original fail-fast behavior. + for (int attempt = 0; ; attempt++) { + try { + return restoreFiles( + snapshot, + partition, + bucket, + scanDynamicBucketIndex, + scanDeleteVectorsIndex, + scanSourceIndexPayloads); + } catch (RuntimeException e) { + if (!shouldFallback(e, snapshot) || attempt >= 2) { + throw e; + } + + Snapshot latest = snapshotManager.latestSnapshotFromFileSystem(); + if (latest == null || latest.id() <= snapshot.id()) { + throw e; + } + snapshot = latest; + fallbackSnapshot = latest; + } + } + } + + private Snapshot snapshot() { + // NOTE: don't use snapshotManager.latestSnapshot() here, + // because we don't want to flood the catalog with high concurrency + if (fallbackSnapshot != null) { + return fallbackSnapshot; + } + try { + return snapshotId == null + ? snapshotManager.latestSnapshotFromFileSystem() + : snapshotManager.snapshot(snapshotId); + } catch (RuntimeException e) { + if (!fallbackToLatest + || snapshotId == null + || !containsExpiredSnapshotFailure(e) + || !isExpiredOrSuperseded(snapshotId)) { + throw e; + } + Snapshot latest = snapshotManager.latestSnapshotFromFileSystem(); + // A missing latest snapshot means the table is empty or the filesystem is still + // inconsistent. Do not turn that ambiguous condition into an empty restore and lose + // the original failure. + if (latest == null) { + throw e; + } + fallbackSnapshot = latest; + return latest; + } + } + + private RestoreFiles restoreFiles( + Snapshot snapshot, + BinaryRow partition, + int bucket, + boolean scanDynamicBucketIndex, + boolean scanDeleteVectorsIndex, + boolean scanSourceIndexPayloads) { List restoreFiles = new ArrayList<>(); List entries = scan.withSnapshot(snapshot).withPartitionBucket(partition, bucket).plan().files(); @@ -131,4 +217,43 @@ public RestoreFiles restoreFiles( deleteVectorsIndex, sourceIndexPayloads); } + + private boolean shouldFallback(RuntimeException failure, Snapshot snapshot) { + return fallbackToLatest + && snapshotId != null + && containsExpiredSnapshotFailure(failure) + && isExpiredOrSuperseded(snapshot.id()); + } + + private boolean isExpiredOrSuperseded(long requestedSnapshotId) { + if (snapshotManager.isSnapshotExpired(requestedSnapshotId)) { + return true; + } + + // Expiration cleans manifests before deleting the snapshot file and advancing the earliest + // hint. Therefore a requested snapshot can still exist while one of its manifests or + // indexes is already gone. A newer complete snapshot is sufficient to identify that race; + // failures without one are left untouched. + try { + Snapshot latest = snapshotManager.latestSnapshotFromFileSystem(); + return latest != null && latest.id() > requestedSnapshotId; + } catch (RuntimeException ignored) { + return false; + } + } + + private boolean containsExpiredSnapshotFailure(Throwable failure) { + Throwable current = failure; + while (current != null) { + // A restore reads concrete snapshot/manifest files. Only explicit expiration signals + // can use this opt-in fallback; retained snapshots and unrelated failures must still + // fail the job. + if (current instanceof FileNotFoundException + || current instanceof OutOfRangeException) { + return true; + } + current = current.getCause(); + } + return false; + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/DataSplit.java b/paimon-core/src/main/java/org/apache/paimon/table/source/DataSplit.java index 88bf60f019c8..6ea5f229e4f4 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/DataSplit.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/DataSplit.java @@ -82,6 +82,8 @@ public class DataSplit implements Split { private boolean isStreaming = false; private boolean rawConvertible; + /** Whether this split is a control split used to rebase a dedicated compaction writer. */ + private transient boolean rebase; public DataSplit() {} @@ -122,6 +124,18 @@ public boolean rawConvertible() { return rawConvertible; } + public boolean isRebase() { + return rebase; + } + + /** Returns a copy of this split marked as a dedicated compaction rebase split. */ + public DataSplit asRebase() { + DataSplit copy = new DataSplit(); + copy.assign(this); + copy.rebase = true; + return copy; + } + public OptionalLong earliestFileCreationEpochMillis() { return this.dataFiles.stream().mapToLong(DataFileMeta::creationTimeEpochMillis).min(); } @@ -349,6 +363,7 @@ public boolean equals(Object o) { && bucket == dataSplit.bucket && isStreaming == dataSplit.isStreaming && rawConvertible == dataSplit.rawConvertible + && rebase == dataSplit.rebase && Objects.equals(partition, dataSplit.partition) && Objects.equals(bucketPath, dataSplit.bucketPath) && Objects.equals(totalBuckets, dataSplit.totalBuckets) @@ -367,7 +382,8 @@ public int hashCode() { dataFiles, dataDeletionFiles, isStreaming, - rawConvertible); + rawConvertible, + rebase); } @Override @@ -404,11 +420,14 @@ protected void assign(DataSplit other) { this.dataDeletionFiles = other.dataDeletionFiles; this.isStreaming = other.isStreaming; this.rawConvertible = other.rawConvertible; + this.rebase = other.rebase; } public void serialize(DataOutputView out) throws IOException { out.writeLong(MAGIC); - out.writeInt(VERSION); + // Keep the wire format byte-for-byte compatible for ordinary splits. Rebase splits are an + // internal extension and carry the extra marker in version 10. + out.writeInt(rebase ? VERSION + 1 : VERSION); out.writeLong(snapshotId); SerializationUtils.serializeBinaryRow(partition, out); out.writeInt(bucket); @@ -436,6 +455,10 @@ public void serialize(DataOutputView out) throws IOException { out.writeBoolean(isStreaming); out.writeBoolean(rawConvertible); + + if (rebase) { + out.writeBoolean(true); + } } public static DataSplit deserialize(DataInputView in) throws IOException { @@ -473,6 +496,7 @@ public static DataSplit deserialize(DataInputView in) throws IOException { boolean isStreaming = in.readBoolean(); boolean rawConvertible = in.readBoolean(); + boolean rebase = version >= 10 && in.readBoolean(); DataSplit.Builder builder = builder() @@ -483,7 +507,8 @@ public static DataSplit deserialize(DataInputView in) throws IOException { .withTotalBuckets(totalBuckets) .withDataFiles(dataFiles) .isStreaming(isStreaming) - .rawConvertible(rawConvertible); + .rawConvertible(rawConvertible) + .rebase(rebase); if (dataDeletionFiles != null) { builder.withDataDeletionFiles(dataDeletionFiles); @@ -513,7 +538,7 @@ private static FunctionWithIOException getFileMetaS DataFileMetaWriteColsLegacySerializer serializer = new DataFileMetaWriteColsLegacySerializer(); return serializer::deserialize; - } else if (version == 9) { + } else if (version == 9 || version == VERSION + 1) { DataFileMetaSerializer serializer = new DataFileMetaSerializer(); return serializer::deserialize; } else { @@ -586,6 +611,11 @@ public Builder rawConvertible(boolean rawConvertible) { return this; } + public Builder rebase(boolean rebase) { + this.split.rebase = rebase; + return this; + } + public DataSplit build() { checkArgument(split.partition != null); checkArgument(split.bucket != -1); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableStreamScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableStreamScan.java index 8b5031de4c1e..1b25b9dea26d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableStreamScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableStreamScan.java @@ -22,6 +22,7 @@ import org.apache.paimon.CoreOptions.StreamScanMode; import org.apache.paimon.Snapshot; import org.apache.paimon.consumer.Consumer; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.schema.TableSchema; @@ -46,6 +47,8 @@ import javax.annotation.Nullable; +import java.io.FileNotFoundException; +import java.util.ArrayList; import java.util.List; import static org.apache.paimon.CoreOptions.ChangelogProducer.FULL_COMPACTION; @@ -200,7 +203,19 @@ private Plan nextPlan() { throw new EndOfScanException(); } - Snapshot snapshot = nextSnapshotProvider.getNextSnapshot(nextSnapshotId); + if (isSnapshotExpiredForRebase()) { + return rebasePlan(); + } + + Snapshot snapshot; + try { + snapshot = nextSnapshotProvider.getNextSnapshot(nextSnapshotId); + } catch (RuntimeException e) { + if (shouldRebaseAfterReadFailure(e)) { + return rebasePlan(); + } + throw e; + } if (snapshot == null) { return SnapshotNotExistPlan.INSTANCE; } @@ -213,30 +228,169 @@ private Plan nextPlan() { return SnapshotNotExistPlan.INSTANCE; } - // first try to get overwrite changes - if (snapshot.commitKind() == Snapshot.CommitKind.OVERWRITE) { - SnapshotReader.Plan overwritePlan = handleOverwriteSnapshot(snapshot); - if (overwritePlan != null) { + try { + // first try to get overwrite changes + if (snapshot.commitKind() == Snapshot.CommitKind.OVERWRITE) { + SnapshotReader.Plan overwritePlan = handleOverwriteSnapshot(snapshot); + if (overwritePlan != null) { + nextSnapshotId++; + if (overwritePlan.splits().isEmpty()) { + continue; + } + return overwritePlan; + } + } + + if (followUpScanner.shouldScanSnapshot(snapshot)) { + LOG.debug("Find snapshot id {}.", nextSnapshotId); + SnapshotReader.Plan plan = followUpScanner.scan(snapshot, snapshotReader); + currentWatermark = plan.watermark(); nextSnapshotId++; - if (overwritePlan.splits().isEmpty()) { + if (plan.splits().isEmpty()) { continue; } - return overwritePlan; + return plan; + } else { + nextSnapshotId++; + } + } catch (RuntimeException e) { + if (shouldRebaseAfterReadFailure(e)) { + return rebasePlan(); } + throw e; + } + } + } + + /** Whether this dedicated compaction scan has lost its checkpointed snapshot. */ + public boolean isSnapshotExpiredForRebase() { + if (scanMode != StreamScanMode.COMPACT_BUCKET_TABLE || nextSnapshotId == null) { + return false; + } + return isSnapshotExpiredForRebase(nextSnapshotId); + } + + /** Whether a snapshot used by a pending rebase split is no longer retained. */ + public boolean isSnapshotExpiredForRebase(long snapshotId) { + if (scanMode != StreamScanMode.COMPACT_BUCKET_TABLE) { + return false; + } + return snapshotManager.isSnapshotExpiredForRebase(snapshotId); + } + + private boolean shouldRebaseAfterReadFailure(Throwable failure) { + if (scanMode != StreamScanMode.COMPACT_BUCKET_TABLE) { + return false; + } + if (!containsExpiredSnapshotFailure(failure)) { + return false; + } + + // Expiration cleans manifests before deleting the snapshot file and advancing the + // earliest hint. During that window the requested snapshot can still exist while its + // manifest is already gone. A newer complete snapshot makes rebasing safe because it + // contains the full table state; unrelated failures without a newer snapshot still fail. + return isSnapshotExpiredForRebase() + || (nextSnapshotId != null && hasNewerLatestSnapshot(nextSnapshotId)); + } + + private boolean containsExpiredSnapshotFailure(Throwable failure) { + Throwable current = failure; + while (current != null) { + if (current instanceof FileNotFoundException + || current instanceof OutOfRangeException) { + return true; + } + current = current.getCause(); + } + return false; + } + + /** + * Rebuild a dedicated compaction baseline from one fixed latest snapshot. The returned splits + * are marked so the compaction sink can restore its writer from the same snapshot without + * treating the complete baseline as newly appended level-0 files. + */ + private Plan rebasePlan() { + return rebasePlan(0); + } + + private Plan rebasePlan(int retryCount) { + long expiredSnapshotId = nextSnapshotId; + Snapshot latestSnapshot = null; + try { + latestSnapshot = snapshotManager.latestSnapshotFromFileSystem(); + if (latestSnapshot == null) { + return SnapshotNotExistPlan.INSTANCE; } - if (followUpScanner.shouldScanSnapshot(snapshot)) { - LOG.debug("Find snapshot id {}.", nextSnapshotId); - SnapshotReader.Plan plan = followUpScanner.scan(snapshot, snapshotReader); - currentWatermark = plan.watermark(); - nextSnapshotId++; - if (plan.splits().isEmpty()) { - continue; + SnapshotReader.Plan plan; + try { + plan = snapshotReader.withMode(ScanMode.ALL).withSnapshot(latestSnapshot).read(); + } finally { + // SnapshotReader is mutable. Keep the dedicated stream scan in DELTA mode after + // the one-shot baseline read so a subsequent plan cannot accidentally become an + // ALL scan if the follow-up scanner is changed or a retry is scheduled. + snapshotReader.withMode(ScanMode.DELTA); + } + List rebaseSplits = new ArrayList<>(); + for (Split split : plan.splits()) { + if (!(split instanceof DataSplit)) { + throw new IllegalStateException( + "Dedicated compaction rebase produced an unsupported split: " + + split.getClass()); } - return plan; - } else { - nextSnapshotId++; + rebaseSplits.add(((DataSplit) split).asRebase()); } + if (rebaseSplits.isEmpty()) { + // A table can be empty (or all files can be filtered out). Still emit one control + // split so the sink drops writer state that references the expired history. + rebaseSplits.add( + DataSplit.builder() + .withSnapshot(latestSnapshot.id()) + .withPartition(BinaryRow.EMPTY_ROW) + .withBucket(0) + .withBucketPath("") + .withDataFiles(new ArrayList<>()) + .isStreaming(true) + .rawConvertible(false) + .rebase(true) + .build()); + } + nextSnapshotId = latestSnapshot.id() + 1; + currentWatermark = plan.watermark(); + LOG.warn( + "Checkpointed snapshot {} expired; rebasing dedicated compaction to latest snapshot {}.", + expiredSnapshotId, + latestSnapshot.id()); + return new PlanImpl(plan.watermark(), latestSnapshot.id(), rebaseSplits); + } catch (RuntimeException e) { + // If a newer snapshot replaced the one selected above while its manifests were being + // read, retry against that newer snapshot. A persistent read failure for the current + // latest snapshot must still fail the job instead of being retried forever. + if (retryCount < 2 + && latestSnapshot != null + && containsExpiredSnapshotFailure(e) + && hasNewerLatestSnapshot(latestSnapshot)) { + LOG.warn( + "Latest snapshot {} was replaced while rebasing dedicated compaction; retrying.", + latestSnapshot.id()); + return rebasePlan(retryCount + 1); + } + throw e; + } + } + + private boolean hasNewerLatestSnapshot(Snapshot previousSnapshot) { + return hasNewerLatestSnapshot(previousSnapshot.id()); + } + + private boolean hasNewerLatestSnapshot(long previousSnapshotId) { + try { + Snapshot latest = snapshotManager.latestSnapshotFromFileSystem(); + return latest != null && latest.id() > previousSnapshotId; + } catch (RuntimeException ignored) { + return false; } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/CompactBucketsTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/CompactBucketsTable.java index f18d184b8437..e51bc5be6058 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/CompactBucketsTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/CompactBucketsTable.java @@ -46,6 +46,7 @@ import org.apache.paimon.table.source.TableRead; import org.apache.paimon.table.source.snapshot.SnapshotReader; import org.apache.paimon.types.BigIntType; +import org.apache.paimon.types.BooleanType; import org.apache.paimon.types.DataType; import org.apache.paimon.types.IntType; import org.apache.paimon.types.RowType; @@ -91,7 +92,8 @@ public class CompactBucketsTable implements DataTable, ReadonlyTable { new IntType(false), newBytesType(false), new VarCharType(true, Integer.MAX_VALUE), - new VarCharType(false, Integer.MAX_VALUE) + new VarCharType(false, Integer.MAX_VALUE), + new BooleanType(false) }, new String[] { "_SNAPSHOT_ID", @@ -99,7 +101,8 @@ public class CompactBucketsTable implements DataTable, ReadonlyTable { "_BUCKET", "_FILES", "_DATABASE_NAME", - "_TABLE_NAME" + "_TABLE_NAME", + "_REBASE" }); public CompactBucketsTable(FileStoreTable wrapped, boolean isContinuous) { @@ -261,15 +264,19 @@ public RecordReader createReader(Split split) throws IOException { } DataSplit dataSplit = (DataSplit) split; - // in case of schema evolution - for (DataFileMeta file : dataSplit.dataFiles()) { - if (file.schemaId() > baseSchemaId) { - throw new RuntimeException( - String.format( - "File %s has schema id %d, " - + "which is larger than the base schema id %d. " - + "Trying to restart the job.", - file.fileName(), file.schemaId(), baseSchemaId)); + // Ordinary metadata records are still rejected when they observe a schema newer than + // the source table. A rebase record is different: it is only a control record and its + // file metadata is precisely what the dedicated sink uses to refresh its writer. + if (!dataSplit.isRebase()) { + for (DataFileMeta file : dataSplit.dataFiles()) { + if (file.schemaId() > baseSchemaId) { + throw new RuntimeException( + String.format( + "File %s has schema id %d, " + + "which is larger than the base schema id %d. " + + "Trying to restart the job.", + file.fileName(), file.schemaId(), baseSchemaId)); + } } } @@ -287,7 +294,8 @@ public RecordReader createReader(Split split) throws IOException { dataSplit.bucket(), dataFileMetaSerializer.serializeList(files), BinaryString.fromString(databaseName), - BinaryString.fromString(wrapped.name())); + BinaryString.fromString(wrapped.name()), + dataSplit.isRebase()); return new IteratorRecordReader<>(Collections.singletonList(row).iterator()); } diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/SnapshotManager.java b/paimon-core/src/main/java/org/apache/paimon/utils/SnapshotManager.java index 8cd159997865..16108a2db5a2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/utils/SnapshotManager.java +++ b/paimon-core/src/main/java/org/apache/paimon/utils/SnapshotManager.java @@ -156,6 +156,39 @@ public boolean snapshotExists(long snapshotId) { } } + /** + * Returns whether a snapshot id is outside the retained snapshot range. + * + *

A missing snapshot file or a read exception alone is not enough to classify a snapshot as + * expired: the table may have been recreated or the file may be corrupt. + */ + public boolean isSnapshotExpired(long snapshotId) { + Long earliestSnapshotId = earliestSnapshotId(); + return earliestSnapshotId != null && snapshotId < earliestSnapshotId; + } + + /** + * Returns whether a snapshot is no longer a safe restore point for dedicated compaction. + * + *

Snapshot cleanup can remove the snapshot file before the earliest-snapshot hint is + * advanced. In that window, a missing snapshot together with a newer filesystem snapshot is + * enough to classify the old id as expired. + */ + public boolean isSnapshotExpiredForRebase(long snapshotId) { + if (isSnapshotExpired(snapshotId)) { + return true; + } + try { + if (snapshotExists(snapshotId)) { + return false; + } + Snapshot latest = latestSnapshotFromFileSystem(); + return latest != null && latest.id() > snapshotId; + } catch (RuntimeException ignored) { + return false; + } + } + public void deleteSnapshot(long snapshotId) { Path path = snapshotPath(snapshotId); if (cache != null) { diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/FileSystemWriteRestoreTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/FileSystemWriteRestoreTest.java index 69d41094c4b6..7e12ef49048c 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/FileSystemWriteRestoreTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/FileSystemWriteRestoreTest.java @@ -26,13 +26,16 @@ import org.junit.jupiter.api.Test; +import java.io.FileNotFoundException; import java.util.Collections; import java.util.HashMap; import static org.apache.paimon.data.BinaryRow.EMPTY_ROW; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -102,4 +105,174 @@ void testRestoreSourceIndexPayloadsWithoutDirectory() { assertThat(restored.sourceIndexPayloads()).containsExactly(ann); } + + @Test + void testPinnedRestoreFallsBackToLatestWhenSnapshotExpires() { + Snapshot pinned = mock(Snapshot.class); + Snapshot latest = mock(Snapshot.class); + when(pinned.id()).thenReturn(5L); + when(latest.id()).thenReturn(6L); + + SnapshotManager snapshotManager = mock(SnapshotManager.class); + when(snapshotManager.snapshot(5L)).thenReturn(pinned); + when(snapshotManager.isSnapshotExpired(5L)).thenReturn(true); + when(snapshotManager.latestSnapshotFromFileSystem()).thenReturn(latest); + + FileStoreScan scan = mock(FileStoreScan.class); + FileStoreScan.Plan plan = mock(FileStoreScan.Plan.class); + when(scan.withSnapshot(pinned)) + .thenThrow(new RuntimeException(new FileNotFoundException("expired"))); + when(scan.withSnapshot(latest)).thenReturn(scan); + when(scan.withPartitionBucket(EMPTY_ROW, 0)).thenReturn(scan); + when(scan.plan()).thenReturn(plan); + when(plan.files()).thenReturn(Collections.emptyList()); + + FileSystemWriteRestore restore = + new FileSystemWriteRestore( + new CoreOptions(new HashMap<>()), + snapshotManager, + scan, + mock(IndexFileHandler.class), + 5L, + true); + + RestoreFiles restored = restore.restoreFiles(EMPTY_ROW, 0, false, false, false); + + assertThat(restored.snapshot()).isSameAs(latest); + } + + @Test + void testPinnedRestoreFallsBackWhenManifestExpiresBeforeSnapshotFile() { + Snapshot pinned = mock(Snapshot.class); + Snapshot latest = mock(Snapshot.class); + when(pinned.id()).thenReturn(5L); + when(latest.id()).thenReturn(6L); + + SnapshotManager snapshotManager = mock(SnapshotManager.class); + when(snapshotManager.snapshot(5L)).thenReturn(pinned); + // Expiration removes manifests before the snapshot file and earliest hint. + when(snapshotManager.isSnapshotExpired(5L)).thenReturn(false); + when(snapshotManager.latestSnapshotFromFileSystem()).thenReturn(latest); + + FileStoreScan scan = mock(FileStoreScan.class); + FileStoreScan.Plan plan = mock(FileStoreScan.Plan.class); + when(scan.withSnapshot(pinned)) + .thenThrow(new RuntimeException(new FileNotFoundException("expired manifest"))); + when(scan.withSnapshot(latest)).thenReturn(scan); + when(scan.withPartitionBucket(EMPTY_ROW, 0)).thenReturn(scan); + when(scan.plan()).thenReturn(plan); + when(plan.files()).thenReturn(Collections.emptyList()); + + FileSystemWriteRestore restore = + new FileSystemWriteRestore( + new CoreOptions(new HashMap<>()), + snapshotManager, + scan, + mock(IndexFileHandler.class), + 5L, + true); + + RestoreFiles restored = restore.restoreFiles(EMPTY_ROW, 0, false, false, false); + + assertThat(restored.snapshot()).isSameAs(latest); + } + + @Test + void testFallbackSnapshotIsPinnedAcrossBuckets() { + Snapshot pinned = mock(Snapshot.class); + Snapshot latest = mock(Snapshot.class); + when(pinned.id()).thenReturn(5L); + when(latest.id()).thenReturn(6L); + + SnapshotManager snapshotManager = mock(SnapshotManager.class); + when(snapshotManager.snapshot(5L)).thenReturn(pinned); + when(snapshotManager.isSnapshotExpired(5L)).thenReturn(true); + when(snapshotManager.latestSnapshotFromFileSystem()).thenReturn(latest); + + FileStoreScan scan = mock(FileStoreScan.class); + FileStoreScan.Plan plan = mock(FileStoreScan.Plan.class); + when(scan.withSnapshot(pinned)) + .thenThrow(new RuntimeException(new FileNotFoundException("expired"))); + when(scan.withSnapshot(latest)).thenReturn(scan); + when(scan.withPartitionBucket(EMPTY_ROW, 0)).thenReturn(scan); + when(scan.withPartitionBucket(EMPTY_ROW, 1)).thenReturn(scan); + when(scan.plan()).thenReturn(plan); + when(plan.files()).thenReturn(Collections.emptyList()); + + FileSystemWriteRestore restore = + new FileSystemWriteRestore( + new CoreOptions(new HashMap<>()), + snapshotManager, + scan, + mock(IndexFileHandler.class), + 5L, + true); + + RestoreFiles firstBucket = restore.restoreFiles(EMPTY_ROW, 0, false, false, false); + RestoreFiles secondBucket = restore.restoreFiles(EMPTY_ROW, 1, false, false, false); + + assertThat(firstBucket.snapshot()).isSameAs(latest); + assertThat(secondBucket.snapshot()).isSameAs(latest); + verify(scan, times(1)).withSnapshot(pinned); + verify(scan, times(2)).withSnapshot(latest); + verify(snapshotManager, times(1)).latestSnapshotFromFileSystem(); + } + + @Test + void testPinnedRestoreDoesNotFallbackForNonExpiredSnapshot() { + Snapshot pinned = mock(Snapshot.class); + SnapshotManager snapshotManager = mock(SnapshotManager.class); + when(pinned.id()).thenReturn(5L); + when(snapshotManager.snapshot(5L)).thenReturn(pinned); + when(snapshotManager.isSnapshotExpired(5L)).thenReturn(false); + + FileStoreScan scan = mock(FileStoreScan.class); + when(scan.withSnapshot(pinned)) + .thenThrow(new RuntimeException(new FileNotFoundException("missing manifest"))); + + FileSystemWriteRestore restore = + new FileSystemWriteRestore( + new CoreOptions(new HashMap<>()), + snapshotManager, + scan, + mock(IndexFileHandler.class), + 5L, + true); + + assertThatThrownBy(() -> restore.restoreFiles(EMPTY_ROW, 0, false, false, false)) + .isInstanceOf(RuntimeException.class) + .hasCauseInstanceOf(FileNotFoundException.class); + verify(snapshotManager).latestSnapshotFromFileSystem(); + } + + @Test + void testNormalRestoreFailsFastWhenIndexIsMissing() { + Snapshot pinned = mock(Snapshot.class); + SnapshotManager snapshotManager = mock(SnapshotManager.class); + when(snapshotManager.snapshot(5L)).thenReturn(pinned); + + FileStoreScan scan = mock(FileStoreScan.class); + FileStoreScan.Plan plan = mock(FileStoreScan.Plan.class); + when(scan.withSnapshot(pinned)).thenReturn(scan); + when(scan.withPartitionBucket(EMPTY_ROW, 0)).thenReturn(scan); + when(scan.plan()).thenReturn(plan); + when(plan.files()).thenReturn(Collections.emptyList()); + + IndexFileHandler indexFileHandler = mock(IndexFileHandler.class); + when(indexFileHandler.scanHashIndex(pinned, EMPTY_ROW, 0)) + .thenThrow(new RuntimeException(new FileNotFoundException("missing index"))); + + FileSystemWriteRestore restore = + new FileSystemWriteRestore( + new CoreOptions(new HashMap<>()), + snapshotManager, + scan, + indexFileHandler, + 5L); + + assertThatThrownBy(() -> restore.restoreFiles(EMPTY_ROW, 0, true, false, false)) + .isInstanceOf(RuntimeException.class) + .hasCauseInstanceOf(FileNotFoundException.class); + verify(snapshotManager, never()).latestSnapshotFromFileSystem(); + } } diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/DataSplitCompatibleTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/DataSplitCompatibleTest.java index e3f5403c68be..bd998bc8cb17 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/DataSplitCompatibleTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/DataSplitCompatibleTest.java @@ -241,6 +241,26 @@ public void testSerializer() throws IOException { assertThat(newSplit).isEqualTo(split); } + @Test + public void testRebaseMarkerSerializer() throws IOException { + DataSplit split = + DataSplit.builder() + .withSnapshot(10) + .withPartition(BinaryRow.EMPTY_ROW) + .withBucket(0) + .withDataFiles(Collections.emptyList()) + .withBucketPath("my path") + .rebase(true) + .build(); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + split.serialize(new DataOutputViewStreamWrapper(out)); + DataSplit restored = DataSplit.deserialize(new DataInputDeserializer(out.toByteArray())); + + assertThat(restored).isEqualTo(split); + assertThat(restored.isRebase()).isTrue(); + } + @Test public void testSerializerCompatibleV1() throws Exception { SimpleStats keyStats = diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/StreamTableScanTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/StreamTableScanTest.java index c494a8977a08..741fb057cde8 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/StreamTableScanTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/StreamTableScanTest.java @@ -29,6 +29,7 @@ import org.apache.paimon.table.sink.StreamWriteBuilder; import org.apache.paimon.table.sink.TableCommitImpl; import org.apache.paimon.table.source.snapshot.ScannerTestBase; +import org.apache.paimon.table.source.snapshot.SnapshotReader; import org.apache.paimon.types.RowKind; import org.junit.jupiter.api.Test; @@ -337,6 +338,57 @@ public void testStartingFromNonExistingSnapshot() throws Exception { commit.close(); } + @Test + public void testDedicatedCompactionRebasesWhenSnapshotExpires() throws Exception { + Map options = new HashMap<>(); + options.put(CoreOptions.SNAPSHOT_NUM_RETAINED_MIN.key(), "1"); + options.put(CoreOptions.SNAPSHOT_NUM_RETAINED_MAX.key(), "1"); + options.put( + CoreOptions.STREAM_SCAN_MODE.key(), + CoreOptions.StreamScanMode.COMPACT_BUCKET_TABLE.getValue()); + FileStoreTable table = this.table.copy(options); + StreamTableWrite write = table.newWrite(commitUser); + StreamTableCommit commit = table.newCommit(commitUser); + StreamTableScan scan = table.newStreamScan(); + + write.write(rowData(1, 10, 100L)); + commit.commit(0, write.prepareCommit(true, 0)); + + // The dedicated compactor starts at the earliest snapshot and discovers it on the next + // poll. + TableScan.Plan initialPlan = scan.plan(); + assertThat(initialPlan).isSameAs(SnapshotNotExistPlan.INSTANCE); + TableScan.Plan startingPlan = scan.plan(); + assertThat(startingPlan.splits()).isNotEmpty(); + assertThat(scan.checkpoint()).isEqualTo(2L); + + write.write(rowData(1, 20, 200L)); + commit.commit(1, write.prepareCommit(true, 1)); + write.write(rowData(1, 30, 300L)); + commit.commit(2, write.prepareCommit(true, 2)); + + SnapshotReader.Plan rebasePlan = (SnapshotReader.Plan) scan.plan(); + assertThat(rebasePlan.splits()) + .isNotEmpty() + .allMatch(split -> split instanceof DataSplit && ((DataSplit) split).isRebase()); + assertThat(rebasePlan.snapshotId()).isEqualTo(3L); + assertThat(scan.checkpoint()).isEqualTo(4L); + + // The rebase plan advances the cursor to latest + 1. A subsequent append snapshot must + // therefore be delivered as a normal delta plan instead of being skipped or rebased again. + write.write(rowData(1, 40, 400L)); + commit.commit(3, write.prepareCommit(true, 3)); + SnapshotReader.Plan deltaPlan = (SnapshotReader.Plan) scan.plan(); + assertThat(deltaPlan.snapshotId()).isEqualTo(4L); + assertThat(deltaPlan.splits()) + .isNotEmpty() + .allMatch(split -> split instanceof DataSplit && !((DataSplit) split).isRebase()); + assertThat(scan.checkpoint()).isEqualTo(5L); + + write.close(); + commit.close(); + } + @Test public void testPlanWithOutOfRange() throws Exception { Map options = new HashMap<>(); diff --git a/paimon-core/src/test/java/org/apache/paimon/utils/SnapshotManagerTest.java b/paimon-core/src/test/java/org/apache/paimon/utils/SnapshotManagerTest.java index bf1a005878a9..034cd22b0a11 100644 --- a/paimon-core/src/test/java/org/apache/paimon/utils/SnapshotManagerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/utils/SnapshotManagerTest.java @@ -83,6 +83,24 @@ public void testSnapshotPath() { } } + @Test + public void testSnapshotExpiredUsesRetainedRange() throws IOException { + FileIO fileIO = LocalFileIO.create(); + SnapshotManager snapshotManager = newSnapshotManager(fileIO, new Path(tempDir.toString())); + fileIO.tryToWriteAtomic( + snapshotManager.snapshotPath(5), createSnapshotWithMillis(5, 5000).toJson()); + fileIO.tryToWriteAtomic( + snapshotManager.snapshotPath(6), createSnapshotWithMillis(6, 6000).toJson()); + snapshotManager.commitEarliestHint(5); + snapshotManager.commitLatestHint(6); + + assertThat(snapshotManager.isSnapshotExpired(4)).isTrue(); + assertThat(snapshotManager.isSnapshotExpired(5)).isFalse(); + // A snapshot id from a recreated table is newer than the current latest snapshot and is + // therefore not an expiration case. + assertThat(snapshotManager.isSnapshotExpired(7)).isFalse(); + } + @Test public void testSnapshotsWithIdSkipsExpiredSnapshot() throws Exception { FileIO fileIO = Mockito.mock(FileIO.class); diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactDatabaseAction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactDatabaseAction.java index 3000c319b54d..8ea4ec444cb9 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactDatabaseAction.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactDatabaseAction.java @@ -55,6 +55,7 @@ import java.util.regex.Pattern; import static org.apache.paimon.flink.sink.FlinkStreamPartitioner.partition; +import static org.apache.paimon.flink.sink.FlinkStreamPartitioner.partitionCompactionRecords; import static org.apache.paimon.flink.sink.FlinkStreamPartitioner.rebalance; /** Database compact action for Flink. */ @@ -226,13 +227,20 @@ private void buildForCombinedMode() { // multi bucket table which has multi bucket in a partition like fix bucket and dynamic // bucket DataStream awareBucketTableSource = - partition( - sourceBuilder - .withEnv(env) - .withContinuousMode(isStreaming) - .buildAwareBucketTableSource(), - new BucketsRowChannelComputer(), - parallelism); + isStreaming + ? partitionCompactionRecords( + sourceBuilder + .withEnv(env) + .withContinuousMode(true) + .buildAwareBucketTableSource(), + parallelism) + : partition( + sourceBuilder + .withEnv(env) + .withContinuousMode(false) + .buildAwareBucketTableSource(), + new BucketsRowChannelComputer(), + parallelism); // unaware bucket table DataStream unawareBucketTableSource = diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/BucketsRowChannelComputer.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/BucketsRowChannelComputer.java index 61c9dd01893d..349666cff9e0 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/BucketsRowChannelComputer.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/BucketsRowChannelComputer.java @@ -43,6 +43,10 @@ public int channel(RowData rowData) { return ChannelComputer.select(partition, bucket, numberOfChannels); } + public static boolean isRebase(RowData rowData) { + return rowData.getArity() > 6 && !rowData.isNullAt(6) && rowData.getBoolean(6); + } + @Override public String toString() { return "compactor-partitioner"; diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CompactorSinkBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CompactorSinkBuilder.java index 4089100ffdfe..71fc91b5a8e7 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CompactorSinkBuilder.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CompactorSinkBuilder.java @@ -23,6 +23,8 @@ import org.apache.paimon.table.BucketMode; import org.apache.paimon.table.FileStoreTable; +import org.apache.flink.api.common.RuntimeExecutionMode; +import org.apache.flink.configuration.ExecutionOptions; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.DataStreamSink; import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; @@ -32,6 +34,7 @@ import static org.apache.paimon.CoreOptions.createCommitUser; import static org.apache.paimon.flink.sink.FlinkStreamPartitioner.partition; +import static org.apache.paimon.flink.sink.FlinkStreamPartitioner.partitionCompactionRecords; /** Builder for {@link CompactorSink}. */ public class CompactorSinkBuilder { @@ -89,8 +92,15 @@ private DataStreamSink buildForBucketAware() { commitUser); case LINEAR: default: + boolean streaming = + input.getExecutionEnvironment() + .getConfiguration() + .get(ExecutionOptions.RUNTIME_MODE) + == RuntimeExecutionMode.STREAMING; DataStream partitioned = - partition(input, new BucketsRowChannelComputer(), parallelism); + streaming + ? partitionCompactionRecords(input, parallelism) + : partition(input, new BucketsRowChannelComputer(), parallelism); return new CompactorSink(table, fullCompaction).sinkFrom(partitioned); } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkStreamPartitioner.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkStreamPartitioner.java index 97c4cba1fcba..6f95fdfabf8d 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkStreamPartitioner.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkStreamPartitioner.java @@ -23,10 +23,15 @@ import org.apache.flink.runtime.io.network.api.writer.SubtaskStateMapper; import org.apache.flink.runtime.plugable.SerializationDelegate; import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; +import org.apache.flink.streaming.api.functions.ProcessFunction; import org.apache.flink.streaming.api.transformations.PartitionTransformation; import org.apache.flink.streaming.runtime.partitioner.RebalancePartitioner; import org.apache.flink.streaming.runtime.partitioner.StreamPartitioner; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.table.data.RowData; +import org.apache.flink.util.Collector; +import org.apache.flink.util.OutputTag; import static org.apache.paimon.flink.utils.ParallelismUtils.forwardParallelism; @@ -92,4 +97,36 @@ public static DataStream rebalance(DataStream input, Integer paralleli } return new DataStream<>(input.getExecutionEnvironment(), partitioned); } + + /** + * Partitions normal compaction records by bucket and broadcasts rebase control records. + * + *

Rebase controls must reach every writer subtask so each subtask can discard its + * checkpointed writer state. They are not bucket data and therefore cannot use the normal + * bucket partitioner. This helper is intentionally scoped to the dedicated compaction row + * format. + */ + public static DataStream partitionCompactionRecords( + DataStream input, Integer parallelism) { + final OutputTag rebaseTag = new OutputTag("paimon-compact-rebase") {}; + SingleOutputStreamOperator normalRecords = + input.process( + new ProcessFunction() { + @Override + public void processElement( + RowData row, Context context, Collector out) { + if (BucketsRowChannelComputer.isRebase(row)) { + context.output(rebaseTag, row); + } else { + out.collect(row); + } + } + }); + if (parallelism != null) { + normalRecords.setParallelism(parallelism); + } + DataStream rebaseRecords = normalRecords.getSideOutput(rebaseTag).broadcast(); + return partition(normalRecords, new BucketsRowChannelComputer(), parallelism) + .union(rebaseRecords); + } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/GlobalFullCompactionSinkWrite.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/GlobalFullCompactionSinkWrite.java index a5702d7c6d0c..a93a684f65b8 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/GlobalFullCompactionSinkWrite.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/GlobalFullCompactionSinkWrite.java @@ -22,6 +22,7 @@ import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.memory.MemoryPoolFactory; +import org.apache.paimon.operation.WriteRestore; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.sink.SinkRecord; import org.apache.paimon.utils.SnapshotManager; @@ -89,6 +90,9 @@ public GlobalFullCompactionSinkWrite( this.deltaCommits = deltaCommits; + // Keep the existing state key for checkpoint compatibility. The rebase operation replaces + // the in-memory writer; the next checkpoint overwrites this state, so it does not need to + // mutate the shared state map while processing a control record. this.tableName = table.name(); this.snapshotManager = table.snapshotManager(); @@ -133,6 +137,14 @@ public void compact(BinaryRow partition, int bucket, boolean fullCompaction) thr touchBucket(partition, bucket); } + @Override + public void rebase(FileStoreTable table, WriteRestore writeRestore) throws Exception { + super.rebase(table, writeRestore); + currentWrittenBuckets.clear(); + writtenBuckets.clear(); + commitIdentifiersToCheck.clear(); + } + private void touchBucket(BinaryRow partition, int bucket) { if (LOG.isDebugEnabled()) { LOG.debug("touch partition {}, bucket {}", partition, bucket); diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/LookupSinkWrite.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/LookupSinkWrite.java index e8b27083b01c..7fdea59b089b 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/LookupSinkWrite.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/LookupSinkWrite.java @@ -59,6 +59,9 @@ public LookupSinkWrite( memoryPoolFactory, metricGroup); + // Keep the existing state key for checkpoint compatibility. Rebase is inherited from + // StoreSinkWriteImpl: it replaces the in-memory writer, and the next checkpoint overwrites + // the active-bucket state. this.tableName = table.name(); List activeBucketsStateValues = diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/MultiTablesStoreCompactOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/MultiTablesStoreCompactOperator.java index 7d7ba29760b0..9cf37931f7e1 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/MultiTablesStoreCompactOperator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/MultiTablesStoreCompactOperator.java @@ -19,6 +19,7 @@ package org.apache.paimon.flink.sink; import org.apache.paimon.CoreOptions; +import org.apache.paimon.Snapshot; import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.CatalogLoader; import org.apache.paimon.catalog.Identifier; @@ -26,11 +27,17 @@ import org.apache.paimon.flink.utils.RuntimeContextUtils; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.io.DataFileMetaSerializer; +import org.apache.paimon.operation.FileSystemWriteRestore; import org.apache.paimon.options.Options; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.sink.ChannelComputer; import org.apache.paimon.utils.Preconditions; +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.api.common.typeinfo.BasicTypeInfo; +import org.apache.flink.api.java.tuple.Tuple3; +import org.apache.flink.api.java.typeutils.TupleTypeInfo; import org.apache.flink.runtime.state.StateInitializationContext; import org.apache.flink.runtime.state.StateSnapshotContext; import org.apache.flink.streaming.api.environment.CheckpointConfig; @@ -77,6 +84,19 @@ public class MultiTablesStoreCompactOperator protected Map tables; protected Map writes; protected String commitUser; + private transient Map compactRefreshers; + /** Snapshot ID of the newest rebase marker processed for each table. */ + private transient Map rebasedSnapshots; + + /** Highest snapshot ID covered by the writer baseline for each table after a rebase. */ + private transient Map rebasedThroughSnapshots; + + /** Highest snapshot ID observed on input for each table. */ + private transient Map highestObservedSnapshots; + + private transient ListState> rebaseSnapshotState; + private transient ListState> rebasedThroughSnapshotState; + private transient ListState> highestObservedSnapshotState; private MultiTablesStoreCompactOperator( StreamOperatorParameters parameters, @@ -100,6 +120,76 @@ private MultiTablesStoreCompactOperator( public void initializeState(StateInitializationContext context) throws Exception { super.initializeState(context); + rebaseSnapshotState = + context.getOperatorStateStore() + .getUnionListState( + new ListStateDescriptor<>( + "paimon_compact_rebase_snapshots", + new TupleTypeInfo<>( + BasicTypeInfo.STRING_TYPE_INFO, + BasicTypeInfo.STRING_TYPE_INFO, + BasicTypeInfo.LONG_TYPE_INFO))); + rebasedThroughSnapshotState = + context.getOperatorStateStore() + .getUnionListState( + new ListStateDescriptor<>( + "paimon_compact_rebase_through_snapshots", + new TupleTypeInfo<>( + BasicTypeInfo.STRING_TYPE_INFO, + BasicTypeInfo.STRING_TYPE_INFO, + BasicTypeInfo.LONG_TYPE_INFO))); + highestObservedSnapshotState = + context.getOperatorStateStore() + .getUnionListState( + new ListStateDescriptor<>( + "paimon_compact_highest_observed_snapshots", + new TupleTypeInfo<>( + BasicTypeInfo.STRING_TYPE_INFO, + BasicTypeInfo.STRING_TYPE_INFO, + BasicTypeInfo.LONG_TYPE_INFO))); + rebasedSnapshots = new HashMap<>(); + for (Tuple3 entry : rebaseSnapshotState.get()) { + if (entry.f2 != null) { + Identifier tableId = Identifier.create(entry.f0, entry.f1); + rebasedSnapshots.put( + tableId, + Math.max(entry.f2, rebasedSnapshots.getOrDefault(tableId, Long.MIN_VALUE))); + } + } + rebasedThroughSnapshots = new HashMap<>(); + for (Tuple3 entry : rebasedThroughSnapshotState.get()) { + if (entry.f2 != null) { + Identifier tableId = Identifier.create(entry.f0, entry.f1); + rebasedThroughSnapshots.put( + tableId, + Math.max( + entry.f2, + rebasedThroughSnapshots.getOrDefault(tableId, Long.MIN_VALUE))); + } + } + highestObservedSnapshots = new HashMap<>(); + for (Tuple3 entry : highestObservedSnapshotState.get()) { + if (entry.f2 != null) { + Identifier tableId = Identifier.create(entry.f0, entry.f1); + highestObservedSnapshots.put( + tableId, + Math.max( + entry.f2, + Math.max( + highestObservedSnapshots.getOrDefault( + tableId, Long.MIN_VALUE), + rebasedThroughSnapshots.getOrDefault( + tableId, Long.MIN_VALUE)))); + } + } + for (Map.Entry entry : rebasedThroughSnapshots.entrySet()) { + highestObservedSnapshots.put( + entry.getKey(), + Math.max( + entry.getValue(), + highestObservedSnapshots.getOrDefault(entry.getKey(), Long.MIN_VALUE))); + } + catalog = catalogLoader.load(); // Each job can only have one username and this name must be consistent across restarts. @@ -124,6 +214,7 @@ public void initializeState(StateInitializationContext context) throws Exception tables = new HashMap<>(); writes = new HashMap<>(); + compactRefreshers = new HashMap<>(); } @Override @@ -143,9 +234,16 @@ public void processElement(StreamRecord element) throws Exception { List files = dataFileMetaSerializer.deserializeList(serializedFiles); String databaseName = record.getString(4).toString(); String tableName = record.getString(5).toString(); + boolean rebase = record.getArity() > 6 && !record.isNullAt(6) && record.getBoolean(6); Identifier tableId = Identifier.create(databaseName, tableName); + highestObservedSnapshots.put( + tableId, + Math.max( + snapshotId, + highestObservedSnapshots.getOrDefault(tableId, Long.MIN_VALUE))); FileStoreTable table = getTable(tableId); + final FileStoreTable initialTable = table; Preconditions.checkArgument( !table.coreOptions().writeOnly(), @@ -161,17 +259,131 @@ public void processElement(StreamRecord element) throws Exception { tableId, id -> storeSinkWriteProvider.provide( - table, + initialTable, commitUser, state, getContainingTask().getEnvironment().getIOManager(), memoryPoolFactory, getMetricGroup())); + if (isStreaming && !compactRefreshers.containsKey(tableId)) { + CompactRefresher refresher = + CompactRefresher.create( + true, + initialTable, + newTable -> { + tables.put(tableId, newTable); + write.replace(newTable); + }); + if (refresher != null) { + compactRefreshers.put(tableId, refresher); + } + } + + Long lastRebaseSnapshot = rebasedSnapshots.get(tableId); + if (rebase) { + Preconditions.checkArgument( + write.streamingMode(), + "Rebase records are only supported by streaming compaction."); + if (lastRebaseSnapshot != null && snapshotId < lastRebaseSnapshot) { + // A marker from an older rebase can arrive after a newer marker on another input + // channel. It must never move the writer back to an expired snapshot. + return; + } + // A rebase is emitted once per bucket. Every bucket may carry the first file written + // with a newer schema, so all markers must be offered to the table refresher even + // though the writer baseline is rebuilt only once for the snapshot. + FileStoreTable tableBeforeRefresh = table; + CompactRefresher refresher = compactRefreshers.get(tableId); + if (refresher != null) { + refresher.tryRefresh(files); + table = tables.get(tableId); + } + boolean tableRefreshed = table != tableBeforeRefresh; + if (lastRebaseSnapshot == null || snapshotId > lastRebaseSnapshot || tableRefreshed) { + long restoreSnapshotId = resolveRebaseSnapshotId(tableId, table, snapshotId); + write.rebase( + table, + new FileSystemWriteRestore( + table.coreOptions(), + table.snapshotManager(), + table.store().newScan(), + table.store().newIndexFileHandler(), + restoreSnapshotId, + true)); + // Keep the marker id for ordering. The restore may fall back to a newer snapshot + // when expiration races with writer initialization; all markers from this batch + // must still be accepted. + // The restore target may include newer records which overtook the marker on the + // union input. Treat that target as the watermark for older markers as well. + // Keep marker ordering separate from the actual restore watermark. A newer + // restore snapshot must not suppress later markers from the same baseline. + rebasedSnapshots.put( + tableId, + Math.max( + snapshotId, + rebasedSnapshots.getOrDefault(tableId, Long.MIN_VALUE))); + rebasedThroughSnapshots.put( + tableId, + Math.max( + restoreSnapshotId, + rebasedThroughSnapshots.getOrDefault(tableId, Long.MIN_VALUE))); + } + } else if (write.streamingMode() + && rebasedThroughSnapshots.containsKey(tableId) + && snapshotId <= rebasedThroughSnapshots.getOrDefault(tableId, Long.MIN_VALUE)) { + // A split assigned before the rebase may still be in flight. + return; + } + + if (!rebase + && write.streamingMode() + && table.snapshotManager().isSnapshotExpiredForRebase(snapshotId)) { + // A delta can reach the sink before the broadcast rebase marker. Rebase immediately so + // its deleted files are never added to the writer. + CompactRefresher refresher = compactRefreshers.get(tableId); + if (refresher != null) { + refresher.tryRefresh(files); + table = tables.get(tableId); + } + Snapshot latest = table.snapshotManager().latestSnapshotFromFileSystem(); + Preconditions.checkState( + latest != null && latest.id() > snapshotId, + "Cannot rebase expired snapshot %s for table %s because no newer snapshot is available.", + snapshotId, + tableId); + write.rebase( + table, + new FileSystemWriteRestore( + table.coreOptions(), + table.snapshotManager(), + table.store().newScan(), + table.store().newIndexFileHandler(), + latest.id(), + true)); + rebasedThroughSnapshots.put( + tableId, + Math.max( + latest.id(), + rebasedThroughSnapshots.getOrDefault(tableId, Long.MIN_VALUE))); + return; + } + if (write.streamingMode()) { - write.notifyNewFiles(snapshotId, partition, bucket, files); + if (!rebase) { + write.notifyNewFiles(snapshotId, partition, bucket, files); + CompactRefresher refresher = compactRefreshers.get(tableId); + if (refresher != null) { + refresher.tryRefresh(files); + } + } // The full compact is not supported in streaming mode. - write.compact(partition, bucket, false); + // A rebase split carries the complete snapshot baseline. It must reset the writer but + // must not turn recovery into a full-table compaction. Ordinary delta records below + // continue to schedule the affected buckets for compaction. + if (!rebase) { + write.compact(partition, bucket, false); + } } else { Preconditions.checkArgument( files.isEmpty(), @@ -181,6 +393,34 @@ public void processElement(StreamRecord element) throws Exception { } } + private long resolveRebaseSnapshotId( + Identifier tableId, FileStoreTable table, long snapshotId) { + long highestSnapshotId = + Math.max( + snapshotId, highestObservedSnapshots.getOrDefault(tableId, Long.MIN_VALUE)); + Snapshot latest = table.snapshotManager().latestSnapshotFromFileSystem(); + if (latest != null && latest.id() >= highestSnapshotId) { + // The marker can arrive after newer deltas due to independent network edges. Restoring + // the current latest snapshot makes the rebase monotonic and subsumes those deltas. + return latest.id(); + } + // A stale latest hint is safe to bridge only when the target snapshot file is explicitly + // visible. Do not retain an unseen future id, which would make restore fail spuriously. + if (table.snapshotManager().snapshotExists(highestSnapshotId)) { + return highestSnapshotId; + } + if (latest != null && latest.id() >= snapshotId) { + return latest.id(); + } + LOG.warn( + "Rebase target snapshot {} for table {} is not visible at the sink yet; retaining " + + "the target instead of silently restoring an older snapshot (currently visible {}).", + snapshotId, + table.name(), + latest == null ? null : latest.id()); + return snapshotId; + } + @Override protected List prepareCommit(boolean waitCompaction, long checkpointId) throws IOException { @@ -206,6 +446,33 @@ public void snapshotState(StateSnapshotContext context) throws Exception { write.snapshotState(); } state.snapshotState(); + List> rebases = new LinkedList<>(); + for (Map.Entry entry : rebasedSnapshots.entrySet()) { + rebases.add( + Tuple3.of( + entry.getKey().getDatabaseName(), + entry.getKey().getObjectName(), + entry.getValue())); + } + rebaseSnapshotState.update(rebases); + List> rebasedThrough = new LinkedList<>(); + for (Map.Entry entry : rebasedThroughSnapshots.entrySet()) { + rebasedThrough.add( + Tuple3.of( + entry.getKey().getDatabaseName(), + entry.getKey().getObjectName(), + entry.getValue())); + } + rebasedThroughSnapshotState.update(rebasedThrough); + List> highestObserved = new LinkedList<>(); + for (Map.Entry entry : highestObservedSnapshots.entrySet()) { + highestObserved.add( + Tuple3.of( + entry.getKey().getDatabaseName(), + entry.getKey().getObjectName(), + entry.getValue())); + } + highestObservedSnapshotState.update(highestObserved); } @Override diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreCompactOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreCompactOperator.java index d1cbfd3c642e..3c94a74edcfa 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreCompactOperator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreCompactOperator.java @@ -19,6 +19,7 @@ package org.apache.paimon.flink.sink; import org.apache.paimon.CoreOptions; +import org.apache.paimon.Snapshot; import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.data.BinaryRow; import org.apache.paimon.flink.sink.coordinator.CoordinatedWriteRestore; @@ -26,6 +27,7 @@ import org.apache.paimon.flink.utils.RuntimeContextUtils; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.io.DataFileMetaSerializer; +import org.apache.paimon.operation.FileSystemWriteRestore; import org.apache.paimon.operation.WriteRestore; import org.apache.paimon.options.Options; import org.apache.paimon.table.FileStoreTable; @@ -33,6 +35,9 @@ import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.Preconditions; +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.api.common.typeutils.base.LongSerializer; import org.apache.flink.runtime.jobgraph.OperatorID; import org.apache.flink.runtime.jobgraph.tasks.TaskOperatorEventGateway; import org.apache.flink.runtime.operators.coordination.OperatorCoordinator; @@ -50,6 +55,7 @@ import javax.annotation.Nullable; import java.io.IOException; +import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; @@ -78,6 +84,19 @@ public class StoreCompactOperator extends PrepareCommitOperator> waitToCompact; protected transient @Nullable WriteRestore writeRestore; + /** Snapshot ID of the newest rebase marker already processed. */ + private transient long lastRebaseSnapshotId; + + /** Highest snapshot ID covered by the writer baseline after a rebase. */ + private transient long rebasedThroughSnapshotId; + + /** Highest snapshot ID observed on input, including records which arrived before a marker. */ + private transient long highestObservedSnapshotId; + + private transient ListState rebaseSnapshotState; + private transient ListState rebasedThroughSnapshotState; + private transient ListState highestObservedSnapshotState; + protected transient @Nullable CompactRefresher compactRefresher; public StoreCompactOperator( @@ -100,6 +119,42 @@ public StoreCompactOperator( public void initializeState(StateInitializationContext context) throws Exception { super.initializeState(context); + rebaseSnapshotState = + context.getOperatorStateStore() + .getUnionListState( + new ListStateDescriptor<>( + "paimon_compact_rebase_snapshot", LongSerializer.INSTANCE)); + rebasedThroughSnapshotState = + context.getOperatorStateStore() + .getUnionListState( + new ListStateDescriptor<>( + "paimon_compact_rebase_through_snapshot", + LongSerializer.INSTANCE)); + highestObservedSnapshotState = + context.getOperatorStateStore() + .getUnionListState( + new ListStateDescriptor<>( + "paimon_compact_highest_observed_snapshot", + LongSerializer.INSTANCE)); + lastRebaseSnapshotId = Long.MIN_VALUE; + for (Long snapshotId : rebaseSnapshotState.get()) { + if (snapshotId != null) { + lastRebaseSnapshotId = Math.max(lastRebaseSnapshotId, snapshotId); + } + } + rebasedThroughSnapshotId = Long.MIN_VALUE; + for (Long snapshotId : rebasedThroughSnapshotState.get()) { + if (snapshotId != null) { + rebasedThroughSnapshotId = Math.max(rebasedThroughSnapshotId, snapshotId); + } + } + highestObservedSnapshotId = rebasedThroughSnapshotId; + for (Long snapshotId : highestObservedSnapshotState.get()) { + if (snapshotId != null) { + highestObservedSnapshotId = Math.max(highestObservedSnapshotId, snapshotId); + } + } + // Each job can only have one username and this name must be consistent across restarts. // We cannot use job id as commit username here because user may change job id by creating // a savepoint, stop the job and then resume from savepoint. @@ -131,7 +186,15 @@ public void initializeState(StateInitializationContext context) throws Exception write.setWriteRestore(writeRestore); } this.compactRefresher = - CompactRefresher.create(write.streamingMode(), table, write::replace); + CompactRefresher.create( + write.streamingMode(), + table, + newTable -> { + // Rebase must use the same schema/configuration as a refresh that + // happened while processing an ordinary compaction record. + table = newTable; + this.write.replace(newTable); + }); } public void setWriteRestore(@Nullable WriteRestore writeRestore) { @@ -154,6 +217,61 @@ public void processElement(StreamRecord element) throws Exception { int bucket = record.getInt(2); byte[] serializedFiles = record.getBinary(3); List files = dataFileMetaSerializer.deserializeList(serializedFiles); + boolean rebase = isRebaseRecord(record); + highestObservedSnapshotId = Math.max(highestObservedSnapshotId, snapshotId); + + if (rebase) { + Preconditions.checkArgument( + write.streamingMode(), + "Rebase records are only supported by streaming compaction."); + if (lastRebaseSnapshotId != Long.MIN_VALUE && snapshotId < lastRebaseSnapshotId) { + // A marker from an older rebase can arrive after a newer marker on another input + // channel. It must never move the writer back to an expired snapshot. + return; + } + // A rebase is emitted once per bucket. Every bucket may carry the first file written + // with a newer schema, so all markers must be offered to the refresher even though the + // writer baseline is rebuilt only once for the snapshot. + FileStoreTable tableBeforeRefresh = table; + tryRefreshWrite(files); + boolean tableRefreshed = table != tableBeforeRefresh; + if (snapshotId > lastRebaseSnapshotId || tableRefreshed) { + // Rebase records contain the complete latest-snapshot baseline. Use their file + // metadata to refresh the writer before rebuilding it, otherwise a schema change + // which happened while the source was behind would be lost by the rebase. + long restoreSnapshotId = resolveRebaseSnapshotId(snapshotId); + rebaseWrite(restoreSnapshotId); + // Keep the marker id for ordering. The restore may fall back to a newer snapshot + // when expiration races with writer initialization; all markers from this batch + // must still be accepted. + // Keep marker ordering separate from the actual restore watermark. The restore + // may fall back to a newer snapshot, but that must not make later markers from + // the same baseline look stale (they may carry a newer schema id). + lastRebaseSnapshotId = Math.max(lastRebaseSnapshotId, snapshotId); + } + } else if (write.streamingMode() + && rebasedThroughSnapshotId != Long.MIN_VALUE + && snapshotId <= rebasedThroughSnapshotId) { + // A split assigned before the rebase may still be in flight. Its files belong to the + // expired history and must not be added to the rebased writer. + return; + } + + if (!rebase + && write.streamingMode() + && table.snapshotManager().isSnapshotExpiredForRebase(snapshotId)) { + // A delta may have been assigned before the source emitted its rebase marker. Do not + // add metadata for that expired snapshot to the writer; rebuild from a complete newer + // snapshot and let subsequent deltas advance the writer normally. + tryRefreshWrite(files); + Snapshot latest = table.snapshotManager().latestSnapshotFromFileSystem(); + Preconditions.checkState( + latest != null && latest.id() > snapshotId, + "Cannot rebase expired snapshot %s because no newer snapshot is available.", + snapshotId); + rebaseWrite(latest.id()); + return; + } if (LOG.isDebugEnabled()) { LOG.debug( @@ -164,17 +282,69 @@ public void processElement(StreamRecord element) throws Exception { files); } - if (write.streamingMode()) { + if (write.streamingMode() && !rebase) { write.notifyNewFiles(snapshotId, partition, bucket, files); tryRefreshWrite(files); - } else { + } else if (!write.streamingMode()) { Preconditions.checkArgument( files.isEmpty(), "Batch compact job does not concern what files are compacted. " + "They only need to know what buckets are compacted."); } - waitToCompact.add(Pair.of(partition, bucket)); + // A rebase split carries the complete snapshot baseline. It is used to refresh the + // writer's restore point, not as a request to compact every bucket in that snapshot. + // Compaction remains driven by ordinary delta records after the rebase. + if (!rebase) { + waitToCompact.add(Pair.of(partition, bucket)); + } + } + + private boolean isRebaseRecord(RowData record) { + return record.getArity() > 6 && !record.isNullAt(6) && record.getBoolean(6); + } + + private long resolveRebaseSnapshotId(long snapshotId) { + long highestSnapshotId = Math.max(snapshotId, highestObservedSnapshotId); + Snapshot latest = table.snapshotManager().latestSnapshotFromFileSystem(); + if (latest != null && latest.id() >= highestSnapshotId) { + // The marker can arrive after newer deltas due to independent network edges. Restoring + // the current latest snapshot makes the rebase monotonic and subsumes those deltas. + return latest.id(); + } + // A snapshot can be present while the latest hint is briefly stale. Use it only after an + // explicit existence check; never retain a future id which is not yet visible to restore. + if (table.snapshotManager().snapshotExists(highestSnapshotId)) { + return highestSnapshotId; + } + if (latest != null && latest.id() >= snapshotId) { + return latest.id(); + } + LOG.warn( + "Rebase target snapshot {} is not visible at the sink yet; retaining the target " + + "instead of silently restoring an older snapshot (currently visible {}).", + snapshotId, + latest == null ? null : latest.id()); + return snapshotId; + } + + private void rebaseWrite(long restoreSnapshotId) throws Exception { + waitToCompact.clear(); + write.rebase(table, rebaseWriteRestore(restoreSnapshotId)); + rebasedThroughSnapshotId = Math.max(rebasedThroughSnapshotId, restoreSnapshotId); + } + + private WriteRestore rebaseWriteRestore(long snapshotId) { + if (writeRestore instanceof CoordinatedWriteRestore) { + return ((CoordinatedWriteRestore) writeRestore).withSnapshot(snapshotId); + } + return new FileSystemWriteRestore( + table.coreOptions(), + table.snapshotManager(), + table.store().newScan(), + table.store().newIndexFileHandler(), + snapshotId, + true); } @Override @@ -198,6 +368,9 @@ public void snapshotState(StateSnapshotContext context) throws Exception { super.snapshotState(context); write.snapshotState(); state.snapshotState(); + rebaseSnapshotState.update(Collections.singletonList(lastRebaseSnapshotId)); + rebasedThroughSnapshotState.update(Collections.singletonList(rebasedThroughSnapshotId)); + highestObservedSnapshotState.update(Collections.singletonList(highestObservedSnapshotId)); } @Override diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreSinkWrite.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreSinkWrite.java index 474a996db612..145b01e4944a 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreSinkWrite.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreSinkWrite.java @@ -51,6 +51,18 @@ public interface StoreSinkWrite { void setWriteRestore(WriteRestore writeRestore); + /** + * Replaces the writer and restores it from the supplied snapshot. + * + *

The default implementation is a compatibility fallback for sink writers which cannot + * recreate their internal writer. Such implementations retain the current writer and only + * update its restore state; the {@code table} argument is intentionally unused. Writers which + * support rebase should override this method. + */ + default void rebase(FileStoreTable table, WriteRestore writeRestore) throws Exception { + setWriteRestore(writeRestore); + } + default void setBlobDescriptorReaderFactory(UriReaderFactory uriReaderFactory) {} @Nullable diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreSinkWriteImpl.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreSinkWriteImpl.java index 797f6ad86d04..092fa5ec3f62 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreSinkWriteImpl.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreSinkWriteImpl.java @@ -132,6 +132,18 @@ public void setWriteRestore(WriteRestore writeRestore) { write.withWriteRestore(writeRestore); } + @Override + public void rebase(FileStoreTable table, WriteRestore writeRestore) throws Exception { + if (write != null) { + write.close(); + } + // Do not mutate the shared checkpoint state map here. A rebase replaces the in-memory + // writer, and the next checkpoint overwrites the writer-specific state. Mutating a shared + // short-name entry could discard another table's state in a multi-table compactor. + write = newTableWrite(table); + write.withWriteRestore(writeRestore); + } + @Override public void setBlobDescriptorReaderFactory(UriReaderFactory uriReaderFactory) { this.blobDescriptorReaderFactory = uriReaderFactory; diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/coordinator/CoordinatedWriteRestore.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/coordinator/CoordinatedWriteRestore.java index a9b3c3c9a2a7..9e058d9ae69d 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/coordinator/CoordinatedWriteRestore.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/coordinator/CoordinatedWriteRestore.java @@ -46,10 +46,27 @@ public class CoordinatedWriteRestore implements WriteRestore { private final TaskOperatorEventGateway gateway; private final OperatorID operatorID; + private Long snapshotId; + private final boolean fallbackToLatest; public CoordinatedWriteRestore(TaskOperatorEventGateway gateway, OperatorID operatorID) { + this(gateway, operatorID, null, false); + } + + private CoordinatedWriteRestore( + TaskOperatorEventGateway gateway, + OperatorID operatorID, + Long snapshotId, + boolean fallbackToLatest) { this.gateway = gateway; this.operatorID = operatorID; + this.snapshotId = snapshotId; + this.fallbackToLatest = fallbackToLatest; + } + + /** Returns a restore which asks the coordinator to read one rebase snapshot. */ + public CoordinatedWriteRestore withSnapshot(long snapshotId) { + return new CoordinatedWriteRestore(gateway, operatorID, snapshotId, true); } @Override @@ -79,7 +96,9 @@ public RestoreFiles restoreFiles( bucket, scanDynamicBucketIndex, scanDeleteVectorsIndex, - scanVectorIndexPayloads); + scanVectorIndexPayloads, + snapshotId, + fallbackToLatest); try { byte[] requestContent = serializeObject(coordinationRequest); Integer nextPageToken = null; @@ -103,6 +122,12 @@ public RestoreFiles restoreFiles( ScanCoordinationResponse response = InstantiationUtil.deserializeObject( responseContent, getClass().getClassLoader()); + if (fallbackToLatest && response.snapshot() != null) { + // Reuse the snapshot selected by the coordinator for every bucket in this restore. + // Otherwise a later request could independently fall back to a newer snapshot and + // produce a baseline assembled from different points in time. + snapshotId = response.snapshot().id(); + } return new RestoreFiles( response.snapshot(), response.totalBuckets(), diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/coordinator/ScanCoordinationRequest.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/coordinator/ScanCoordinationRequest.java index b37c00a0ff18..5b2799261e0a 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/coordinator/ScanCoordinationRequest.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/coordinator/ScanCoordinationRequest.java @@ -20,6 +20,8 @@ import org.apache.flink.runtime.operators.coordination.CoordinationRequest; +import javax.annotation.Nullable; + /** Write request to initial data files for partition and bucket. */ public class ScanCoordinationRequest implements CoordinationRequest { @@ -30,6 +32,8 @@ public class ScanCoordinationRequest implements CoordinationRequest { private final boolean scanDynamicBucketIndex; private final boolean scanDeleteVectorsIndex; private final boolean scanVectorIndexPayloads; + @Nullable private final Long snapshotId; + private final boolean fallbackToLatest; public ScanCoordinationRequest( byte[] partition, @@ -37,11 +41,31 @@ public ScanCoordinationRequest( boolean scanDynamicBucketIndex, boolean scanDeleteVectorsIndex, boolean scanVectorIndexPayloads) { + this( + partition, + bucket, + scanDynamicBucketIndex, + scanDeleteVectorsIndex, + scanVectorIndexPayloads, + null, + false); + } + + public ScanCoordinationRequest( + byte[] partition, + int bucket, + boolean scanDynamicBucketIndex, + boolean scanDeleteVectorsIndex, + boolean scanVectorIndexPayloads, + @Nullable Long snapshotId, + boolean fallbackToLatest) { this.partition = partition; this.bucket = bucket; this.scanDynamicBucketIndex = scanDynamicBucketIndex; this.scanDeleteVectorsIndex = scanDeleteVectorsIndex; this.scanVectorIndexPayloads = scanVectorIndexPayloads; + this.snapshotId = snapshotId; + this.fallbackToLatest = fallbackToLatest; } public byte[] partition() { @@ -63,4 +87,13 @@ public boolean scanDeleteVectorsIndex() { public boolean scanVectorIndexPayloads() { return scanVectorIndexPayloads; } + + @Nullable + public Long snapshotId() { + return snapshotId; + } + + public boolean fallbackToLatest() { + return fallbackToLatest; + } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/coordinator/TableWriteCoordinator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/coordinator/TableWriteCoordinator.java index ac7d5475ee80..c0eea346eb67 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/coordinator/TableWriteCoordinator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/coordinator/TableWriteCoordinator.java @@ -28,10 +28,14 @@ import org.apache.paimon.operation.FileStoreScan; import org.apache.paimon.operation.WriteRestore; import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.source.OutOfRangeException; import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Cache; import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Caffeine; +import javax.annotation.Nullable; + +import java.io.FileNotFoundException; import java.io.IOException; import java.time.Duration; import java.util.ArrayList; @@ -116,13 +120,6 @@ private synchronized void refresh() { public synchronized PagedCoordinationResponse scan(PagedCoordinationRequest request) throws IOException { - if (snapshot == null) { - return new PagedCoordinationResponse( - serializeObject( - new ScanCoordinationResponse(null, null, null, null, null, null)), - null); - } - Integer pageToken = request.pageToken(); CoordinationKey requestKey = new CoordinationKey(request.content(), request.requestId()); if (pageToken != null) { @@ -161,36 +158,67 @@ public synchronized PagedCoordinationResponse scan(PagedCoordinationRequest requ public synchronized ScanCoordinationResponse scan(ScanCoordinationRequest request) throws IOException { - if (snapshot == null) { + Snapshot snapshotForRequest = snapshotForRequest(request); + if (snapshotForRequest == null) { return new ScanCoordinationResponse(null, null, null, null, null, null); } BinaryRow partition = deserializeBinaryRow(request.partition()); int bucket = request.bucket(); + for (int attempt = 0; ; attempt++) { + try { + return scanAtSnapshot(request, snapshotForRequest, partition, bucket); + } catch (RuntimeException e) { + if (!shouldFallbackToLatest(request, snapshotForRequest, e) || attempt >= 2) { + throw e; + } + + Snapshot latest = table.snapshotManager().latestSnapshotFromFileSystem(); + if (latest == null || latest.id() <= snapshotForRequest.id()) { + throw e; + } + snapshotForRequest = latest; + } + } + } + private ScanCoordinationResponse scanAtSnapshot( + ScanCoordinationRequest request, + Snapshot snapshotForRequest, + BinaryRow partition, + int bucket) + throws IOException { List restoreFiles = new ArrayList<>(); - List entries = scan.withPartitionBucket(partition, bucket).plan().files(); + List entries = + scan.withSnapshot(snapshotForRequest) + .withPartitionBucket(partition, bucket) + .plan() + .files(); Integer totalBuckets = WriteRestore.extractDataFiles(entries, restoreFiles); IndexFileMeta dynamicBucketIndex = null; if (request.scanDynamicBucketIndex()) { dynamicBucketIndex = - indexFileHandler.scanHashIndex(snapshot, partition, bucket).orElse(null); + indexFileHandler + .scanHashIndex(snapshotForRequest, partition, bucket) + .orElse(null); } List deleteVectorsIndex = null; if (request.scanDeleteVectorsIndex()) { deleteVectorsIndex = - indexFileHandler.scan(snapshot, DELETION_VECTORS_INDEX, partition, bucket); + indexFileHandler.scan( + snapshotForRequest, DELETION_VECTORS_INDEX, partition, bucket); } List vectorIndexPayloads = null; if (request.scanVectorIndexPayloads()) { - vectorIndexPayloads = indexFileHandler.scanSourceIndexes(snapshot, partition, bucket); + vectorIndexPayloads = + indexFileHandler.scanSourceIndexes(snapshotForRequest, partition, bucket); } return new ScanCoordinationResponse( - snapshot, + snapshotForRequest, totalBuckets, restoreFiles, dynamicBucketIndex, @@ -198,6 +226,70 @@ public synchronized ScanCoordinationResponse scan(ScanCoordinationRequest reques vectorIndexPayloads); } + private boolean shouldFallbackToLatest( + ScanCoordinationRequest request, Snapshot snapshot, RuntimeException failure) { + return request.fallbackToLatest() + && request.snapshotId() != null + && containsExpiredSnapshotFailure(failure) + && isExpiredOrSuperseded(snapshot.id()); + } + + @Nullable + private Snapshot snapshotForRequest(ScanCoordinationRequest request) { + if (request.snapshotId() == null) { + return snapshot; + } + + try { + Snapshot requested = table.snapshotManager().snapshot(request.snapshotId()); + if (requested != null) { + return requested; + } + } catch (RuntimeException e) { + if (!request.fallbackToLatest() + || !containsExpiredSnapshotFailure(e) + || !isExpiredOrSuperseded(request.snapshotId())) { + throw e; + } + } + + if (request.fallbackToLatest()) { + Snapshot latest = table.snapshotManager().latestSnapshotFromFileSystem(); + if (latest != null && latest.id() > request.snapshotId()) { + return latest; + } + } + throw new OutOfRangeException( + "Rebase snapshot " + request.snapshotId() + " is no longer available."); + } + + private boolean isExpiredOrSuperseded(long requestedSnapshotId) { + if (table.snapshotManager().isSnapshotExpired(requestedSnapshotId)) { + return true; + } + try { + // Expiration cleans manifests before deleting the snapshot file and advancing the + // earliest hint. A newer complete snapshot is therefore the reliable signal that a + // missing manifest belongs to that race rather than an isolated read failure. + Snapshot latest = table.snapshotManager().latestSnapshotFromFileSystem(); + return latest != null && latest.id() > requestedSnapshotId; + } catch (RuntimeException ignored) { + return false; + } + } + + private boolean containsExpiredSnapshotFailure(Throwable failure) { + Throwable current = failure; + while (current != null) { + if (current instanceof FileNotFoundException + || current instanceof OutOfRangeException) { + return true; + } + current = current.getCause(); + } + return false; + } + public synchronized long latestCommittedIdentifier(String user) { return latestCommittedIdentifiers.computeIfAbsent(user, this::computeLatestIdentifier); } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/CompactorSourceBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/CompactorSourceBuilder.java index 9bc38ebbbb2e..31e38fe8f3ef 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/CompactorSourceBuilder.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/CompactorSourceBuilder.java @@ -168,6 +168,9 @@ public DataStreamSource build() { SingleOutputStreamOperator filterStream = dataStream.filter( rowData -> { + if (isRebaseRecord(rowData)) { + return true; + } BinaryRow partition = deserializeBinaryRow(rowData.getBinary(1)); return partitionInfo.get(partition) <= historyMilli; }); @@ -189,6 +192,9 @@ public DataStreamSource build() { SingleOutputStreamOperator filterStream = dataStream.filter( rowData -> { + if (isRebaseRecord(rowData)) { + return true; + } LocalDateTime expireDateTime = LocalDateTime.now().minus(expireTime); BinaryRow partition = deserializeBinaryRow(rowData.getBinary(1)); @@ -202,6 +208,10 @@ public DataStreamSource build() { return dataStream; } + private static boolean isRebaseRecord(RowData rowData) { + return rowData.getArity() > 6 && !rowData.isNullAt(6) && rowData.getBoolean(6); + } + private Map streamingCompactOptions() { // set 'streaming-compact' and remove 'scan.bounded.watermark' return new HashMap() { diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/ContinuousFileSplitEnumerator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/ContinuousFileSplitEnumerator.java index 75c75cfddb71..0a99531f5412 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/ContinuousFileSplitEnumerator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/ContinuousFileSplitEnumerator.java @@ -28,9 +28,11 @@ import org.apache.paimon.table.sink.ChannelComputer; import org.apache.paimon.table.source.ChainSplit; import org.apache.paimon.table.source.DataSplit; +import org.apache.paimon.table.source.DataTableStreamScan; import org.apache.paimon.table.source.EndOfScanException; import org.apache.paimon.table.source.IncrementalSplit; import org.apache.paimon.table.source.SnapshotNotExistPlan; +import org.apache.paimon.table.source.Split; import org.apache.paimon.table.source.StreamTableScan; import org.apache.paimon.table.source.TableScan; @@ -172,7 +174,7 @@ public void addReader(int subtaskId) { } @Override - public void handleSplitRequest(int subtaskId, @Nullable String requesterHostname) { + public synchronized void handleSplitRequest(int subtaskId, @Nullable String requesterHostname) { readersAwaitingSplit.add(subtaskId); assignSplits(); // if current task assigned no split, we check conditions to scan one more time @@ -186,7 +188,7 @@ public void handleSplitRequest(int subtaskId, @Nullable String requesterHostname } @Override - public void handleSourceEvent(int subtaskId, SourceEvent sourceEvent) { + public synchronized void handleSourceEvent(int subtaskId, SourceEvent sourceEvent) { if (sourceEvent instanceof ReaderConsumeProgressEvent) { consumerProgressCalculator.updateConsumeProgress( subtaskId, (ReaderConsumeProgressEvent) sourceEvent); @@ -196,13 +198,13 @@ public void handleSourceEvent(int subtaskId, SourceEvent sourceEvent) { } @Override - public void addSplitsBack(List splits, int subtaskId) { + public synchronized void addSplitsBack(List splits, int subtaskId) { LOG.debug("File Source Enumerator adds splits back: {}", splits); splitAssigner.addSplitsBack(subtaskId, splits); } @Override - public PendingSplitsCheckpoint snapshotState(long checkpointId) throws Exception { + public synchronized PendingSplitsCheckpoint snapshotState(long checkpointId) throws Exception { List splits = new ArrayList<>(splitAssigner.remainingSplits()); final PendingSplitsCheckpoint checkpoint = new PendingSplitsCheckpoint(splits, nextSnapshotId); @@ -218,7 +220,7 @@ public PendingSplitsCheckpoint snapshotState(long checkpointId) throws Exception } @Override - public void notifyCheckpointComplete(long checkpointId) throws Exception { + public synchronized void notifyCheckpointComplete(long checkpointId) throws Exception { consumerProgressCalculator .notifyCheckpointComplete(checkpointId) .ifPresent(scan::notifyCheckpointComplete); @@ -231,10 +233,26 @@ public void notifyCheckpointComplete(long checkpointId) throws Exception { // context.callAsync will invoke this. This method runs in workerExecutorThreadPool in // parallelism. protected synchronized Optional scanNextSnapshot() { - if (splitAssigner.numberOfRemainingSplits() >= splitMaxNum) { + boolean mustRebase = false; + if (scan instanceof DataTableStreamScan) { + DataTableStreamScan dataScan = (DataTableStreamScan) scan; + Long expiredPendingSnapshot = expiredPendingSnapshot(dataScan); + if (expiredPendingSnapshot != null) { + splitAssigner.clear(); + scan.restore(expiredPendingSnapshot); + mustRebase = true; + } else if (dataScan.isSnapshotExpiredForRebase()) { + Long checkpoint = dataScan.checkpoint(); + if (checkpoint != null) { + mustRebase = true; + } + } + } + + if (!mustRebase && splitAssigner.numberOfRemainingSplits() >= splitMaxNum) { return Optional.empty(); } - if (maxSnapshotCount > 0 && handledSnapshotCount >= maxSnapshotCount) { + if (!mustRebase && maxSnapshotCount > 0 && handledSnapshotCount >= maxSnapshotCount) { LOG.debug( "There is {} in-flight snapshot, pending to scan next snapshot.", handledSnapshotCount); @@ -253,9 +271,32 @@ protected synchronized Optional scanNextSnapshot() { return Optional.of(new PlanWithNextSnapshotId(plan, nextSnapshotId)); } - // this mothod could not be synchronized, because it runs in coordinatorThread, which will make - // it serialize. - protected void processDiscoveredSplits( + @Nullable + protected Long expiredPendingSnapshot(DataTableStreamScan dataScan) { + long earliestPendingSnapshot = Long.MAX_VALUE; + for (FileStoreSourceSplit sourceSplit : splitAssigner.remainingSplits()) { + if (sourceSplit.split() instanceof DataSplit) { + long snapshotId = ((DataSplit) sourceSplit.split()).snapshotId(); + earliestPendingSnapshot = Math.min(earliestPendingSnapshot, snapshotId); + } + } + return earliestPendingSnapshot != Long.MAX_VALUE + && dataScan.isSnapshotExpiredForRebase(earliestPendingSnapshot) + ? earliestPendingSnapshot + : null; + } + + /** Whether pending splits belong to snapshots which are no longer retained. */ + protected boolean hasExpiredPendingSnapshot() { + if (!(scan instanceof DataTableStreamScan)) { + return false; + } + return expiredPendingSnapshot((DataTableStreamScan) scan) != null; + } + + // Keep discovery result application serialized with snapshot scanning. The callback runs on + // the coordinator thread, while scanNextSnapshot runs in Flink's worker pool. + protected synchronized void processDiscoveredSplits( Optional planWithNextSnapshotIdOptional, Throwable error) { if (error != null) { if (error instanceof EndOfScanException) { @@ -286,7 +327,11 @@ protected void processDiscoveredSplits( return; } - addSplits(splitGenerator.createSplits(plan)); + List discoveredSplits = splitGenerator.createSplits(plan); + if (containsRebaseSplit(discoveredSplits)) { + splitAssigner.clear(); + } + addSplits(discoveredSplits); assignSplits(); } @@ -430,6 +475,24 @@ protected boolean noMoreSplits() { return finished; } + protected boolean containsRebaseSplit(Collection splits) { + for (FileStoreSourceSplit split : splits) { + if (split.split() instanceof DataSplit && ((DataSplit) split.split()).isRebase()) { + return true; + } + } + return false; + } + + protected boolean containsRebaseSplit(TableScan.Plan plan) { + for (Split split : plan.splits()) { + if (split instanceof DataSplit && ((DataSplit) split).isRebase()) { + return true; + } + } + return false; + } + /** The result of scan. */ protected static class PlanWithNextSnapshotId { private final TableScan.Plan plan; diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/ContinuousFileStoreSource.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/ContinuousFileStoreSource.java index 4da7ea9c4fe7..05f300ebf9d5 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/ContinuousFileStoreSource.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/ContinuousFileStoreSource.java @@ -23,6 +23,8 @@ import org.apache.paimon.flink.NestedProjectedRowData; import org.apache.paimon.flink.metrics.FlinkMetricRegistry; import org.apache.paimon.options.Options; +import org.apache.paimon.table.source.DataSplit; +import org.apache.paimon.table.source.DataTableStreamScan; import org.apache.paimon.table.source.ReadBuilder; import org.apache.paimon.table.source.StreamDataTableScan; import org.apache.paimon.table.source.StreamTableScan; @@ -36,6 +38,7 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.Map; /** Unbounded {@link FlinkSource} for reading records. It continuously monitors new snapshots. */ @@ -92,9 +95,40 @@ public SplitEnumerator restoreEnu .withMetricRegistry(new FlinkMetricRegistry(context.metricGroup())); } scan.restore(nextSnapshotId); + if (scan instanceof DataTableStreamScan) { + DataTableStreamScan dataScan = (DataTableStreamScan) scan; + // Pending splits belong to the expired history and must not be assigned before the + // scan emits its latest-snapshot rebase plan. A rebase split can expire even when the + // checkpointed next snapshot itself is still retained (the split target is one id + // behind it), so inspect both pieces of state. + Long expiredPendingSnapshot = expiredPendingSnapshot(splits, dataScan); + if (expiredPendingSnapshot != null) { + splits = Collections.emptyList(); + nextSnapshotId = expiredPendingSnapshot; + scan.restore(nextSnapshotId); + } else if (dataScan.isSnapshotExpiredForRebase()) { + splits = Collections.emptyList(); + } + } return buildEnumerator(context, splits, nextSnapshotId, scan); } + @Nullable + private Long expiredPendingSnapshot( + Collection splits, DataTableStreamScan scan) { + long earliestPendingSnapshot = Long.MAX_VALUE; + for (FileStoreSourceSplit sourceSplit : splits) { + if (sourceSplit.split() instanceof DataSplit) { + DataSplit split = (DataSplit) sourceSplit.split(); + earliestPendingSnapshot = Math.min(earliestPendingSnapshot, split.snapshotId()); + } + } + return earliestPendingSnapshot != Long.MAX_VALUE + && scan.isSnapshotExpiredForRebase(earliestPendingSnapshot) + ? earliestPendingSnapshot + : null; + } + @Nullable private SplitEnumeratorMetricGroup metricGroup(SplitEnumeratorContext context) { try { diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/align/AlignedContinuousFileSplitEnumerator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/align/AlignedContinuousFileSplitEnumerator.java index c728134c07ed..5cfde08031a7 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/align/AlignedContinuousFileSplitEnumerator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/align/AlignedContinuousFileSplitEnumerator.java @@ -24,8 +24,10 @@ import org.apache.paimon.flink.source.assigners.AlignedSplitAssigner; import org.apache.paimon.flink.source.assigners.SplitAssigner; import org.apache.paimon.table.source.DataSplit; +import org.apache.paimon.table.source.DataTableStreamScan; import org.apache.paimon.table.source.EndOfScanException; import org.apache.paimon.table.source.SnapshotNotExistPlan; +import org.apache.paimon.table.source.Split; import org.apache.paimon.table.source.StreamTableScan; import org.apache.paimon.table.source.TableScan; import org.apache.paimon.utils.Preconditions; @@ -83,7 +85,7 @@ public class AlignedContinuousFileSplitEnumerator extends ContinuousFileSplitEnu private Long lastConsumedSnapshotId; - private boolean closed; + private volatile boolean closed; public AlignedContinuousFileSplitEnumerator( SplitEnumeratorContext context, @@ -162,6 +164,17 @@ public void addSplitsBack(List splits, int subtaskId) { @Override public PendingSplitsCheckpoint snapshotState(long checkpointId) throws Exception { + synchronized (this) { + Long expiredPendingSnapshot = expiredPendingSnapshot(); + if (expiredPendingSnapshot != null) { + pendingPlans.clear(); + alignedAssigner.clear(); + // The scan cursor may already be ahead of the split which was just discarded. + // Restore it to the expired snapshot so the next scan emits a rebase plan instead + // of silently continuing from a later snapshot. + scan.restore(expiredPendingSnapshot); + } + } if (!alignedAssigner.isAligned() && !closed) { synchronized (lock) { if (pendingPlans.isEmpty()) { @@ -172,22 +185,35 @@ public PendingSplitsCheckpoint snapshotState(long checkpointId) throws Exception "Timeout while waiting for snapshot from paimon source."); } } - PlanWithNextSnapshotId pendingPlan = pendingPlans.poll(); - addSplits(splitGenerator.createSplits(Objects.requireNonNull(pendingPlan).plan())); - nextSnapshotId = pendingPlan.nextSnapshotId(); - assignSplits(); + synchronized (this) { + if (!alignedAssigner.isAligned()) { + PlanWithNextSnapshotId pendingPlan = pendingPlans.poll(); + addSplits( + splitGenerator.createSplits( + Objects.requireNonNull(pendingPlan).plan())); + nextSnapshotId = pendingPlan.nextSnapshotId(); + assignSplits(); + } + } + } + + List remainingSplits; + Long checkpointNextSnapshotId; + synchronized (this) { + Preconditions.checkArgument(alignedAssigner.isAligned()); + lastConsumedSnapshotId = alignedAssigner.getNextSnapshotId(0).orElse(null); + alignedAssigner.removeFirst(); + currentCheckpointId = checkpointId; + remainingSplits = new ArrayList<>(alignedAssigner.remainingSplits()); + checkpointNextSnapshotId = nextSnapshotId; } - Preconditions.checkArgument(alignedAssigner.isAligned()); - lastConsumedSnapshotId = alignedAssigner.getNextSnapshotId(0).orElse(null); - alignedAssigner.removeFirst(); - currentCheckpointId = checkpointId; // send checkpoint event to the source reader CheckpointEvent event = new CheckpointEvent(checkpointId); for (int i = 0; i < context.currentParallelism(); i++) { context.sendEventToSourceReader(i, event); } - return new PendingSplitsCheckpoint(alignedAssigner.remainingSplits(), nextSnapshotId); + return new PendingSplitsCheckpoint(remainingSplits, checkpointNextSnapshotId); } @Override @@ -198,7 +224,7 @@ public void notifyCheckpointAborted(long checkpointId) { } @Override - public void notifyCheckpointComplete(long checkpointId) { + public synchronized void notifyCheckpointComplete(long checkpointId) { currentCheckpointId = Long.MIN_VALUE; Long nextSnapshot = lastConsumedSnapshotId == null ? null : lastConsumedSnapshotId + 1; scan.notifyCheckpointComplete(nextSnapshot); @@ -207,12 +233,30 @@ public void notifyCheckpointComplete(long checkpointId) { // ------------------------------------------------------------------------ @Override - protected Optional scanNextSnapshot() { - if (pendingPlans.remainingCapacity() > 0) { + protected synchronized Optional scanNextSnapshot() { + // Even when the queue is full, inspect pending splits for expiration. Otherwise an + // expired plan could be handed out by snapshotState before the next scan has a chance to + // emit the rebase control plan. + Long expiredPendingSnapshot = expiredPendingSnapshot(); + boolean expiredPendingPlan = expiredPendingSnapshot != null; + if (expiredPendingPlan) { + pendingPlans.clear(); + alignedAssigner.clear(); + scan.restore(expiredPendingSnapshot); + } + if (pendingPlans.remainingCapacity() > 0 + || expiredPendingPlan + || hasExpiredPendingSnapshot()) { Optional scannedPlanOptional = super.scanNextSnapshot(); if (scannedPlanOptional.isPresent()) { PlanWithNextSnapshotId scannedPlan = scannedPlanOptional.get(); if (!(scannedPlan.plan() instanceof SnapshotNotExistPlan)) { + if (containsRebaseSplit(scannedPlan.plan())) { + synchronized (lock) { + pendingPlans.clear(); + } + alignedAssigner.clear(); + } synchronized (lock) { pendingPlans.add(scannedPlan); lock.notifyAll(); @@ -223,8 +267,38 @@ protected Optional scanNextSnapshot() { return Optional.empty(); } + @Nullable + private Long expiredPendingSnapshot() { + if (!(scan instanceof DataTableStreamScan)) { + return null; + } + DataTableStreamScan dataScan = (DataTableStreamScan) scan; + // A plan may already have moved from pendingPlans into the aligned assigner while the + // checkpoint is being prepared. Inspect both queues so an expired split cannot be + // persisted and assigned after recovery without first producing a rebase plan. + long earliestPendingSnapshot = Long.MAX_VALUE; + for (FileStoreSourceSplit split : alignedAssigner.remainingSplits()) { + if (split.split() instanceof DataSplit + && dataScan.isSnapshotExpiredForRebase( + ((DataSplit) split.split()).snapshotId())) { + earliestPendingSnapshot = + Math.min(earliestPendingSnapshot, ((DataSplit) split.split()).snapshotId()); + } + } + for (PlanWithNextSnapshotId pendingPlan : pendingPlans) { + for (Split split : pendingPlan.plan().splits()) { + if (split instanceof DataSplit + && dataScan.isSnapshotExpiredForRebase(((DataSplit) split).snapshotId())) { + earliestPendingSnapshot = + Math.min(earliestPendingSnapshot, ((DataSplit) split).snapshotId()); + } + } + } + return earliestPendingSnapshot == Long.MAX_VALUE ? null : earliestPendingSnapshot; + } + @Override - protected void processDiscoveredSplits( + protected synchronized void processDiscoveredSplits( Optional ignore, Throwable error) { if (error != null) { if (error instanceof EndOfScanException) { @@ -254,7 +328,11 @@ protected void processDiscoveredSplits( PLACEHOLDER_SPLIT, new PlaceholderSplit(nextSnapshotId - 1)))); } else { - addSplits(splitGenerator.createSplits(plan)); + List discoveredSplits = splitGenerator.createSplits(plan); + if (containsRebaseSplit(discoveredSplits)) { + splitAssigner.clear(); + } + addSplits(discoveredSplits); } } assignSplits(); diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/assigners/AlignedSplitAssigner.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/assigners/AlignedSplitAssigner.java index 648758f83846..55fd3e03c969 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/assigners/AlignedSplitAssigner.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/assigners/AlignedSplitAssigner.java @@ -52,7 +52,7 @@ public AlignedSplitAssigner() { } @Override - public List getNext(int subtask, @Nullable String hostname) { + public synchronized List getNext(int subtask, @Nullable String hostname) { PendingSnapshot head = pendingSplitAssignment.peek(); if (head != null && !head.isPlaceHolder) { List subtaskSplits = head.remove(subtask); @@ -65,7 +65,7 @@ public List getNext(int subtask, @Nullable String hostname } @Override - public void addSplit(int subtask, FileStoreSourceSplit splits) { + public synchronized void addSplit(int subtask, FileStoreSourceSplit splits) { long snapshotId = ((DataSplit) splits.split()).snapshotId(); PendingSnapshot last = pendingSplitAssignment.peekLast(); boolean isPlaceholder = splits.split() instanceof PlaceholderSplit; @@ -80,7 +80,7 @@ public void addSplit(int subtask, FileStoreSourceSplit splits) { } @Override - public void addSplitsBack(int suggestedTask, List splits) { + public synchronized void addSplitsBack(int suggestedTask, List splits) { if (splits.isEmpty()) { return; } @@ -99,7 +99,7 @@ public void addSplitsBack(int suggestedTask, List splits) } @Override - public Collection remainingSplits() { + public synchronized Collection remainingSplits() { List remainingSplits = new ArrayList<>(); for (PendingSnapshot pendingSnapshot : pendingSplitAssignment) { pendingSnapshot.subtaskSplits.values().forEach(remainingSplits::addAll); @@ -108,26 +108,32 @@ public Collection remainingSplits() { } @Override - public Optional getNextSnapshotId(int subtask) { + public synchronized void clear() { + pendingSplitAssignment.clear(); + numberOfPendingSplits.set(0); + } + + @Override + public synchronized Optional getNextSnapshotId(int subtask) { PendingSnapshot head = pendingSplitAssignment.peek(); return Optional.ofNullable(head != null ? head.snapshotId : null); } @Override - public int numberOfRemainingSplits() { + public synchronized int numberOfRemainingSplits() { return numberOfPendingSplits.get(); } - public boolean isAligned() { + public synchronized boolean isAligned() { PendingSnapshot head = pendingSplitAssignment.peek(); return head != null && head.empty(); } - public int remainingSnapshots() { + public synchronized int remainingSnapshots() { return pendingSplitAssignment.size(); } - public void removeFirst() { + public synchronized void removeFirst() { PendingSnapshot head = pendingSplitAssignment.poll(); Preconditions.checkArgument( head != null && head.empty(), diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/assigners/DynamicPartitionPruningAssigner.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/assigners/DynamicPartitionPruningAssigner.java index 9221f1f27eb6..7d44821459b6 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/assigners/DynamicPartitionPruningAssigner.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/assigners/DynamicPartitionPruningAssigner.java @@ -54,7 +54,7 @@ public DynamicPartitionPruningAssigner( } @Override - public List getNext(int subtask, @Nullable String hostname) { + public synchronized List getNext(int subtask, @Nullable String hostname) { List sourceSplits = innerAssigner.getNext(subtask, hostname); while (!sourceSplits.isEmpty()) { List filtered = @@ -69,24 +69,29 @@ public List getNext(int subtask, @Nullable String hostname } @Override - public void addSplit(int suggestedTask, FileStoreSourceSplit splits) { + public synchronized void addSplit(int suggestedTask, FileStoreSourceSplit splits) { if (filter(splits)) { innerAssigner.addSplit(suggestedTask, splits); } } @Override - public void addSplitsBack(int subtask, List splits) { + public synchronized void addSplitsBack(int subtask, List splits) { innerAssigner.addSplitsBack(subtask, splits); } @Override - public Collection remainingSplits() { + public synchronized Collection remainingSplits() { return innerAssigner.remainingSplits().stream() .filter(this::filter) .collect(Collectors.toList()); } + @Override + public synchronized void clear() { + innerAssigner.clear(); + } + public static SplitAssigner createDynamicPartitionPruningAssignerIfNeeded( int subtaskId, SplitAssigner oriAssigner, @@ -105,12 +110,12 @@ public static SplitAssigner createDynamicPartitionPruningAssignerIfNeeded( } @Override - public Optional getNextSnapshotId(int subtask) { + public synchronized Optional getNextSnapshotId(int subtask) { return innerAssigner.getNextSnapshotId(subtask); } @Override - public int numberOfRemainingSplits() { + public synchronized int numberOfRemainingSplits() { return innerAssigner.numberOfRemainingSplits(); } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/assigners/FIFOSplitAssigner.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/assigners/FIFOSplitAssigner.java index a2f0b983cdd8..47c8bb395e73 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/assigners/FIFOSplitAssigner.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/assigners/FIFOSplitAssigner.java @@ -46,18 +46,18 @@ public FIFOSplitAssigner(Collection splits) { } @Override - public List getNext(int subtask, @Nullable String hostname) { + public synchronized List getNext(int subtask, @Nullable String hostname) { FileStoreSourceSplit split = pendingSplitAssignment.poll(); return split == null ? Collections.emptyList() : Collections.singletonList(split); } @Override - public void addSplit(int suggestedTask, FileStoreSourceSplit split) { + public synchronized void addSplit(int suggestedTask, FileStoreSourceSplit split) { pendingSplitAssignment.add(split); } @Override - public void addSplitsBack(int subtask, List splits) { + public synchronized void addSplitsBack(int subtask, List splits) { ListIterator iterator = splits.listIterator(splits.size()); while (iterator.hasPrevious()) { pendingSplitAssignment.addFirst(iterator.previous()); @@ -65,19 +65,24 @@ public void addSplitsBack(int subtask, List splits) { } @Override - public Collection remainingSplits() { + public synchronized Collection remainingSplits() { return new ArrayList<>(pendingSplitAssignment); } @Override - public Optional getNextSnapshotId(int subtask) { + public synchronized void clear() { + pendingSplitAssignment.clear(); + } + + @Override + public synchronized Optional getNextSnapshotId(int subtask) { return pendingSplitAssignment.isEmpty() ? Optional.empty() : getSnapshotId(pendingSplitAssignment.peekFirst()); } @Override - public int numberOfRemainingSplits() { + public synchronized int numberOfRemainingSplits() { return pendingSplitAssignment.size(); } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/assigners/PreAssignSplitAssigner.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/assigners/PreAssignSplitAssigner.java index 24ac4a291164..eaab98dd9eef 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/assigners/PreAssignSplitAssigner.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/assigners/PreAssignSplitAssigner.java @@ -149,7 +149,7 @@ public PreAssignSplitAssigner( } @Override - public List getNext(int subtask, @Nullable String hostname) { + public synchronized List getNext(int subtask, @Nullable String hostname) { // The following batch assignment operation is for two purposes: // To distribute splits evenly when batch reading to prevent a few tasks from reading all // the data (for example, the current resource can only schedule part of the tasks). @@ -163,13 +163,13 @@ public List getNext(int subtask, @Nullable String hostname } @Override - public void addSplit(int suggestedTask, FileStoreSourceSplit split) { + public synchronized void addSplit(int suggestedTask, FileStoreSourceSplit split) { pendingSplitAssignment.computeIfAbsent(suggestedTask, k -> new LinkedList<>()).add(split); numberOfPendingSplits.incrementAndGet(); } @Override - public void addSplitsBack(int subtask, List splits) { + public synchronized void addSplitsBack(int subtask, List splits) { LinkedList remainingSplits = pendingSplitAssignment.computeIfAbsent(subtask, k -> new LinkedList<>()); ListIterator iterator = splits.listIterator(splits.size()); @@ -180,12 +180,18 @@ public void addSplitsBack(int subtask, List splits) { } @Override - public Collection remainingSplits() { + public synchronized Collection remainingSplits() { List splits = new ArrayList<>(); pendingSplitAssignment.values().forEach(splits::addAll); return splits; } + @Override + public synchronized void clear() { + pendingSplitAssignment.clear(); + numberOfPendingSplits.set(0); + } + /** * this method only reload restore for batch execute, because in streaming mode, we need to * assign certain bucket to certain task. @@ -266,7 +272,7 @@ private long weight() { } @Override - public Optional getNextSnapshotId(int subtask) { + public synchronized Optional getNextSnapshotId(int subtask) { LinkedList pendingSplits = pendingSplitAssignment.get(subtask); return (pendingSplits == null || pendingSplits.isEmpty()) ? Optional.empty() @@ -274,7 +280,7 @@ public Optional getNextSnapshotId(int subtask) { } @Override - public int numberOfRemainingSplits() { + public synchronized int numberOfRemainingSplits() { return numberOfPendingSplits.get(); } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/assigners/SplitAssigner.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/assigners/SplitAssigner.java index 2ce3af5d765f..65f8c736a679 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/assigners/SplitAssigner.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/assigners/SplitAssigner.java @@ -52,6 +52,9 @@ public interface SplitAssigner { /** Gets the remaining splits that this assigner has pending. */ Collection remainingSplits(); + /** Discards all splits which have not been assigned yet. */ + default void clear() {} + /** Gets the snapshot id of the next split. */ Optional getNextSnapshotId(int subtask); diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/CompactDatabaseActionITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/CompactDatabaseActionITCase.java index acb0267440d7..ddc8ea5207eb 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/CompactDatabaseActionITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/CompactDatabaseActionITCase.java @@ -24,6 +24,9 @@ import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.GenericRow; import org.apache.paimon.flink.FlinkConnectorOptions; +import org.apache.paimon.flink.LogicalTypeConversion; +import org.apache.paimon.flink.sink.CompactorSinkBuilder; +import org.apache.paimon.flink.source.CompactorSourceBuilder; import org.apache.paimon.schema.Schema; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.sink.StreamTableCommit; @@ -32,6 +35,7 @@ import org.apache.paimon.table.source.DataSplit; import org.apache.paimon.table.source.StreamTableScan; import org.apache.paimon.table.source.TableScan; +import org.apache.paimon.table.system.CompactBucketsTable; import org.apache.paimon.types.DataType; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; @@ -39,8 +43,19 @@ import org.apache.paimon.shade.guava30.com.google.common.collect.Lists; +import org.apache.flink.api.common.JobStatus; import org.apache.flink.core.execution.JobClient; +import org.apache.flink.runtime.checkpoint.CheckpointOptions; +import org.apache.flink.runtime.state.CheckpointStreamFactory; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.operators.OperatorSnapshotFutures; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.runtime.typeutils.InternalTypeInfo; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import org.junit.jupiter.params.ParameterizedTest; @@ -58,6 +73,10 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import java.util.stream.Stream; import static org.apache.paimon.utils.CommonTestUtils.waitUtil; @@ -78,13 +97,29 @@ public class CompactDatabaseActionITCase extends CompactActionITCaseBase { new String[] {"k", "v", "hh", "dt"}); private static Stream testData() { - return Stream.of( - Arguments.of("combined", "action"), - Arguments.of("divided", "action"), - Arguments.of("combined", "procedure_indexed"), - Arguments.of("divided", "procedure_indexed"), - Arguments.of("combined", "procedure_named"), - Arguments.of("divided", "procedure_named")); + Stream all = + Stream.of( + Arguments.of("combined", "action"), + Arguments.of("divided", "action"), + Arguments.of("combined", "procedure_indexed"), + Arguments.of("divided", "procedure_indexed"), + Arguments.of("combined", "procedure_named"), + Arguments.of("divided", "procedure_named")); + String selected = System.getProperty("paimon.compact.test.case"); + return selected == null ? all : selectTestCase(all, selected); + } + + private static Stream selectTestCase(Stream all, String selected) { + List matches = + all.filter( + argument -> + (argument.get()[0] + ":" + argument.get()[1]) + .equals(selected)) + .collect(Collectors.toList()); + assertThat(matches) + .as("paimon.compact.test.case must identify a known test case") + .isNotEmpty(); + return matches.stream(); } protected FileStoreTable createTable( @@ -1073,6 +1108,315 @@ public void testCombinedModeWithDynamicOptions(String type) throws Exception { jobClient.cancel(); } + /** + * Verifies that a dedicated streaming compactor can recover from a checkpoint whose next + * snapshot has expired. The failed checkpoint is injected after the first compact commit and + * after several append snapshots have forced retention, so recovery must rebuild the source + * baseline and replace the sink writer before processing the next append. + */ + @Test + @Timeout(value = 240, unit = TimeUnit.SECONDS) + public void testStreamingCompactRecoversAfterExpiredSnapshot() throws Exception { + FailOnCheckpointOperator.reset(); + + Map options = new HashMap<>(); + options.put(CoreOptions.CHANGELOG_PRODUCER.key(), "full-compaction"); + options.put(CoreOptions.FULL_COMPACTION_DELTA_COMMITS.key(), "1"); + options.put(CoreOptions.WRITE_ONLY.key(), "true"); + options.put(CoreOptions.BUCKET.key(), "1"); + options.put(CoreOptions.SNAPSHOT_NUM_RETAINED_MIN.key(), "1"); + options.put(CoreOptions.SNAPSHOT_NUM_RETAINED_MAX.key(), "1"); + // Keep the source at its checkpointed snapshot while the test creates newer snapshots. + options.put(CoreOptions.CONTINUOUS_DISCOVERY_INTERVAL.key(), "20s"); + // Match the options used by the production dedicated streaming compactor. Compaction is + // explicitly triggered by the sink for each bucket, while new files are kept visible to + // the writer instead of being compacted eagerly by the write path. + options.put(CoreOptions.NUM_SORTED_RUNS_STOP_TRIGGER.key(), "2147483647"); + options.put(CoreOptions.SORT_SPILL_THRESHOLD.key(), "10"); + options.put(CoreOptions.LOOKUP_WAIT.key(), "false"); + + FileStoreTable table = + createTable( + database, + "expired_snapshot_recovery", + Arrays.asList("dt", "hh"), + Arrays.asList("dt", "hh", "k"), + options); + + // The first source plan starts from the earliest append snapshot because no compact + // snapshot exists yet. + StreamWriteBuilder streamWriteBuilder = + table.newStreamWriteBuilder().withCommitUser(commitUser); + write = streamWriteBuilder.newWrite(); + commit = streamWriteBuilder.newCommit(); + writeData(rowData(1, 100, 15, BinaryString.fromString("20221208"))); + write.close(); + commit.close(); + write = null; + commit = null; + + // A single append file is not a compaction candidate. Add a second append snapshot before + // starting the dedicated job so its first plan can produce a compact snapshot. + appendRow(table, rowData(2, 101, 15, BinaryString.fromString("20221208"))); + + FileStoreTable compactionTable = + table.copy(Collections.singletonMap(CoreOptions.WRITE_ONLY.key(), "false")); + StreamExecutionEnvironment env = + streamExecutionEnvironmentBuilder() + .streamingMode() + .parallelism(1) + .checkpointIntervalMs(100) + .allowRestart() + .build(); + DataStreamSource source = + new CompactorSourceBuilder("default.expired_snapshot_recovery", compactionTable) + .withEnv(env) + .withContinuousMode(true) + .build(); + DataStream observed = + source.transform( + "fail-after-snapshot-expiration", + InternalTypeInfo.of( + LogicalTypeConversion.toLogicalType( + CompactBucketsTable.getRowType())), + new FailOnCheckpointOperator()); + new CompactorSinkBuilder(compactionTable, false).withInput(observed).build(); + + JobClient jobClient = env.executeAsync("compact-recovery-after-expiration"); + try { + SnapshotManager snapshotManager = table.snapshotManager(); + waitUtil( + () -> { + Long latest = snapshotManager.latestSnapshotId(); + if (latest == null || latest <= 1) { + try { + JobStatus status = jobClient.getJobStatus().get(); + if (status.isGloballyTerminalState()) { + throw new IllegalStateException( + "Compaction job terminated before initial compact snapshot: " + + status + + ", input records=" + + FailOnCheckpointOperator.inputRecords() + + ", checkpoints=" + + FailOnCheckpointOperator + .completedCheckpoints()); + } + } catch (Exception e) { + if (e instanceof RuntimeException) { + throw (RuntimeException) e; + } + throw new RuntimeException( + "Unable to inspect compaction job status", e); + } + return false; + } + try { + return snapshotManager.snapshot(latest).commitKind() + == Snapshot.CommitKind.COMPACT; + } catch (RuntimeException ignored) { + return false; + } + }, + Duration.ofSeconds(60), + Duration.ofMillis(100), + "The initial compact snapshot was not committed."); + long firstCompactionSnapshot = snapshotManager.latestSnapshotId(); + assertThat(snapshotManager.snapshot(firstCompactionSnapshot).commitKind()) + .isEqualTo(Snapshot.CommitKind.COMPACT); + + waitUtil( + () -> { + Long earliest = snapshotManager.earliestSnapshotId(); + return earliest != null && earliest >= firstCompactionSnapshot; + }, + Duration.ofSeconds(60), + Duration.ofMillis(100), + "The first append snapshot was not expired."); + + int completedCheckpoints = FailOnCheckpointOperator.completedCheckpoints(); + waitUtil( + () -> FailOnCheckpointOperator.completedCheckpoints() > completedCheckpoints, + Duration.ofSeconds(30), + Duration.ofMillis(100), + "No checkpoint completed after the initial compact snapshot."); + + // Create a suffix of newer append snapshots while the source is waiting for its next + // discovery. This expires any checkpoint cursor at or before the first compaction. + appendRow(table, rowData(3, 102, 15, BinaryString.fromString("20221208"))); + appendRow(table, rowData(4, 103, 15, BinaryString.fromString("20221208"))); + appendRow(table, rowData(5, 104, 15, BinaryString.fromString("20221208"))); + waitUtil( + () -> { + Long earliest = snapshotManager.earliestSnapshotId(); + Long latest = snapshotManager.latestSnapshotId(); + return earliest != null + && latest != null + && latest >= firstCompactionSnapshot + 3 + && earliest > firstCompactionSnapshot; + }, + Duration.ofSeconds(60), + Duration.ofMillis(100), + "The checkpointed snapshot was not expired."); + + FailOnCheckpointOperator.arm(); + waitUtil( + () -> { + try { + return FailOnCheckpointOperator.failed() + && FailOnCheckpointOperator.openedInstances() >= 2 + && jobClient.getJobStatus().get() == JobStatus.RUNNING; + } catch (Exception e) { + return false; + } + }, + Duration.ofSeconds(60), + Duration.ofMillis(100), + "The compaction job did not recover from the injected failover."); + + // A rebase marker must be observed before the next append is compacted. This also + // proves the restored source did not simply resume from the expired cursor. + waitUtil( + () -> FailOnCheckpointOperator.rebaseRecordsAfterRestart() > 0, + Duration.ofSeconds(60), + Duration.ofMillis(100), + "The restored source did not emit a rebase plan."); + + long latestBeforeFinalAppend = snapshotManager.latestSnapshotId(); + appendRow(table, rowData(6, 105, 15, BinaryString.fromString("20221208"))); + waitUtil( + () -> { + Long latest = snapshotManager.latestSnapshotId(); + if (latest == null || latest <= latestBeforeFinalAppend) { + return false; + } + try { + return snapshotManager.snapshot(latest).commitKind() + == Snapshot.CommitKind.COMPACT; + } catch (RuntimeException ignored) { + return false; + } + }, + Duration.ofSeconds(120), + Duration.ofMillis(200), + "No compact snapshot was committed after source restore."); + + List actual = + getResult( + table.newReadBuilder().newRead(), + table.newSnapshotReader().read().splits(), + ROW_TYPE); + assertThat(actual) + .containsExactlyInAnyOrder( + "+I[1, 100, 15, 20221208]", + "+I[2, 101, 15, 20221208]", + "+I[3, 102, 15, 20221208]", + "+I[4, 103, 15, 20221208]", + "+I[5, 104, 15, 20221208]", + "+I[6, 105, 15, 20221208]"); + } finally { + jobClient.cancel().get(30, TimeUnit.SECONDS); + FailOnCheckpointOperator.reset(); + } + } + + private void appendRow(FileStoreTable table, GenericRow row) throws Exception { + StreamWriteBuilder streamWriteBuilder = + table.newStreamWriteBuilder().withCommitUser(commitUser); + write = streamWriteBuilder.newWrite(); + commit = streamWriteBuilder.newCommit(); + try { + writeData(row); + } finally { + write.close(); + commit.close(); + write = null; + commit = null; + } + } + + /** Pass-through operator that fails exactly once on a checkpoint after the test arms it. */ + private static class FailOnCheckpointOperator extends AbstractStreamOperator + implements OneInputStreamOperator { + + private static final long serialVersionUID = 1L; + private static final AtomicBoolean ARMED = new AtomicBoolean(false); + private static final AtomicBoolean FAILED = new AtomicBoolean(false); + private static final AtomicInteger OPENED_INSTANCES = new AtomicInteger(); + private static final AtomicInteger COMPLETED_CHECKPOINTS = new AtomicInteger(); + private static final AtomicInteger REBASE_RECORDS_AFTER_RESTART = new AtomicInteger(); + private static final AtomicInteger INPUT_RECORDS = new AtomicInteger(); + + static void reset() { + ARMED.set(false); + FAILED.set(false); + OPENED_INSTANCES.set(0); + COMPLETED_CHECKPOINTS.set(0); + REBASE_RECORDS_AFTER_RESTART.set(0); + INPUT_RECORDS.set(0); + } + + static void arm() { + ARMED.set(true); + } + + static boolean failed() { + return FAILED.get(); + } + + static int openedInstances() { + return OPENED_INSTANCES.get(); + } + + static int completedCheckpoints() { + return COMPLETED_CHECKPOINTS.get(); + } + + static int rebaseRecordsAfterRestart() { + return REBASE_RECORDS_AFTER_RESTART.get(); + } + + static int inputRecords() { + return INPUT_RECORDS.get(); + } + + @Override + public void open() throws Exception { + super.open(); + OPENED_INSTANCES.incrementAndGet(); + } + + @Override + public void processElement(StreamRecord element) { + INPUT_RECORDS.incrementAndGet(); + RowData row = element.getValue(); + if (row.getArity() > 6 && !row.isNullAt(6) && row.getBoolean(6)) { + if (OPENED_INSTANCES.get() >= 2) { + REBASE_RECORDS_AFTER_RESTART.incrementAndGet(); + } + } + output.collect(element); + } + + @Override + public OperatorSnapshotFutures snapshotState( + long checkpointId, + long timestamp, + CheckpointOptions checkpointOptions, + CheckpointStreamFactory storageLocation) + throws Exception { + if (ARMED.get() && FAILED.compareAndSet(false, true)) { + throw new RuntimeException("intentional failover after snapshot expiration"); + } + return super.snapshotState(checkpointId, timestamp, checkpointOptions, storageLocation); + } + + @Override + public void notifyCheckpointComplete(long checkpointId) throws Exception { + super.notifyCheckpointComplete(checkpointId); + COMPLETED_CHECKPOINTS.incrementAndGet(); + } + } + private void writeData( StreamTableWrite write, StreamTableCommit commit, diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/FlinkStreamPartitionerTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/FlinkStreamPartitionerTest.java new file mode 100644 index 000000000000..482690ffd35e --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/FlinkStreamPartitionerTest.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.sink; + +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.flink.FlinkRowData; +import org.apache.paimon.flink.LogicalTypeConversion; +import org.apache.paimon.table.system.CompactBucketsTable; +import org.apache.paimon.utils.SerializationUtils; + +import org.apache.flink.api.common.functions.MapFunction; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.table.data.RowData; +import org.apache.flink.util.CloseableIterator; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; + +import static org.apache.paimon.data.BinaryRow.EMPTY_ROW; +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for the dedicated compaction stream partitioning. */ +class FlinkStreamPartitionerTest { + + @Test + void testCompactionPartitioningBroadcastsRebaseAndRoutesNormalRecordsOnce() throws Exception { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(2); + + RowData bucketZero = row(10L, 0, false); + RowData bucketOne = row(11L, 1, false); + RowData rebase = row(12L, 0, true); + TypeInformation rowTypeInfo = + org.apache.flink.table.runtime.typeutils.InternalTypeInfo.of( + LogicalTypeConversion.toLogicalType(CompactBucketsTable.getRowType())); + + DataStream input = + env.fromCollection(Arrays.asList(bucketZero, bucketOne, rebase), rowTypeInfo); + DataStream partitioned = + FlinkStreamPartitioner.partitionCompactionRecords(input, 2); + + // Keep two downstream subtasks so the broadcast edge is observable in the collected + // output. Normal records still travel through the bucket partitioner and occur once. + DataStream observed = + partitioned + .map( + new MapFunction() { + @Override + public RowData map(RowData value) { + return value; + } + }) + .setParallelism(2); + + int normalRecords = 0; + int rebaseRecords = 0; + try (CloseableIterator records = observed.executeAndCollect()) { + while (records.hasNext()) { + RowData record = records.next(); + if (BucketsRowChannelComputer.isRebase(record)) { + rebaseRecords++; + } else { + normalRecords++; + } + } + } + + assertThat(normalRecords).isEqualTo(2); + assertThat(rebaseRecords).isEqualTo(2); + } + + private RowData row(long snapshotId, int bucket, boolean rebase) { + return new FlinkRowData( + GenericRow.of( + snapshotId, + SerializationUtils.serializeBinaryRow(EMPTY_ROW), + bucket, + new byte[] {0x00, 0x00, 0x00, 0x00}, + null, + null, + rebase)); + } +} diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/MultiTablesStoreCompactOperatorTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/MultiTablesStoreCompactOperatorTest.java new file mode 100644 index 000000000000..9007cf1b9ee8 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/MultiTablesStoreCompactOperatorTest.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.sink; + +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.flink.FlinkRowData; +import org.apache.paimon.options.Options; +import org.apache.paimon.table.TableTestBase; +import org.apache.paimon.utils.SerializationUtils; + +import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.runtime.checkpoint.OperatorSubtaskState; +import org.apache.flink.streaming.api.environment.CheckpointConfig; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness; +import org.apache.flink.table.data.RowData; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for per-table state in {@link MultiTablesStoreCompactOperator}. */ +class MultiTablesStoreCompactOperatorTest extends TableTestBase { + + @Test + public void testRebaseMarkersUseIndependentSnapshotWatermarks() throws Exception { + Identifier firstTable = identifier("first"); + Identifier secondTable = identifier("second"); + catalog.createTable(firstTable, schemaDefault(), false); + catalog.createTable(secondTable, schemaDefault(), false); + + MultiTablesStoreCompactOperator.Factory operatorFactory = + new MultiTablesStoreCompactOperator.Factory( + catalog.catalogLoader(), + "10086", + new CheckpointConfig(), + true, + false, + false, + new Options()); + TypeSerializer serializer = + new MultiTableCommittableTypeInfo().createSerializer(new ExecutionConfig()); + OneInputStreamOperatorTestHarness harness = + new OneInputStreamOperatorTestHarness<>(operatorFactory); + harness.setup(serializer); + harness.initializeEmptyState(); + harness.open(); + + // Establish a writer for the second table before the first table advances to snapshot 10. + harness.processElement(new StreamRecord<>(record(secondTable, 2L, false))); + MultiTablesStoreCompactOperator operator = + (MultiTablesStoreCompactOperator) harness.getOperator(); + StoreSinkWriteImpl secondWrite = (StoreSinkWriteImpl) operator.writes.get(secondTable); + Object secondWriterBeforeRebase = secondWrite.getWrite(); + + // Snapshot IDs are local to a table. A marker for the first table at 10 must not make the + // second table's marker at 2 look stale. + harness.processElement(new StreamRecord<>(record(firstTable, 10L, true))); + harness.processElement(new StreamRecord<>(record(secondTable, 2L, true))); + + assertThat(secondWrite.getWrite()).isNotSameAs(secondWriterBeforeRebase); + harness.close(); + } + + @Test + public void testPerTableRebaseWatermarksSurviveCheckpointRestore() throws Exception { + Identifier firstTable = identifier("first"); + Identifier secondTable = identifier("second"); + catalog.createTable(firstTable, schemaDefault(), false); + catalog.createTable(secondTable, schemaDefault(), false); + + MultiTablesStoreCompactOperator.Factory operatorFactory = + new MultiTablesStoreCompactOperator.Factory( + catalog.catalogLoader(), + "10086", + new CheckpointConfig(), + true, + false, + false, + new Options()); + TypeSerializer serializer = + new MultiTableCommittableTypeInfo().createSerializer(new ExecutionConfig()); + + OneInputStreamOperatorTestHarness firstHarness = + new OneInputStreamOperatorTestHarness<>(operatorFactory); + firstHarness.setup(serializer); + firstHarness.initializeEmptyState(); + firstHarness.open(); + firstHarness.processElement(new StreamRecord<>(record(firstTable, 10L, true))); + firstHarness.processElement(new StreamRecord<>(record(secondTable, 2L, true))); + OperatorSubtaskState checkpoint = firstHarness.snapshot(1L, 1L); + firstHarness.close(); + + OneInputStreamOperatorTestHarness restoredHarness = + new OneInputStreamOperatorTestHarness<>(operatorFactory); + restoredHarness.setup(serializer); + restoredHarness.initializeState(checkpoint); + restoredHarness.open(); + + MultiTablesStoreCompactOperator operator = + (MultiTablesStoreCompactOperator) restoredHarness.getOperator(); + restoredHarness.processElement(new StreamRecord<>(record(firstTable, 9L, true))); + StoreSinkWrite firstWrite = operator.writes.get(firstTable); + Object firstWriter = ((StoreSinkWriteImpl) firstWrite).getWrite(); + + restoredHarness.processElement(new StreamRecord<>(record(secondTable, 2L, true))); + StoreSinkWrite secondWrite = operator.writes.get(secondTable); + Object secondWriterAtSnapshotTwo = ((StoreSinkWriteImpl) secondWrite).getWrite(); + restoredHarness.processElement(new StreamRecord<>(record(secondTable, 3L, true))); + Object secondWriterAtSnapshotThree = ((StoreSinkWriteImpl) secondWrite).getWrite(); + + assertThat(((StoreSinkWriteImpl) firstWrite).getWrite()).isSameAs(firstWriter); + assertThat(secondWriterAtSnapshotThree).isNotSameAs(secondWriterAtSnapshotTwo); + restoredHarness.close(); + } + + private RowData record(Identifier table, long snapshotId, boolean rebase) { + return new FlinkRowData( + GenericRow.of( + snapshotId, + SerializationUtils.serializeBinaryRow(BinaryRow.EMPTY_ROW), + 0, + new byte[] {0x00, 0x00, 0x00, 0x00}, + org.apache.paimon.data.BinaryString.fromString(table.getDatabaseName()), + org.apache.paimon.data.BinaryString.fromString(table.getObjectName()), + rebase)); + } +} diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/StoreCompactOperatorTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/StoreCompactOperatorTest.java index cf78c431923a..b027fffe4706 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/StoreCompactOperatorTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/StoreCompactOperatorTest.java @@ -21,6 +21,7 @@ import org.apache.paimon.Snapshot; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.flink.FlinkRowData; @@ -46,6 +47,7 @@ import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.runtime.checkpoint.OperatorSubtaskState; import org.apache.flink.runtime.jobgraph.OperatorID; import org.apache.flink.streaming.api.environment.CheckpointConfig; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; @@ -57,10 +59,12 @@ import javax.annotation.Nullable; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.atomic.AtomicReference; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -127,6 +131,303 @@ public void testCoordinatorProvider() throws Exception { .isEqualTo(StoreCompactOperator.class); } + @Test + public void testRebaseMarkerRestoresOnceWithoutNotifyingBaselineFiles() throws Exception { + createTableDefault(); + + CompactRememberStoreWrite compactRememberStoreWrite = new CompactRememberStoreWrite(true); + StoreCompactOperator.Factory operatorFactory = + new StoreCompactOperator.Factory( + getTableDefault(), + (table, commitUser, state, ioManager, memoryPoolFactory, metricGroup) -> + compactRememberStoreWrite, + "10086", + false); + + TypeSerializer serializer = + new CommittableTypeInfo().createSerializer(new ExecutionConfig()); + OneInputStreamOperatorTestHarness harness = + new OneInputStreamOperatorTestHarness<>(operatorFactory); + harness.setup(serializer); + harness.initializeEmptyState(); + harness.open(); + + byte[] emptyFiles = new byte[] {0x00, 0x00, 0x00, 0x00}; + byte[] partition = SerializationUtils.serializeBinaryRow(BinaryRow.EMPTY_ROW); + RowData marker = + new FlinkRowData(GenericRow.of(7L, partition, 1, emptyFiles, null, null, true)); + harness.processElement(new StreamRecord<>(marker)); + harness.processElement( + new StreamRecord<>( + new FlinkRowData( + GenericRow.of(7L, partition, 2, emptyFiles, null, null, true)))); + harness.processElement( + new StreamRecord<>( + new FlinkRowData( + GenericRow.of(8L, partition, 1, emptyFiles, null, null, false)))); + + assertThat(compactRememberStoreWrite.rebaseTime).isEqualTo(1); + assertThat(compactRememberStoreWrite.notifyTime).isEqualTo(1); + assertThat(((StoreCompactOperator) harness.getOperator()).compactionWaitingSet()) + .containsExactly(Pair.of(BinaryRow.EMPTY_ROW, 1)); + } + + @Test + public void testDeltaCoveredByRebaseIsDroppedWhenItArrivesInFlight() throws Exception { + createTableDefault(); + + CompactRememberStoreWrite compactRememberStoreWrite = new CompactRememberStoreWrite(true); + StoreCompactOperator.Factory operatorFactory = + new StoreCompactOperator.Factory( + getTableDefault(), + (table, commitUser, state, ioManager, memoryPoolFactory, metricGroup) -> + compactRememberStoreWrite, + "10086", + false); + + TypeSerializer serializer = + new CommittableTypeInfo().createSerializer(new ExecutionConfig()); + OneInputStreamOperatorTestHarness harness = + new OneInputStreamOperatorTestHarness<>(operatorFactory); + harness.setup(serializer); + harness.initializeEmptyState(); + harness.open(); + + byte[] emptyFiles = new byte[] {0x00, 0x00, 0x00, 0x00}; + byte[] partition = SerializationUtils.serializeBinaryRow(BinaryRow.EMPTY_ROW); + + // The marker establishes snapshot 7 as the baseline. A split for snapshot 7 can still be + // in flight and must be ignored, while a later delta remains eligible for compaction. + harness.processElement( + new StreamRecord<>( + new FlinkRowData( + GenericRow.of(7L, partition, 0, emptyFiles, null, null, true)))); + harness.processElement( + new StreamRecord<>( + new FlinkRowData( + GenericRow.of(7L, partition, 1, emptyFiles, null, null, false)))); + harness.processElement( + new StreamRecord<>( + new FlinkRowData( + GenericRow.of(8L, partition, 1, emptyFiles, null, null, false)))); + + assertThat(compactRememberStoreWrite.rebaseTime).isEqualTo(1); + assertThat(compactRememberStoreWrite.notifyTime).isEqualTo(1); + assertThat(((StoreCompactOperator) harness.getOperator()).compactionWaitingSet()) + .containsExactly(Pair.of(BinaryRow.EMPTY_ROW, 1)); + } + + @Test + public void testOlderRebaseMarkerCannotMoveWriterBackwards() throws Exception { + createTableDefault(); + + CompactRememberStoreWrite compactRememberStoreWrite = new CompactRememberStoreWrite(true); + StoreCompactOperator.Factory operatorFactory = + new StoreCompactOperator.Factory( + getTableDefault(), + (table, commitUser, state, ioManager, memoryPoolFactory, metricGroup) -> + compactRememberStoreWrite, + "10086", + false); + + TypeSerializer serializer = + new CommittableTypeInfo().createSerializer(new ExecutionConfig()); + OneInputStreamOperatorTestHarness harness = + new OneInputStreamOperatorTestHarness<>(operatorFactory); + harness.setup(serializer); + harness.initializeEmptyState(); + harness.open(); + + byte[] emptyFiles = new byte[] {0x00, 0x00, 0x00, 0x00}; + byte[] partition = SerializationUtils.serializeBinaryRow(BinaryRow.EMPTY_ROW); + + // A stale marker may arrive after a newer marker on another input channel. It must not + // close the current writer and restore it from the older snapshot. + harness.processElement( + new StreamRecord<>( + new FlinkRowData( + GenericRow.of(10L, partition, 0, emptyFiles, null, null, true)))); + harness.processElement( + new StreamRecord<>( + new FlinkRowData( + GenericRow.of(9L, partition, 0, emptyFiles, null, null, true)))); + harness.processElement( + new StreamRecord<>( + new FlinkRowData( + GenericRow.of(11L, partition, 0, emptyFiles, null, null, false)))); + + assertThat(compactRememberStoreWrite.rebaseTime).isEqualTo(1); + assertThat(compactRememberStoreWrite.notifyTime).isEqualTo(1); + assertThat(((StoreCompactOperator) harness.getOperator()).compactionWaitingSet()) + .containsExactly(Pair.of(BinaryRow.EMPTY_ROW, 0)); + } + + @Test + public void testRebaseUsesNewestObservedSnapshotWhenItIsVisible() throws Exception { + createTableDefault(); + batchWriteAndCommit( + getTableDefault(), + commitUser, + null, + GenericRow.of(1, BinaryString.fromString("first"), new byte[] {1})); + batchWriteAndCommit( + getTableDefault(), + commitUser, + null, + GenericRow.of(2, BinaryString.fromString("second"), new byte[] {2})); + + CompactRememberStoreWrite compactRememberStoreWrite = new CompactRememberStoreWrite(true); + StoreCompactOperator.Factory operatorFactory = + new StoreCompactOperator.Factory( + getTableDefault(), + (table, commitUser, state, ioManager, memoryPoolFactory, metricGroup) -> + compactRememberStoreWrite, + "10086", + false); + + TypeSerializer serializer = + new CommittableTypeInfo().createSerializer(new ExecutionConfig()); + OneInputStreamOperatorTestHarness harness = + new OneInputStreamOperatorTestHarness<>(operatorFactory); + harness.setup(serializer); + harness.initializeEmptyState(); + harness.open(); + + byte[] emptyFiles = new byte[] {0x00, 0x00, 0x00, 0x00}; + byte[] partition = SerializationUtils.serializeBinaryRow(BinaryRow.EMPTY_ROW); + + // The newer delta is observed before the older marker. Since snapshot 2 is visible, the + // rebase must use it and retain the delta instead of rebuilding the writer from snapshot 1. + harness.processElement( + new StreamRecord<>( + new FlinkRowData( + GenericRow.of(2L, partition, 0, emptyFiles, null, null, false)))); + harness.processElement( + new StreamRecord<>( + new FlinkRowData( + GenericRow.of(1L, partition, 0, emptyFiles, null, null, true)))); + + assertThat(compactRememberStoreWrite.rebaseTime).isEqualTo(1); + assertThat(compactRememberStoreWrite.capturedWriteRestore).isNotNull(); + assertThat( + compactRememberStoreWrite + .capturedWriteRestore + .restoreFiles(BinaryRow.EMPTY_ROW, 0, false, false, false) + .snapshot() + .id()) + .isEqualTo(2L); + harness.close(); + } + + @Test + public void testRebaseReplacesActualWriter() throws Exception { + createTableDefault(); + + AtomicReference sinkWriteReference = new AtomicReference<>(); + StoreCompactOperator.Factory operatorFactory = + new StoreCompactOperator.Factory( + getTableDefault(), + (table, commitUser, state, ioManager, memoryPoolFactory, metricGroup) -> { + StoreSinkWriteImpl sinkWrite = + new StoreSinkWriteImpl( + table, + commitUser, + state, + ioManager, + false, + false, + true, + memoryPoolFactory, + metricGroup); + sinkWriteReference.set(sinkWrite); + return sinkWrite; + }, + "10086", + false); + + TypeSerializer serializer = + new CommittableTypeInfo().createSerializer(new ExecutionConfig()); + OneInputStreamOperatorTestHarness harness = + new OneInputStreamOperatorTestHarness<>(operatorFactory); + harness.setup(serializer); + harness.initializeEmptyState(); + harness.open(); + + TableWriteImpl initialWriter = sinkWriteReference.get().getWrite(); + byte[] emptyFiles = new byte[] {0x00, 0x00, 0x00, 0x00}; + byte[] partition = SerializationUtils.serializeBinaryRow(BinaryRow.EMPTY_ROW); + harness.processElement( + new StreamRecord<>( + new FlinkRowData( + GenericRow.of(7L, partition, 0, emptyFiles, null, null, true)))); + + assertThat(sinkWriteReference.get().getWrite()).isNotSameAs(initialWriter); + harness.close(); + } + + @Test + public void testRebaseStateSurvivesCheckpointRestore() throws Exception { + createTableDefault(); + + List writes = new ArrayList<>(); + StoreCompactOperator.Factory operatorFactory = + new StoreCompactOperator.Factory( + getTableDefault(), + (table, commitUser, state, ioManager, memoryPoolFactory, metricGroup) -> { + CompactRememberStoreWrite sinkWrite = + new CompactRememberStoreWrite(true); + writes.add(sinkWrite); + return sinkWrite; + }, + "10086", + false); + + TypeSerializer serializer = + new CommittableTypeInfo().createSerializer(new ExecutionConfig()); + OneInputStreamOperatorTestHarness firstHarness = + new OneInputStreamOperatorTestHarness<>(operatorFactory); + firstHarness.setup(serializer); + firstHarness.initializeEmptyState(); + firstHarness.open(); + + byte[] emptyFiles = new byte[] {0x00, 0x00, 0x00, 0x00}; + byte[] partition = SerializationUtils.serializeBinaryRow(BinaryRow.EMPTY_ROW); + firstHarness.processElement( + new StreamRecord<>( + new FlinkRowData( + GenericRow.of(10L, partition, 0, emptyFiles, null, null, true)))); + OperatorSubtaskState checkpoint = firstHarness.snapshot(1L, 1L); + firstHarness.close(); + + OneInputStreamOperatorTestHarness restoredHarness = + new OneInputStreamOperatorTestHarness<>(operatorFactory); + restoredHarness.setup(serializer); + restoredHarness.initializeState(checkpoint); + restoredHarness.open(); + + // Restore must retain both marker ordering and the through-watermark. An older marker and + // an in-flight split from the baseline are ignored, while a later delta is still handled. + restoredHarness.processElement( + new StreamRecord<>( + new FlinkRowData( + GenericRow.of(9L, partition, 0, emptyFiles, null, null, true)))); + restoredHarness.processElement( + new StreamRecord<>( + new FlinkRowData( + GenericRow.of(10L, partition, 0, emptyFiles, null, null, false)))); + restoredHarness.processElement( + new StreamRecord<>( + new FlinkRowData( + GenericRow.of(11L, partition, 0, emptyFiles, null, null, false)))); + + assertThat(writes).hasSize(2); + assertThat(writes.get(1).rebaseTime).isZero(); + assertThat(writes.get(1).notifyTime).isEqualTo(1); + assertThat(((StoreCompactOperator) restoredHarness.getOperator()).compactionWaitingSet()) + .containsExactly(Pair.of(BinaryRow.EMPTY_ROW, 0)); + restoredHarness.close(); + } + @ParameterizedTest @ValueSource(booleans = {true, false}) public void testCompactWithCoordinator(boolean streamingMode) throws Exception { @@ -305,6 +606,8 @@ private static class CompactRememberStoreWrite implements StoreSinkWrite { private final boolean streamingMode; private int compactTime = 0; + private int rebaseTime = 0; + private int notifyTime = 0; private @Nullable WriteRestore capturedWriteRestore; public CompactRememberStoreWrite(boolean streamingMode) { @@ -316,6 +619,12 @@ public void setWriteRestore(WriteRestore writeRestore) { this.capturedWriteRestore = writeRestore; } + @Override + public void rebase(FileStoreTable table, WriteRestore writeRestore) { + rebaseTime++; + setWriteRestore(writeRestore); + } + @Override public SinkRecord write(InternalRow rowData) { return null; @@ -338,7 +647,9 @@ public void compact(BinaryRow partition, int bucket, boolean fullCompaction) { @Override public void notifyNewFiles( - long snapshotId, BinaryRow partition, int bucket, List files) {} + long snapshotId, BinaryRow partition, int bucket, List files) { + notifyTime++; + } @Override public List prepareCommit(boolean waitCompaction, long checkpointId) { diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/coordinator/CoordinatedWriteRestoreTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/coordinator/CoordinatedWriteRestoreTest.java new file mode 100644 index 000000000000..4d1472199290 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/coordinator/CoordinatedWriteRestoreTest.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.sink.coordinator; + +import org.apache.paimon.Snapshot; +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.operation.RestoreFiles; + +import org.apache.flink.runtime.jobgraph.OperatorID; +import org.apache.flink.runtime.jobgraph.tasks.TaskOperatorEventGateway; +import org.apache.flink.runtime.operators.coordination.CoordinationRequest; +import org.apache.flink.util.SerializedValue; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.paimon.utils.InstantiationUtil.deserializeObject; +import static org.apache.paimon.utils.InstantiationUtil.serializeObject; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** Tests for {@link CoordinatedWriteRestore}. */ +class CoordinatedWriteRestoreTest { + + @Test + void testFallbackSnapshotIsReusedAcrossBuckets() throws Exception { + Snapshot requested = snapshot(1L); + Snapshot fallback = snapshot(2L); + Snapshot newer = snapshot(3L); + TaskOperatorEventGateway gateway = mock(TaskOperatorEventGateway.class); + + AtomicInteger requestCount = new AtomicInteger(); + when(gateway.sendRequestToCoordinator(any(), any())) + .thenAnswer( + invocation -> { + SerializedValue serializedRequest = + invocation.getArgument(1); + PagedCoordinationRequest pagedRequest = + (PagedCoordinationRequest) + serializedRequest.deserializeValue( + getClass().getClassLoader()); + ScanCoordinationRequest request = + deserializeObject( + pagedRequest.content(), getClass().getClassLoader()); + Snapshot responseSnapshot = + request.snapshotId() != null + && request.snapshotId() == fallback.id() + ? fallback + : requestCount.getAndIncrement() == 0 + ? fallback + : newer; + ScanCoordinationResponse response = + new ScanCoordinationResponse( + responseSnapshot, null, null, null, null, null); + PagedCoordinationResponse pagedResponse = + new PagedCoordinationResponse(serializeObject(response), null); + return CompletableFuture.completedFuture( + CoordinationResponseUtils.wrap(pagedResponse)); + }); + + CoordinatedWriteRestore restore = + new CoordinatedWriteRestore(gateway, new OperatorID()).withSnapshot(requested.id()); + + RestoreFiles firstBucket = + restore.restoreFiles(BinaryRow.EMPTY_ROW, 0, false, false, false); + RestoreFiles secondBucket = + restore.restoreFiles(BinaryRow.EMPTY_ROW, 1, false, false, false); + + assertThat(firstBucket.snapshot().id()).isEqualTo(fallback.id()); + assertThat(secondBucket.snapshot().id()).isEqualTo(fallback.id()); + verify(gateway, times(2)).sendRequestToCoordinator(any(), any()); + } + + private Snapshot snapshot(long id) { + return new Snapshot( + id, + 0L, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 0L, + Snapshot.CommitKind.APPEND, + 0L, + 0L, + 0L, + null, + null, + null, + null, + null, + null); + } +} diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/coordinator/TableWriteCoordinatorTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/coordinator/TableWriteCoordinatorTest.java index 87b6bbca6cf7..9d3c86adbc47 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/coordinator/TableWriteCoordinatorTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/coordinator/TableWriteCoordinatorTest.java @@ -31,6 +31,7 @@ import org.apache.paimon.io.CompactIncrement; import org.apache.paimon.io.DataIncrement; import org.apache.paimon.manifest.ManifestEntry; +import org.apache.paimon.options.ExpireConfig; import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; import org.apache.paimon.schema.Schema; @@ -130,6 +131,39 @@ public void testScanVectorIndexPayloads() throws Exception { assertThat(scan.extractVectorIndexPayloads()).containsExactly(ann); } + @Test + public void testScanFallsBackToLatestWhenRequestedSnapshotExpired() throws Exception { + Identifier identifier = new Identifier("db", "table"); + Schema schema = Schema.newBuilder().column("f0", DataTypes.INT()).build(); + catalog.createDatabase("db", false); + catalog.createTable(identifier, schema, false); + FileStoreTable table = getTable(identifier); + + write(table, GenericRow.of(1)); + Snapshot expired = table.latestSnapshot().get(); + write(table, GenericRow.of(2)); + Snapshot latest = table.latestSnapshot().get(); + table.newExpireSnapshots() + .config( + ExpireConfig.builder() + .snapshotRetainMin(1) + .snapshotRetainMax(1) + .snapshotTimeRetain(Duration.ZERO) + .build()) + .expire(); + + assertThat(table.snapshotManager().isSnapshotExpired(expired.id())).isTrue(); + TableWriteCoordinator coordinator = new TableWriteCoordinator(table); + ScanCoordinationRequest request = + new ScanCoordinationRequest( + serializeBinaryRow(EMPTY_ROW), 0, false, false, false, expired.id(), true); + + ScanCoordinationResponse response = coordinator.scan(request); + + assertThat(response.snapshot().id()).isEqualTo(latest.id()); + assertThat(response.extractDataFiles()).hasSize(2); + } + @Test public void testPrefetchManifestsWarmsCache() throws Exception { Identifier identifier = new Identifier("db", "table"); diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/ContinuousFileSplitEnumeratorTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/ContinuousFileSplitEnumeratorTest.java index 0a50038bb405..6dac3ba421b3 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/ContinuousFileSplitEnumeratorTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/ContinuousFileSplitEnumeratorTest.java @@ -22,6 +22,7 @@ import org.apache.paimon.table.BucketMode; import org.apache.paimon.table.source.DataFilePlan; import org.apache.paimon.table.source.DataSplit; +import org.apache.paimon.table.source.DataTableStreamScan; import org.apache.paimon.table.source.IncrementalSplit; import org.apache.paimon.table.source.StreamTableScan; import org.apache.paimon.table.source.TableScan; @@ -38,6 +39,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.TreeMap; import java.util.UUID; import java.util.concurrent.atomic.AtomicReference; @@ -47,6 +49,9 @@ import static org.apache.paimon.io.DataFileTestUtils.row; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; /** Unit tests for the {@link ContinuousFileSplitEnumerator}. */ public class ContinuousFileSplitEnumeratorTest @@ -688,6 +693,33 @@ public void testEnumeratorWithCheckpoint() { assertThat(toDataSplits(state.splits())).containsExactlyElementsOf(expectedResults.get(2L)); } + @Test + public void testExpiredPendingSplitIsReplacedByRebasePlan() { + final TestingSplitEnumeratorContext context = + getSplitEnumeratorContext(1); + DataTableStreamScan scan = mock(DataTableStreamScan.class); + FileStoreSourceSplit expiredSplit = createSnapshotSplit(1, 0, Collections.emptyList()); + DataSplit rebaseSplit = createDataSplit(3, 0, Collections.emptyList()).asRebase(); + when(scan.isSnapshotExpiredForRebase(1L)).thenReturn(true); + when(scan.plan()).thenReturn(new DataFilePlan(Collections.singletonList(rebaseSplit))); + when(scan.checkpoint()).thenReturn(4L); + + ContinuousFileSplitEnumerator enumerator = + new Builder() + .setSplitEnumeratorContext(context) + .setInitialSplits(Collections.singletonList(expiredSplit)) + .setScan(scan) + .build(); + + Optional result = + enumerator.scanNextSnapshot(); + enumerator.processDiscoveredSplits(result, null); + + assertThat(toDataSplits(enumerator.splitAssigner.remainingSplits())) + .containsExactly(rebaseSplit); + verify(scan).restore(1L); + } + @Test public void testEnumeratorWithConsumer() throws Exception { final TestingAsyncSplitEnumeratorContext context = diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/ContinuousFileStoreSourceTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/ContinuousFileStoreSourceTest.java new file mode 100644 index 000000000000..567982ac5a8f --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/ContinuousFileStoreSourceTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.source; + +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.table.source.DataSplit; +import org.apache.paimon.table.source.DataTableStreamScan; +import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.table.source.StreamTableScan; + +import org.apache.flink.api.connector.source.SplitEnumerator; +import org.apache.flink.api.connector.source.SplitEnumeratorContext; +import org.apache.flink.connector.testutils.source.reader.TestingSplitEnumeratorContext; +import org.junit.jupiter.api.Test; + +import javax.annotation.Nullable; + +import java.util.Collection; +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** Tests for checkpoint recovery in {@link ContinuousFileStoreSource}. */ +class ContinuousFileStoreSourceTest { + + @Test + void testRestoreEnumeratorDropsExpiredPendingSplit() { + ReadBuilder readBuilder = mock(ReadBuilder.class); + DataTableStreamScan scan = mock(DataTableStreamScan.class); + when(readBuilder.newStreamScan()).thenReturn(scan); + when(scan.isSnapshotExpiredForRebase(1L)).thenReturn(true); + + CapturingSource source = new CapturingSource(readBuilder); + FileStoreSourceSplit expiredSplit = + new FileStoreSourceSplit( + "expired", + DataSplit.builder() + .withSnapshot(1L) + .withPartition(BinaryRow.EMPTY_ROW) + .withBucket(0) + .withBucketPath("") + .withDataFiles(Collections.emptyList()) + .build()); + PendingSplitsCheckpoint checkpoint = + new PendingSplitsCheckpoint(Collections.singletonList(expiredSplit), 2L); + + TestingSplitEnumeratorContext context = + new TestingSplitEnumeratorContext<>(1); + source.restoreEnumerator(context, checkpoint); + + assertThat(source.capturedSplits).isEmpty(); + assertThat(source.capturedNextSnapshotId).isEqualTo(1L); + verify(scan).restore(2L); + verify(scan).restore(1L); + } + + private static class CapturingSource extends ContinuousFileStoreSource { + + private Collection capturedSplits; + private Long capturedNextSnapshotId; + + private CapturingSource(ReadBuilder readBuilder) { + super(readBuilder, Collections.emptyMap(), null); + } + + @Override + protected SplitEnumerator buildEnumerator( + SplitEnumeratorContext context, + Collection splits, + @Nullable Long nextSnapshotId, + StreamTableScan scan) { + capturedSplits = splits; + capturedNextSnapshotId = nextSnapshotId; + return null; + } + } +} diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/align/AlignedContinuousFileSplitEnumeratorTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/align/AlignedContinuousFileSplitEnumeratorTest.java index ad71fa81ca12..29dc751f7e99 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/align/AlignedContinuousFileSplitEnumeratorTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/align/AlignedContinuousFileSplitEnumeratorTest.java @@ -33,7 +33,9 @@ import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FileStoreTableFactory; +import org.apache.paimon.table.source.DataFilePlan; import org.apache.paimon.table.source.DataSplit; +import org.apache.paimon.table.source.DataTableStreamScan; import org.apache.paimon.table.source.StreamTableScan; import org.apache.paimon.types.DataType; import org.apache.paimon.types.DataTypes; @@ -47,18 +49,24 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import javax.annotation.Nullable; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.UUID; import static org.apache.paimon.io.DataFileTestUtils.row; import static org.apache.paimon.testutils.assertj.PaimonAssertions.anyCauseMatches; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; /** Unit tests for the {@link AlignedContinuousFileSplitEnumerator}. */ public class AlignedContinuousFileSplitEnumeratorTest @@ -206,6 +214,34 @@ public void testScanWithConsumerId() throws Exception { new Condition<>(consumer -> consumer.nextSnapshot() == 3L, "condition")); } + @Test + public void testExpiredPendingSplitIsReplacedByRebasePlan() { + final TestingSplitEnumeratorContext context = + getSplitEnumeratorContext(1); + DataTableStreamScan scan = mock(DataTableStreamScan.class); + FileStoreSourceSplit expiredSplit = createSnapshotSplit(1, 0, Collections.emptyList()); + DataSplit rebaseSplit = + ((DataSplit) createSnapshotSplit(3, 0, Collections.emptyList()).split()).asRebase(); + when(scan.isSnapshotExpiredForRebase(1L)).thenReturn(true); + when(scan.plan()).thenReturn(new DataFilePlan(Collections.singletonList(rebaseSplit))); + when(scan.checkpoint()).thenReturn(4L); + + TestingAlignedEnumerator enumerator = + new Builder() + .setSplitEnumeratorContext(context) + .setInitialSplits(Collections.singletonList(expiredSplit)) + .setScan(scan) + .build(); + + assertThat(enumerator.scanNextSnapshotForTest()).isEmpty(); + enumerator.processDiscoveredSplitsForTest(Optional.empty(), null); + + assertThat(enumerator.remainingSplitsForTest()) + .extracting(split -> ((DataSplit) split.split()).isRebase()) + .containsExactly(true); + verify(scan).restore(1L); + } + private static class Builder { private SplitEnumeratorContext context; private Collection initialSplits = Collections.emptyList(); @@ -241,8 +277,8 @@ public Builder setAlignedTimeout(long timeout) { return this; } - public AlignedContinuousFileSplitEnumerator build() { - return new AlignedContinuousFileSplitEnumerator( + public TestingAlignedEnumerator build() { + return new TestingAlignedEnumerator( context, initialSplits, null, @@ -257,6 +293,48 @@ public AlignedContinuousFileSplitEnumerator build() { } } + private static class TestingAlignedEnumerator extends AlignedContinuousFileSplitEnumerator { + + private TestingAlignedEnumerator( + SplitEnumeratorContext context, + Collection remainSplits, + @Nullable Long nextSnapshotId, + long discoveryInterval, + StreamTableScan scan, + boolean unawareBucket, + long alignTimeout, + int splitPerTaskMax, + boolean shuffleBucketWithPartition, + int maxSnapshotCount, + int sourceParallelismUpperBound) { + super( + context, + remainSplits, + nextSnapshotId, + discoveryInterval, + scan, + unawareBucket, + alignTimeout, + splitPerTaskMax, + shuffleBucketWithPartition, + maxSnapshotCount, + sourceParallelismUpperBound); + } + + private Optional scanNextSnapshotForTest() { + return scanNextSnapshot(); + } + + private void processDiscoveredSplitsForTest( + Optional plan, Throwable error) { + processDiscoveredSplits(plan, error); + } + + private Collection remainingSplitsForTest() { + return splitAssigner.remainingSplits(); + } + } + @Override protected FileStoreSourceSplit createSnapshotSplit( int snapshotId, int bucket, List files, int... partitions) {