diff --git a/docs/docs/multimodal-table/data-evolution-maintenance.mdx b/docs/docs/multimodal-table/data-evolution-maintenance.mdx
index 6e47487ccd50..ab0f2d258afb 100644
--- a/docs/docs/multimodal-table/data-evolution-maintenance.mdx
+++ b/docs/docs/multimodal-table/data-evolution-maintenance.mdx
@@ -89,6 +89,20 @@ Ordinary compaction **does not physically remove rows hidden by deletion
vectors**. Removing their positions would change the alignment with untouched
column files.
+### Recovering Files After a Failed Rewrite
+
+Flink and Spark rewrite DV indexes on workers. A failed worker attempt cleans
+up the new DV files it has tracked. Files from tasks that already succeeded,
+or from a lost worker process, can remain unreferenced if the overall rewrite
+fails before committing.
+
+After the failed job and its retries have stopped, use
+[Remove Orphan Files](../maintenance/manage-snapshots#remove-orphan-files)
+to reclaim these files. The default retention window is one day. If setting
+`older_than`, choose a cutoff that keeps files from any active writes or
+maintenance jobs. Snapshot expiration alone does not reclaim files that were
+never committed.
+
### Incremental Column-Version Compaction
To reduce repeated rewrites of large base files when only a few columns
diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactDeletionVectorRewriter.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactDeletionVectorRewriter.java
index 474eeaaa3864..b60a7c3d8b21 100644
--- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactDeletionVectorRewriter.java
+++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactDeletionVectorRewriter.java
@@ -19,57 +19,14 @@
package org.apache.paimon.append.dataevolution;
import org.apache.paimon.Snapshot;
-import org.apache.paimon.data.BinaryRow;
-import org.apache.paimon.deletionvectors.Bitmap64DeletionVector;
-import org.apache.paimon.deletionvectors.BitmapDeletionVector;
-import org.apache.paimon.deletionvectors.DeletionVector;
-import org.apache.paimon.deletionvectors.append.AppendDeleteFileMaintainer;
-import org.apache.paimon.deletionvectors.append.BaseAppendDeleteFileMaintainer;
-import org.apache.paimon.index.IndexFileHandler;
-import org.apache.paimon.index.IndexFileMeta;
-import org.apache.paimon.io.CompactIncrement;
-import org.apache.paimon.io.DataFileMeta;
-import org.apache.paimon.io.DataIncrement;
-import org.apache.paimon.manifest.FileKind;
-import org.apache.paimon.manifest.IndexManifestEntry;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.sink.CommitMessage;
-import org.apache.paimon.table.sink.CommitMessageImpl;
-import org.apache.paimon.utils.Pair;
-import org.apache.paimon.utils.Range;
-import org.apache.paimon.utils.RangeHelper;
+import org.apache.paimon.table.sink.TableCommit;
import java.util.ArrayList;
-import java.util.Collections;
-import java.util.LinkedHashMap;
import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.stream.Collectors;
-import static org.apache.paimon.deletionvectors.DeletionVectorsIndexFile.DELETION_VECTORS_INDEX;
-import static org.apache.paimon.format.blob.BlobFileFormat.isBlobFile;
-import static org.apache.paimon.table.BucketMode.UNAWARE_BUCKET;
-import static org.apache.paimon.types.VectorType.isVectorStoreFile;
-import static org.apache.paimon.utils.DataEvolutionUtils.checkContiguousRowRange;
-import static org.apache.paimon.utils.DataEvolutionUtils.retrieveAnchorFile;
-import static org.apache.paimon.utils.Preconditions.checkState;
-
-/**
- * Rewrites deletion vectors for non-materialized data-evolution compaction.
- *
- *
Compaction tasks rewrite data files without applying deletion vectors. This class consumes all
- * compaction messages before commit, moves old anchor-file DVs to new compacted files, and returns
- * index-only compact messages for the rewritten DV index files.
- *
- *
Unlike append compaction, data-evolution compaction does not pack compact tasks by deletion
- * vector index files. Data-evolution compaction is batch-oriented, and packing by DV may create a
- * huge compact task when many files contain only a few deleted rows. This is especially risky when
- * blob compaction is involved. Instead, compact tasks are planned normally and DVs are rewritten
- * centrally from compact commit messages. This is acceptable because most DVs only need to be
- * renamed to the new anchor file and are not fully rewritten.
- */
+/** Local execution of the same DV rewrite tasks used by distributed compaction. */
public class DataEvolutionCompactDeletionVectorRewriter {
private final FileStoreTable table;
@@ -81,231 +38,22 @@ public DataEvolutionCompactDeletionVectorRewriter(FileStoreTable table, Snapshot
}
public List rewriteDeletionVectors(List messages) {
- if (!table.coreOptions().deletionVectorsEnabled() || messages.isEmpty()) {
- return Collections.emptyList();
- }
-
- Map maintainerByParts =
- collectMaintainers(snapshot, messages);
- if (maintainerByParts.isEmpty()) {
- return Collections.emptyList();
- }
-
List result = new ArrayList<>();
- for (Map.Entry entry :
- maintainerByParts.entrySet()) {
- List indexChanges = entry.getValue().persist();
- if (!indexChanges.isEmpty()) {
- result.add(toCommitMessage(entry.getKey(), indexChanges));
+ try {
+ for (DataEvolutionDeletionVectorRewriteTask task :
+ new DataEvolutionDeletionVectorRewritePlanner(table, snapshot).plan(messages)) {
+ result.add(task.execute(table));
}
- }
- return result;
- }
-
- private Map collectMaintainers(
- Snapshot snapshot, List messages) {
- Map result = createMaintainers(snapshot, messages);
- RangeHelper rangeHelper = new RangeHelper<>(DataFileMeta::nonNullRowIdRange);
-
- for (CommitMessage message : messages) {
- CommitMessageImpl commitMessage = (CommitMessageImpl) message;
- CompactIncrement compactIncrement = commitMessage.compactIncrement();
- List before = normalFiles(compactIncrement.compactBefore());
- List after = normalFiles(compactIncrement.compactAfter());
-
- // Skip blob-only tasks and index-only commit messages.
- if (before.isEmpty() && after.isEmpty()) {
- continue;
+ } catch (RuntimeException failure) {
+ if (!result.isEmpty()) {
+ try (TableCommit commit = table.newCommit("data-evolution-dv-rewrite-abort")) {
+ commit.abort(result);
+ } catch (Exception cleanupFailure) {
+ failure.addSuppressed(cleanupFailure);
+ }
}
-
- AppendDeleteFileMaintainer maintainer = result.get(commitMessage.partition());
- if (isMaterialized(compactIncrement)) {
- removeMaterializedDeletionVectors(maintainer, rangeHelper, before);
- continue;
- }
-
- Range beforeRange = checkContiguousRowRange(before);
- Range afterRange = checkContiguousRowRange(after);
- checkState(
- beforeRange.equals(afterRange),
- "Non-materialized data evolution compaction should keep the same row-id "
- + "range, but compact before range is %s and compact after range is %s.",
- beforeRange,
- afterRange);
- TreeMap outputFiles = new TreeMap<>();
- for (DataFileMeta file : after) {
- outputFiles.put(file.nonNullFirstRowId(), file);
- }
- Map rewritten = new LinkedHashMap<>();
-
- // Redistribute old deletion positions across the output file boundaries.
- for (List previousRowGroup : rangeHelper.mergeOverlappingRanges(before)) {
- DataFileMeta oldAnchor = retrieveAnchorFile(previousRowGroup, file -> file);
- moveDeletionVector(
- maintainer,
- outputFiles,
- rewritten,
- oldAnchor.fileName(),
- oldAnchor.nonNullRowIdRange());
- }
- rewritten.forEach(maintainer::notifyNewDeletionVector);
+ throw failure;
}
return result;
}
-
- private Map createMaintainers(
- Snapshot snapshot, List messages) {
- Map> entriesByPartition = new LinkedHashMap<>();
- for (CommitMessage message : messages) {
- CommitMessageImpl commitMessage = (CommitMessageImpl) message;
- CompactIncrement increment = commitMessage.compactIncrement();
- checkNoCompactIndexChanges(increment);
- checkState(
- commitMessage.bucket() == UNAWARE_BUCKET
- && commitMessage.totalBuckets() == null,
- "Data evolution compaction should only produce unaware-bucket commit "
- + "messages, but got bucket %s and total buckets %s.",
- commitMessage.bucket(),
- commitMessage.totalBuckets());
- List before = normalFiles(increment.compactBefore());
- List after = normalFiles(increment.compactAfter());
- if (before.isEmpty() && after.isEmpty()) {
- continue;
- }
- entriesByPartition.computeIfAbsent(
- commitMessage.partition(), ignored -> new ArrayList<>());
- }
- if (entriesByPartition.isEmpty()) {
- return Collections.emptyMap();
- }
-
- IndexFileHandler handler = table.store().newIndexFileHandler();
- Set> buckets =
- entriesByPartition.keySet().stream()
- .map(partition -> Pair.of(partition, UNAWARE_BUCKET))
- .collect(Collectors.toSet());
- // Filter before decoding DV ranges and scan only once for all affected partitions.
- for (IndexManifestEntry entry :
- handler.scanBucketEntries(snapshot, DELETION_VECTORS_INDEX, buckets)) {
- entriesByPartition.get(entry.partition()).add(entry);
- }
-
- Map result = new LinkedHashMap<>();
- entriesByPartition.forEach(
- (partition, entries) ->
- result.put(
- partition,
- BaseAppendDeleteFileMaintainer.forUnawareAppend(
- handler, partition, entries)));
- return result;
- }
-
- private void removeMaterializedDeletionVectors(
- AppendDeleteFileMaintainer maintainer,
- RangeHelper rangeHelper,
- List before) {
- for (List previousRowGroup : rangeHelper.mergeOverlappingRanges(before)) {
- DataFileMeta oldAnchor = retrieveAnchorFile(previousRowGroup, file -> file);
- maintainer.notifyRemovedDeletionVector(oldAnchor.fileName());
- }
- }
-
- private boolean isMaterialized(CompactIncrement compactIncrement) {
- List before = normalFiles(compactIncrement.compactBefore());
- if (before.isEmpty()) {
- return false;
- }
- return compactIncrement.compactAfter().stream().allMatch(file -> file.firstRowId() == null);
- }
-
- /**
- * Moves old anchor-file deletion vectors to the new files, merging or splitting them at the
- * output row-id boundaries.
- */
- private void moveDeletionVector(
- AppendDeleteFileMaintainer maintainer,
- TreeMap outputFiles,
- Map rewritten,
- String oldFileName,
- Range oldRange) {
- DeletionVector old = maintainer.getDeletionVector(oldFileName);
- if (old == null || old.isEmpty()) {
- return;
- }
-
- // Fast path for renaming deletion vectors only.
- DataFileMeta firstOutput = outputFiles.floorEntry(oldRange.from).getValue();
- if (oldRange.equals(firstOutput.nonNullRowIdRange())) {
- rewritten
- .computeIfAbsent(firstOutput.fileName(), ignored -> newDeletionVector())
- .merge(old);
- } else {
- old.forEachDeletedPosition(
- position -> {
- long absolutePosition = oldRange.from + position;
- DataFileMeta output = outputFiles.floorEntry(absolutePosition).getValue();
- Range newRange = output.nonNullRowIdRange();
- long newPosition = absolutePosition - newRange.from;
- checkState(
- newPosition >= 0 && newPosition < newRange.count(),
- "Cannot move deletion position %s from old range %s to new range %s.",
- absolutePosition,
- oldRange,
- newRange);
- rewritten
- .computeIfAbsent(output.fileName(), ignored -> newDeletionVector())
- .delete(newPosition);
- });
- }
- maintainer.notifyRemovedDeletionVector(oldFileName);
- }
-
- private List normalFiles(List files) {
- return files.stream().filter(this::isNormalFile).collect(Collectors.toList());
- }
-
- private boolean isNormalFile(DataFileMeta fileMeta) {
- return !isBlobFile(fileMeta.fileName()) && !isVectorStoreFile(fileMeta.fileName());
- }
-
- private DeletionVector newDeletionVector() {
- return table.coreOptions().deletionVectorBitmap64()
- ? new Bitmap64DeletionVector()
- : new BitmapDeletionVector();
- }
-
- private CommitMessage toCommitMessage(
- BinaryRow partition, List indexChanges) {
- List newIndexFiles = new ArrayList<>();
- List deletedIndexFiles = new ArrayList<>();
- for (IndexManifestEntry entry : indexChanges) {
- if (entry.kind() == FileKind.ADD) {
- newIndexFiles.add(entry.indexFile());
- } else {
- deletedIndexFiles.add(entry.indexFile());
- }
- }
-
- return new CommitMessageImpl(
- partition,
- UNAWARE_BUCKET,
- null,
- DataIncrement.emptyIncrement(),
- new CompactIncrement(
- Collections.emptyList(),
- Collections.emptyList(),
- Collections.emptyList(),
- newIndexFiles,
- deletedIndexFiles));
- }
-
- private void checkNoCompactIndexChanges(CompactIncrement compactIncrement) {
- checkState(
- compactIncrement.newIndexFiles().isEmpty()
- && compactIncrement.deletedIndexFiles().isEmpty(),
- "Data evolution compaction should not produce index changes before deletion vector rewrite, "
- + "but got new index files %s and deleted index files %s.",
- compactIncrement.newIndexFiles(),
- compactIncrement.deletedIndexFiles());
- }
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactionCommitPreparation.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactionCommitPreparation.java
index 238f98e2b2e9..75ab8067f52d 100644
--- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactionCommitPreparation.java
+++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactionCommitPreparation.java
@@ -48,4 +48,9 @@ public List prepare(List messages) {
.rewriteDeletionVectors(messages));
return result;
}
+
+ public List planDeletionVectorRewrites(
+ List messages) {
+ return new DataEvolutionDeletionVectorRewritePlanner(table, snapshot).plan(messages);
+ }
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionDeletionVectorRewritePlanner.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionDeletionVectorRewritePlanner.java
new file mode 100644
index 000000000000..1bdc46d993d6
--- /dev/null
+++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionDeletionVectorRewritePlanner.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.append.dataevolution;
+
+import org.apache.paimon.Snapshot;
+import org.apache.paimon.append.dataevolution.DataEvolutionDeletionVectorRewriteTask.IndexRewrite;
+import org.apache.paimon.append.dataevolution.DataEvolutionDeletionVectorRewriteTask.Rewrite;
+import org.apache.paimon.append.dataevolution.DataEvolutionDeletionVectorRewriteTask.Source;
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.index.DeletionVectorMeta;
+import org.apache.paimon.index.IndexFileHandler;
+import org.apache.paimon.io.CompactIncrement;
+import org.apache.paimon.io.DataFileMeta;
+import org.apache.paimon.manifest.IndexManifestEntry;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.sink.CommitMessage;
+import org.apache.paimon.table.sink.CommitMessageImpl;
+import org.apache.paimon.table.source.DeletionFile;
+import org.apache.paimon.utils.Pair;
+import org.apache.paimon.utils.Range;
+import org.apache.paimon.utils.RangeHelper;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.apache.paimon.deletionvectors.DeletionVectorsIndexFile.DELETION_VECTORS_INDEX;
+import static org.apache.paimon.format.blob.BlobFileFormat.isBlobFile;
+import static org.apache.paimon.table.BucketMode.UNAWARE_BUCKET;
+import static org.apache.paimon.types.VectorType.isVectorStoreFile;
+import static org.apache.paimon.utils.DataEvolutionUtils.checkContiguousRowRange;
+import static org.apache.paimon.utils.DataEvolutionUtils.retrieveAnchorFile;
+import static org.apache.paimon.utils.Preconditions.checkState;
+
+/**
+ * Plans bitmap work without reading bitmap contents or changing data-compaction boundaries.
+ *
+ *
All compact messages are collected first, so one old index file has a single owner even when
+ * its anchors are replaced by different data tasks.
+ */
+public class DataEvolutionDeletionVectorRewritePlanner {
+
+ private static final int MAX_DELETE_ONLY_FILES_PER_TASK = 128;
+
+ private final FileStoreTable table;
+ private final Snapshot snapshot;
+
+ public DataEvolutionDeletionVectorRewritePlanner(FileStoreTable table, Snapshot snapshot) {
+ this.table = table;
+ this.snapshot = snapshot;
+ }
+
+ public List plan(List messages) {
+ if (!table.coreOptions().deletionVectorsEnabled() || messages.isEmpty()) {
+ return Collections.emptyList();
+ }
+ Map partitions = new LinkedHashMap<>();
+ RangeHelper ranges = new RangeHelper<>(DataFileMeta::nonNullRowIdRange);
+ for (CommitMessage message : messages) {
+ CommitMessageImpl commit = (CommitMessageImpl) message;
+ CompactIncrement increment = commit.compactIncrement();
+ checkState(
+ commit.bucket() == UNAWARE_BUCKET && commit.totalBuckets() == null,
+ "DV rewrite requires unaware-bucket compact messages.");
+ checkState(
+ increment.newIndexFiles().isEmpty() && increment.deletedIndexFiles().isEmpty(),
+ "DV rewrite input must not contain prepared index changes.");
+ List before = normalFiles(increment.compactBefore());
+ List after = normalFiles(increment.compactAfter());
+ if (before.isEmpty() && after.isEmpty()) {
+ continue;
+ }
+ PartitionPlan partition =
+ partitions.computeIfAbsent(commit.partition(), ignored -> new PartitionPlan());
+ Map anchors = new LinkedHashMap<>();
+ for (List group : ranges.mergeOverlappingRanges(before)) {
+ DataFileMeta anchor = retrieveAnchorFile(group, file -> file);
+ checkState(
+ partition.removed.add(anchor.fileName()),
+ "Anchor %s belongs to multiple DV rewrite messages.",
+ anchor.fileName());
+ anchors.put(anchor.fileName(), anchor.nonNullRowIdRange());
+ }
+ boolean materialized =
+ !before.isEmpty()
+ && increment.compactAfter().stream()
+ .allMatch(file -> file.firstRowId() == null);
+ if (!materialized) {
+ checkState(
+ checkContiguousRowRange(before).equals(checkContiguousRowRange(after)),
+ "DV rewrite must preserve the input row-ID range.");
+ Map outputs = new LinkedHashMap<>();
+ after.forEach(file -> outputs.put(file.fileName(), file.nonNullRowIdRange()));
+ partition.specifications.add(new Specification(anchors, outputs));
+ }
+ }
+ if (partitions.isEmpty()) {
+ return Collections.emptyList();
+ }
+ IndexFileHandler handler = table.store().newIndexFileHandler();
+ Set> buckets =
+ partitions.keySet().stream()
+ .map(partition -> Pair.of(partition, UNAWARE_BUCKET))
+ .collect(Collectors.toSet());
+ for (IndexManifestEntry entry :
+ handler.scanBucketEntries(snapshot, DELETION_VECTORS_INDEX, buckets)) {
+ PartitionPlan partition = partitions.get(entry.partition());
+ Set removed = new LinkedHashSet<>();
+ String path = null;
+ for (DeletionVectorMeta dv : entry.indexFile().dvRanges().values()) {
+ if (partition.removed.contains(dv.dataFileName())) {
+ if (path == null) {
+ path = handler.filePath(entry).toString();
+ }
+ checkState(
+ partition.deletions.put(
+ dv.dataFileName(),
+ new DeletionFile(
+ path,
+ dv.offset(),
+ dv.length(),
+ dv.cardinality()))
+ == null,
+ "Duplicate DV for anchor %s.",
+ dv.dataFileName());
+ removed.add(dv.dataFileName());
+ }
+ }
+ if (!removed.isEmpty()) {
+ partition.indexes.add(new IndexRewrite(entry.indexFile(), removed));
+ }
+ }
+
+ List result = new ArrayList<>();
+ long target = table.coreOptions().dvIndexFileTargetSize().getBytes();
+ partitions.forEach(
+ (partition, plan) -> {
+ Batch batch = new Batch(partition, target, result);
+ for (Specification spec : plan.specifications) {
+ List sources = new ArrayList<>();
+ long weight = 0;
+ for (Map.Entry anchor : spec.anchors.entrySet()) {
+ DeletionFile deletion = plan.deletions.get(anchor.getKey());
+ if (deletion != null) {
+ sources.add(
+ new Source(anchor.getKey(), anchor.getValue(), deletion));
+ weight += deletion.length();
+ }
+ }
+ if (!sources.isEmpty()) {
+ // Keep all contributors to an output anchor together, even across old
+ // indexes.
+ batch.ensureCapacity(weight);
+ batch.task.rewrites.add(new Rewrite(sources, spec.outputs));
+ }
+ }
+ for (IndexRewrite index : plan.indexes) {
+ long weight = 0;
+ for (DeletionVectorMeta dv : index.file.dvRanges().values()) {
+ if (!index.removed.contains(dv.dataFileName())) {
+ weight += dv.length();
+ }
+ }
+ batch.ensureCapacity(weight);
+ batch.task.indexes.add(index);
+ }
+ batch.finish();
+ });
+ return result;
+ }
+
+ private static List normalFiles(List files) {
+ return files.stream()
+ .filter(file -> !isBlobFile(file.fileName()) && !isVectorStoreFile(file.fileName()))
+ .collect(Collectors.toList());
+ }
+
+ private static class PartitionPlan {
+ private final Set removed = new LinkedHashSet<>();
+ private final Map deletions = new LinkedHashMap<>();
+ private final List specifications = new ArrayList<>();
+ private final List indexes = new ArrayList<>();
+ }
+
+ private static class Specification {
+ private final Map anchors;
+ private final Map outputs;
+
+ private Specification(Map anchors, Map outputs) {
+ this.anchors = anchors;
+ this.outputs = outputs;
+ }
+ }
+
+ private static class Batch {
+ private final BinaryRow partition;
+ private final long target;
+ private final List result;
+ private DataEvolutionDeletionVectorRewriteTask task;
+ private long weight;
+
+ private Batch(
+ BinaryRow partition,
+ long target,
+ List result) {
+ this.partition = partition;
+ this.target = target;
+ this.result = result;
+ this.task = new DataEvolutionDeletionVectorRewriteTask(partition);
+ }
+
+ private void ensureCapacity(long nextWeight) {
+ if (task.workCount() > 0
+ && (weight + nextWeight > target
+ || (nextWeight == 0
+ && task.indexes.size() >= MAX_DELETE_ONLY_FILES_PER_TASK))) {
+ finish();
+ }
+ weight += nextWeight;
+ }
+
+ private void finish() {
+ if (task.workCount() > 0) {
+ result.add(task);
+ task = new DataEvolutionDeletionVectorRewriteTask(partition);
+ weight = 0;
+ }
+ }
+ }
+}
diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionDeletionVectorRewriteTask.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionDeletionVectorRewriteTask.java
new file mode 100644
index 000000000000..05b34647956a
--- /dev/null
+++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionDeletionVectorRewriteTask.java
@@ -0,0 +1,323 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.append.dataevolution;
+
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.data.serializer.VersionedSerializer;
+import org.apache.paimon.deletionvectors.Bitmap64DeletionVector;
+import org.apache.paimon.deletionvectors.BitmapDeletionVector;
+import org.apache.paimon.deletionvectors.DeletionVector;
+import org.apache.paimon.deletionvectors.DeletionVectorsIndexFile;
+import org.apache.paimon.index.IndexFileMeta;
+import org.apache.paimon.index.IndexFileMetaSerializer;
+import org.apache.paimon.io.CompactIncrement;
+import org.apache.paimon.io.DataIncrement;
+import org.apache.paimon.io.DataInputDeserializer;
+import org.apache.paimon.io.DataOutputSerializer;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.sink.CommitMessage;
+import org.apache.paimon.table.sink.CommitMessageImpl;
+import org.apache.paimon.table.source.DeletionFile;
+import org.apache.paimon.utils.Range;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Set;
+import java.util.TreeMap;
+
+import static org.apache.paimon.table.BucketMode.UNAWARE_BUCKET;
+import static org.apache.paimon.utils.Preconditions.checkArgument;
+import static org.apache.paimon.utils.Preconditions.checkState;
+import static org.apache.paimon.utils.SerializationUtils.deserializeBinaryRow;
+import static org.apache.paimon.utils.SerializationUtils.readCount;
+import static org.apache.paimon.utils.SerializationUtils.serializeBinaryRow;
+
+/**
+ * A partition-local batch of complete anchor rewrites and uniquely owned old index files.
+ *
+ *
New anchor bitmaps share rolling output files. Retained entries are written separately for
+ * each old index, preserving the conflict boundaries of subsequent deletes.
+ */
+public class DataEvolutionDeletionVectorRewriteTask {
+
+ final BinaryRow partition;
+ final List rewrites = new ArrayList<>();
+ final List indexes = new ArrayList<>();
+
+ DataEvolutionDeletionVectorRewriteTask(BinaryRow partition) {
+ this.partition = partition;
+ }
+
+ public BinaryRow partition() {
+ return partition;
+ }
+
+ public int workCount() {
+ return rewrites.size() + indexes.size();
+ }
+
+ public CommitMessage execute(FileStoreTable table) {
+ DeletionVectorsIndexFile index =
+ table.store().newIndexFileHandler().dvIndex(partition, UNAWARE_BUCKET);
+ boolean bitmap64 = table.coreOptions().deletionVectorBitmap64();
+ Iterator> bitmaps =
+ new Iterator>() {
+ private int rewritePosition;
+ private Iterator> current =
+ Collections.emptyIterator();
+
+ @Override
+ public boolean hasNext() {
+ while (!current.hasNext()) {
+ if (rewritePosition < rewrites.size()) {
+ current =
+ rewrite(rewrites.get(rewritePosition++), index, bitmap64)
+ .entrySet()
+ .iterator();
+ } else {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public Map.Entry next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ return current.next();
+ }
+ };
+ List added = new ArrayList<>();
+ try {
+ added.addAll(index.writeWithRolling(bitmaps));
+ for (IndexRewrite old : indexes) {
+ Map retained =
+ index.toDeletionFiles(Collections.singletonList(old.file));
+ old.removed.forEach(retained::remove);
+ added.addAll(index.writeWithRolling(index.readDeletionVector(retained)));
+ }
+ } catch (RuntimeException failure) {
+ for (IndexFileMeta file : added) {
+ try {
+ table.fileIO().deleteQuietly(index.path(file));
+ } catch (RuntimeException cleanupFailure) {
+ failure.addSuppressed(cleanupFailure);
+ }
+ }
+ throw failure;
+ }
+ List deleted = new ArrayList<>(indexes.size());
+ indexes.forEach(old -> deleted.add(old.file));
+ return new CommitMessageImpl(
+ partition,
+ UNAWARE_BUCKET,
+ null,
+ DataIncrement.emptyIncrement(),
+ new CompactIncrement(
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ added,
+ deleted));
+ }
+
+ private static Map rewrite(
+ Rewrite rewrite, DeletionVectorsIndexFile index, boolean bitmap64) {
+ Map> byIndex = new LinkedHashMap<>();
+ Map inputRanges = new LinkedHashMap<>();
+ for (Source source : rewrite.sources) {
+ byIndex.computeIfAbsent(source.deletion.path(), ignored -> new LinkedHashMap<>())
+ .put(source.anchor, source.deletion);
+ inputRanges.put(source.anchor, source.range);
+ }
+ TreeMap> outputs = new TreeMap<>();
+ rewrite.outputs.entrySet().forEach(output -> outputs.put(output.getValue().from, output));
+ Map result = new LinkedHashMap<>();
+ for (Map inputs : byIndex.values()) {
+ index.readDeletionVector(inputs)
+ .forEach(
+ (anchor, bitmap) ->
+ move(
+ bitmap,
+ inputRanges.get(anchor),
+ outputs,
+ result,
+ bitmap64));
+ }
+ return result;
+ }
+
+ static void move(
+ DeletionVector bitmap,
+ Range input,
+ TreeMap> outputs,
+ Map result,
+ boolean bitmap64) {
+ if (bitmap.isEmpty()) {
+ return;
+ }
+ Map.Entry first = outputs.floorEntry(input.from).getValue();
+ if (input.equals(first.getValue())) {
+ // Disjoint input ranges make this output exclusive to the current anchor.
+ result.put(first.getKey(), bitmap);
+ return;
+ }
+ bitmap.forEachDeletedPosition(
+ position -> {
+ long rowId = input.from + position;
+ Map.Entry output = outputs.floorEntry(rowId).getValue();
+ long offset = rowId - output.getValue().from;
+ checkState(
+ offset >= 0 && offset < output.getValue().count(),
+ "Deleted row ID %s is outside output range %s.",
+ rowId,
+ output.getValue());
+ result.computeIfAbsent(output.getKey(), ignored -> newBitmap(bitmap64))
+ .delete(offset);
+ });
+ }
+
+ private static DeletionVector newBitmap(boolean bitmap64) {
+ return bitmap64 ? new Bitmap64DeletionVector() : new BitmapDeletionVector();
+ }
+
+ static class Source {
+ final String anchor;
+ final Range range;
+ final DeletionFile deletion;
+
+ Source(String anchor, Range range, DeletionFile deletion) {
+ this.anchor = anchor;
+ this.range = range;
+ this.deletion = deletion;
+ }
+ }
+
+ static class Rewrite {
+ final List sources;
+ final Map outputs;
+
+ Rewrite(List sources, Map outputs) {
+ this.sources = sources;
+ this.outputs = outputs;
+ }
+ }
+
+ static class IndexRewrite {
+ final IndexFileMeta file;
+ final Set removed;
+
+ IndexRewrite(IndexFileMeta file, Set removed) {
+ this.file = file;
+ this.removed = removed;
+ }
+ }
+
+ /** Runtime-only task format shared by Flink and Spark. */
+ public static class Serializer
+ implements VersionedSerializer {
+
+ private final IndexFileMetaSerializer indexes = new IndexFileMetaSerializer();
+
+ @Override
+ public int getVersion() {
+ return 1;
+ }
+
+ @Override
+ public byte[] serialize(DataEvolutionDeletionVectorRewriteTask task) throws IOException {
+ DataOutputSerializer out = new DataOutputSerializer(1024);
+ serializeBinaryRow(task.partition, out);
+ out.writeInt(task.rewrites.size());
+ for (Rewrite rewrite : task.rewrites) {
+ out.writeInt(rewrite.sources.size());
+ for (Source source : rewrite.sources) {
+ out.writeUTF(source.anchor);
+ out.writeLong(source.range.from);
+ out.writeLong(source.range.to);
+ DeletionFile.serialize(out, source.deletion);
+ }
+ out.writeInt(rewrite.outputs.size());
+ for (Map.Entry output : rewrite.outputs.entrySet()) {
+ out.writeUTF(output.getKey());
+ out.writeLong(output.getValue().from);
+ out.writeLong(output.getValue().to);
+ }
+ }
+ out.writeInt(task.indexes.size());
+ for (IndexRewrite index : task.indexes) {
+ indexes.serializeList(Collections.singletonList(index.file), out);
+ out.writeInt(index.removed.size());
+ for (String anchor : index.removed) {
+ out.writeUTF(anchor);
+ }
+ }
+ return out.getCopyOfBuffer();
+ }
+
+ @Override
+ public DataEvolutionDeletionVectorRewriteTask deserialize(int version, byte[] bytes)
+ throws IOException {
+ checkArgument(
+ version == getVersion(), "Unsupported DV rewrite task version %s.", version);
+ DataInputDeserializer in = new DataInputDeserializer(bytes);
+ DataEvolutionDeletionVectorRewriteTask task =
+ new DataEvolutionDeletionVectorRewriteTask(deserializeBinaryRow(in));
+ int rewriteCount = readCount(in, "DV rewrites");
+ for (int i = 0; i < rewriteCount; i++) {
+ int sourceCount = readCount(in, "DV sources");
+ List sources = new ArrayList<>();
+ for (int j = 0; j < sourceCount; j++) {
+ sources.add(
+ new Source(
+ in.readUTF(),
+ new Range(in.readLong(), in.readLong()),
+ DeletionFile.deserialize(in)));
+ }
+ int outputCount = readCount(in, "DV outputs");
+ Map outputs = new LinkedHashMap<>();
+ for (int j = 0; j < outputCount; j++) {
+ outputs.put(in.readUTF(), new Range(in.readLong(), in.readLong()));
+ }
+ task.rewrites.add(new Rewrite(sources, outputs));
+ }
+ int indexCount = readCount(in, "DV index rewrites");
+ for (int i = 0; i < indexCount; i++) {
+ List files = indexes.deserializeList(in);
+ checkArgument(files.size() == 1, "Expected one owned DV index file.");
+ int removedCount = readCount(in, "Removed DV anchors");
+ Set removed = new HashSet<>();
+ for (int j = 0; j < removedCount; j++) {
+ removed.add(in.readUTF());
+ }
+ task.indexes.add(new IndexRewrite(files.get(0), removed));
+ }
+ return task;
+ }
+ }
+}
diff --git a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorIndexFileWriter.java b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorIndexFileWriter.java
index 6ce9e1c3875e..aebabc5f6545 100644
--- a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorIndexFileWriter.java
+++ b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorIndexFileWriter.java
@@ -25,7 +25,6 @@
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -65,13 +64,22 @@ public IndexFileMeta writeSingleFile(Map input) throws I
public List writeWithRolling(Map input)
throws IOException {
- if (input.isEmpty()) {
- return Collections.emptyList();
- }
+ return writeWithRolling(input.entrySet().iterator());
+ }
+
+ /** Streams complete anchor bitmaps across work units into shared rolling files. */
+ public List writeWithRolling(
+ Iterator> iterator) throws IOException {
List result = new ArrayList<>();
- Iterator> iterator = input.entrySet().iterator();
- while (iterator.hasNext()) {
- result.add(tryWriter(iterator));
+ try {
+ while (iterator.hasNext()) {
+ result.add(tryWriter(iterator));
+ }
+ } catch (IOException | RuntimeException e) {
+ for (IndexFileMeta file : result) {
+ fileIO.deleteQuietly(indexPathFactory.toPath(file));
+ }
+ throw e;
}
return result;
}
@@ -87,8 +95,20 @@ private IndexFileMeta tryWriter(Iterator> iter
break;
}
}
- } finally {
+ } catch (IOException | RuntimeException e) {
+ try {
+ writer.close();
+ } catch (IOException closeError) {
+ e.addSuppressed(closeError);
+ }
+ fileIO.deleteQuietly(indexPathFactory.toPath(writer.result()));
+ throw e;
+ }
+ try {
writer.close();
+ } catch (IOException e) {
+ fileIO.deleteQuietly(indexPathFactory.toPath(writer.result()));
+ throw e;
}
return writer.result();
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFile.java b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFile.java
index 04b37e2deb12..b53334719f83 100644
--- a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFile.java
+++ b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFile.java
@@ -33,6 +33,7 @@
import java.io.InputStream;
import java.io.UncheckedIOException;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -177,6 +178,14 @@ public List writeWithRolling(Map input) {
}
}
+ public List writeWithRolling(Iterator> input) {
+ try {
+ return createWriter().writeWithRolling(input);
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to write deletion vectors.", e);
+ }
+ }
+
private DeletionVectorIndexFileWriter createWriter() {
return new DeletionVectorIndexFileWriter(
this.fileIO, this.pathFactory, this.targetSizePerIndexFile);
diff --git a/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFileTest.java b/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFileTest.java
index c1a8e2a0fed2..0d3bbcd608b7 100644
--- a/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFileTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFileTest.java
@@ -38,6 +38,7 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -54,6 +55,41 @@ public class DeletionVectorsIndexFileTest {
@TempDir java.nio.file.Path tempPath;
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ public void testStreamingRewriteFailureRemovesOutputs(boolean rolling) throws IOException {
+ DeletionVectorsIndexFile index =
+ deletionVectorsIndexFile(
+ getPathFactory(), MemorySize.ofBytes(rolling ? 1 : Long.MAX_VALUE), false);
+ DeletionVector bitmap = new BitmapDeletionVector();
+ bitmap.delete(1);
+ Iterator> input =
+ new Iterator>() {
+ private int position;
+
+ @Override
+ public boolean hasNext() {
+ if (position == 2) {
+ throw new IllegalStateException("Injected bitmap read failure");
+ }
+ return true;
+ }
+
+ @Override
+ public Map.Entry next() {
+ return Collections.singletonMap("file-" + position++, bitmap)
+ .entrySet()
+ .iterator()
+ .next();
+ }
+ };
+ assertThatThrownBy(() -> index.writeWithRolling(input))
+ .hasMessage("Injected bitmap read failure");
+ try (Stream files = Files.list(tempPath)) {
+ assertThat(files).isEmpty();
+ }
+ }
+
@ParameterizedTest
@ValueSource(booleans = {false, true})
public void testReadDvIndex(boolean bitmap64) {
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionDeletionVectorTest.java b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionDeletionVectorTest.java
index e1de0d660b02..507f33137d92 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionDeletionVectorTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionDeletionVectorTest.java
@@ -27,6 +27,8 @@
import org.apache.paimon.append.dataevolution.DataEvolutionCompactTaskSerializer;
import org.apache.paimon.append.dataevolution.DataEvolutionCompactionCommitPreparation;
import org.apache.paimon.append.dataevolution.DataEvolutionDeletionVectorMaterializeCoordinator;
+import org.apache.paimon.append.dataevolution.DataEvolutionDeletionVectorRewritePlanner;
+import org.apache.paimon.append.dataevolution.DataEvolutionDeletionVectorRewriteTask;
import org.apache.paimon.append.dataevolution.DataEvolutionNormalCompactTask;
import org.apache.paimon.append.dataevolution.DataEvolutionRangeSortPlan;
import org.apache.paimon.append.dataevolution.DataEvolutionRowIdReassigner;
@@ -39,6 +41,7 @@
import org.apache.paimon.data.serializer.InternalRowSerializer;
import org.apache.paimon.deletionvectors.BitmapDeletionVector;
import org.apache.paimon.deletionvectors.DeletionVector;
+import org.apache.paimon.deletionvectors.DeletionVectorsIndexFile;
import org.apache.paimon.deletionvectors.append.BaseAppendDeleteFileMaintainer;
import org.apache.paimon.format.blob.BlobFileFormat;
import org.apache.paimon.fs.FileIO;
@@ -129,6 +132,178 @@ public class DataEvolutionDeletionVectorTest extends DataEvolutionTestBase {
new DvSpec(new Range(5, 9), 6),
new DvSpec(new Range(10, 14), 10, 12));
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ public void testDistributedDvRewriteWithSharedOldIndexes(boolean splitTasks) throws Exception {
+ FileStoreTable table = createPartitionedReassignTable("distributed_dv", false);
+ for (int first = 0; first < 18; first += 3) {
+ writePartitionRows(table, "a", first, first + 1, first + 2);
+ }
+ BinaryRow partition = partition(table, "a");
+ // The first index is shared by both compact messages and an untouched anchor.
+ commitDeletionVectors(
+ table,
+ partition,
+ Arrays.asList(
+ new DvSpec(new Range(0, 2), 1),
+ new DvSpec(new Range(6, 8), 7),
+ new DvSpec(new Range(12, 14), 13)));
+ commitDeletionVectors(
+ table,
+ partition,
+ Arrays.asList(
+ new DvSpec(new Range(3, 5), 4),
+ new DvSpec(new Range(9, 11), 10),
+ new DvSpec(new Range(15, 17), 16)));
+ List rows = readPartitionedRows(table);
+ List oldIndexes = liveDeletionVectorIndexFileNames(table);
+ assertThat(oldIndexes).hasSize(2);
+ Snapshot snapshot = table.latestSnapshot().get();
+ List files = currentDataFiles(table, partition);
+ List messages = new ArrayList<>();
+ for (long first : new long[] {0, 6}) {
+ List inputs =
+ files.stream()
+ .filter(
+ file ->
+ file.nonNullFirstRowId() >= first
+ && file.nonNullFirstRowId() < first + 6)
+ .collect(Collectors.toList());
+ messages.add(
+ new DataEvolutionNormalCompactTask(
+ partition, inputs, Collections.emptyList(), false)
+ .doCompact(table, "distributed-dv"));
+ }
+ FileStoreTable rewriteTable =
+ splitTasks
+ ? table.copy(
+ Collections.singletonMap(
+ CoreOptions.DELETION_VECTOR_INDEX_FILE_TARGET_SIZE.key(),
+ "1 b"))
+ : table;
+ List tasks =
+ new DataEvolutionDeletionVectorRewritePlanner(rewriteTable, snapshot)
+ .plan(messages);
+ if (splitTasks) {
+ assertThat(tasks.size()).isGreaterThan(1);
+ } else {
+ assertThat(tasks).hasSize(1);
+ }
+ // Reverse execution order after transport, so no task can depend on another's bitmap
+ // output.
+ Collections.reverse(tasks);
+ DataEvolutionDeletionVectorRewriteTask.Serializer serializer =
+ new DataEvolutionDeletionVectorRewriteTask.Serializer();
+ List deleted = new ArrayList<>();
+ List added = new ArrayList<>();
+ for (DataEvolutionDeletionVectorRewriteTask task : tasks) {
+ CommitMessage rewritten =
+ serializer
+ .deserialize(serializer.getVersion(), serializer.serialize(task))
+ .execute(rewriteTable);
+ CompactIncrement increment = ((CommitMessageImpl) rewritten).compactIncrement();
+ increment.deletedIndexFiles().forEach(file -> deleted.add(file.fileName()));
+ added.addAll(increment.newIndexFiles());
+ messages.add(rewritten);
+ }
+ assertThat(deleted).containsExactlyInAnyOrderElementsOf(oldIndexes);
+ if (!splitTasks) {
+ assertThat(added).hasSize(3);
+ }
+ commit(table, messages);
+ assertThat(readPartitionedRows(table)).containsExactlyInAnyOrderElementsOf(rows);
+ assertThat(liveDeletionVectorDataFileNames(table)).hasSize(4);
+
+ // Prepare both deletes from the same snapshot. Retained anchors from different old indexes
+ // must remain independently updatable after compaction.
+ CommitMessage firstDelete =
+ prepareDeletionVectors(
+ table,
+ partition,
+ Collections.singletonList(new DvSpec(new Range(12, 14), 14)));
+ CommitMessage secondDelete =
+ prepareDeletionVectors(
+ table,
+ partition,
+ Collections.singletonList(new DvSpec(new Range(15, 17), 17)));
+ commit(table, Collections.singletonList(firstDelete));
+ commit(table, Collections.singletonList(secondDelete));
+ assertThat(readPartitionedRows(table))
+ .containsExactlyInAnyOrderElementsOf(
+ rows.stream()
+ .filter(row -> !row.startsWith("a|14|") && !row.startsWith("a|17|"))
+ .collect(Collectors.toList()));
+ }
+
+ @Test
+ public void testFailedDvRewriteRemovesPreviousTaskOutputs() throws Exception {
+ FileStoreTable table =
+ createPartitionedReassignTable("failed_dv_rewrite", false)
+ .copy(
+ Collections.singletonMap(
+ CoreOptions.DELETION_VECTOR_INDEX_FILE_TARGET_SIZE.key(),
+ "1 b"));
+ for (int first = 0; first < 12; first += 3) {
+ writePartitionRows(table, "a", first, first + 1, first + 2);
+ }
+ BinaryRow partition = partition(table, "a");
+ for (long first : new long[] {0, 6}) {
+ commitDeletionVectors(
+ table,
+ partition,
+ Collections.singletonList(new DvSpec(new Range(first, first + 2), first + 1)));
+ }
+ Snapshot snapshot = table.latestSnapshot().get();
+ List files = currentDataFiles(table, partition);
+ List messages = new ArrayList<>();
+ for (long first : new long[] {0, 6}) {
+ List inputs =
+ files.stream()
+ .filter(
+ file ->
+ file.nonNullFirstRowId() >= first
+ && file.nonNullFirstRowId() < first + 6)
+ .collect(Collectors.toList());
+ messages.add(
+ new DataEvolutionNormalCompactTask(
+ partition, inputs, Collections.emptyList(), false)
+ .doCompact(table, "failed-dv-rewrite"));
+ }
+ assertThat(new DataEvolutionDeletionVectorRewritePlanner(table, snapshot).plan(messages))
+ .hasSizeGreaterThan(1);
+
+ IndexFileHandler handler = table.store().newIndexFileHandler();
+ DeletionVectorsIndexFile index = handler.dvIndex(partition, UNAWARE_BUCKET);
+ List oldIndexes =
+ handler.scan(snapshot, DELETION_VECTORS_INDEX, partition, UNAWARE_BUCKET);
+ String unreadableAnchor = anchorFilesByRange(table, partition).get(new Range(6, 8));
+ IndexFileMeta unreadable =
+ oldIndexes.stream()
+ .filter(file -> file.dvRanges().containsKey(unreadableAnchor))
+ .findFirst()
+ .get();
+ table.fileIO().delete(index.path(unreadable), false);
+ Path indexDirectory = index.path(unreadable).getParent();
+ Set filesBefore =
+ Arrays.stream(table.fileIO().listStatus(indexDirectory))
+ .map(status -> status.getPath())
+ .collect(Collectors.toSet());
+
+ assertThatThrownBy(
+ () ->
+ new DataEvolutionCompactDeletionVectorRewriter(table, snapshot)
+ .rewriteDeletionVectors(messages))
+ .isInstanceOf(RuntimeException.class)
+ .hasMessageContaining("Unable to read deletion vector");
+
+ assertThat(
+ Arrays.stream(table.fileIO().listStatus(indexDirectory))
+ .map(status -> status.getPath())
+ .collect(Collectors.toSet()))
+ .isEqualTo(filesBefore);
+ assertThat(table.latestSnapshot().get().id()).isEqualTo(snapshot.id());
+ }
+
@Test
public void testCompactScansDeletionVectorIndexesOnceAcrossPartitions() throws Exception {
FileStoreTable table = createPartitionedReassignTable("compact_dv_scan", false);
@@ -2112,6 +2287,14 @@ private void commitDeletionVectors(FileStoreTable table, List deletionVe
private void commitDeletionVectors(
FileStoreTable table, BinaryRow partition, List deletionVectorSpecs)
throws Exception {
+ commit(
+ table,
+ Collections.singletonList(
+ prepareDeletionVectors(table, partition, deletionVectorSpecs)));
+ }
+
+ private CommitMessage prepareDeletionVectors(
+ FileStoreTable table, BinaryRow partition, List deletionVectorSpecs) {
BaseAppendDeleteFileMaintainer maintainer =
BaseAppendDeleteFileMaintainer.forUnawareAppend(
table.store().newIndexFileHandler(),
@@ -2137,20 +2320,17 @@ private void commitDeletionVectors(
}
}
- commit(
- table,
- Collections.singletonList(
- new CommitMessageImpl(
- partition,
- UNAWARE_BUCKET,
- null,
- new DataIncrement(
- Collections.emptyList(),
- Collections.emptyList(),
- Collections.emptyList(),
- newIndexFiles,
- deletedIndexFiles),
- CompactIncrement.emptyIncrement())));
+ return new CommitMessageImpl(
+ partition,
+ UNAWARE_BUCKET,
+ null,
+ new DataIncrement(
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ newIndexFiles,
+ deletedIndexFiles),
+ CompactIncrement.emptyIncrement());
}
private static void commit(FileStoreTable table, List commitMessages)
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionCommitPreparationOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionCommitPreparationOperator.java
index f2c0aa7bffdb..3ca97ef9f20c 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionCommitPreparationOperator.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionCommitPreparationOperator.java
@@ -19,12 +19,13 @@
package org.apache.paimon.flink.sink;
import org.apache.paimon.Snapshot;
-import org.apache.paimon.append.dataevolution.DataEvolutionCompactionCommitPreparation;
import org.apache.paimon.options.Options;
import org.apache.paimon.table.FileStoreTable;
-import org.apache.paimon.table.sink.CommitMessage;
import org.apache.paimon.table.sink.CommitMessageImpl;
+import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.operators.StreamOperator;
import org.apache.flink.streaming.api.operators.StreamOperatorFactory;
import org.apache.flink.streaming.api.operators.StreamOperatorParameters;
@@ -34,22 +35,50 @@
import java.util.ArrayList;
import java.util.List;
-/** Prepares deletion-vector and index changes for a data-evolution rewrite commit. */
+/** Prepares commits after all distributed DV rewrites finish. */
public class DataEvolutionCommitPreparationOperator
extends PrepareCommitOperator {
- private final FileStoreTable table;
private final Snapshot snapshot;
private final boolean materializeDvRowIdCheck;
private final List committables;
+ static DataStream prepare(
+ DataStream written,
+ FileStoreTable table,
+ Snapshot snapshot,
+ boolean materializeDvRowIdCheck) {
+ SingleOutputStreamOperator planned =
+ written.transform(
+ "Data Evolution DV Rewrite Planner : " + table.name(),
+ PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO,
+ new DataEvolutionDeletionVectorRewriteOperator.PlannerFactory(
+ table, snapshot))
+ .forceNonParallel();
+ DataStream rewritten =
+ planned.rebalance()
+ .transform(
+ "Data Evolution DV Rewriter : " + table.name(),
+ new CommittableTypeInfo(),
+ new DataEvolutionDeletionVectorRewriteOperator.Factory(table))
+ .setParallelism(written.getParallelism());
+ return rewritten
+ .union(
+ planned.getSideOutput(
+ DataEvolutionDeletionVectorRewriteOperator.DATA_COMMITS))
+ .transform(
+ "Data Evolution Commit Preparation : " + table.name(),
+ new CommittableTypeInfo(),
+ new Factory(table, snapshot, materializeDvRowIdCheck))
+ .forceNonParallel();
+ }
+
private DataEvolutionCommitPreparationOperator(
StreamOperatorParameters parameters,
FileStoreTable table,
Snapshot snapshot,
boolean materializeDvRowIdCheck) {
super(parameters, Options.fromMap(table.options()));
- this.table = table;
this.snapshot = snapshot;
this.materializeDvRowIdCheck = materializeDvRowIdCheck;
this.committables = new ArrayList<>();
@@ -69,14 +98,6 @@ protected List prepareCommit(boolean waitCompaction, long checkpoin
return toCommit;
}
- List messages = new ArrayList<>(toCommit.size());
- for (Committable committable : toCommit) {
- messages.add(committable.commitMessage());
- }
- for (CommitMessage message :
- new DataEvolutionCompactionCommitPreparation(table, snapshot).prepare(messages)) {
- toCommit.add(new Committable(toCommit.get(0).checkpointId(), message));
- }
if (!materializeDvRowIdCheck) {
return toCommit;
}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionDeletionVectorRewriteOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionDeletionVectorRewriteOperator.java
new file mode 100644
index 000000000000..5e181049feac
--- /dev/null
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionDeletionVectorRewriteOperator.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.flink.sink;
+
+import org.apache.paimon.Snapshot;
+import org.apache.paimon.append.dataevolution.DataEvolutionDeletionVectorRewritePlanner;
+import org.apache.paimon.append.dataevolution.DataEvolutionDeletionVectorRewriteTask;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.index.IndexFileMeta;
+import org.apache.paimon.index.IndexPathFactory;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.sink.CommitMessage;
+import org.apache.paimon.table.sink.CommitMessageImpl;
+
+import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
+import org.apache.flink.streaming.api.operators.AbstractStreamOperatorFactory;
+import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
+import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory;
+import org.apache.flink.streaming.api.operators.StreamOperator;
+import org.apache.flink.streaming.api.operators.StreamOperatorParameters;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.util.OutputTag;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/** Executes partition-local DV batches in parallel, emitting only commit metadata. */
+public class DataEvolutionDeletionVectorRewriteOperator extends AbstractStreamOperator
+ implements OneInputStreamOperator {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DataEvolutionDeletionVectorRewriteOperator.class);
+
+ static final OutputTag DATA_COMMITS =
+ new OutputTag<>("data-evolution-data-commits", new CommittableTypeInfo());
+
+ private final FileStoreTable table;
+ private final DataEvolutionDeletionVectorRewriteTask.Serializer serializer =
+ new DataEvolutionDeletionVectorRewriteTask.Serializer();
+ private final List outputs = new ArrayList<>();
+ private boolean finished;
+
+ private DataEvolutionDeletionVectorRewriteOperator(
+ StreamOperatorParameters parameters, FileStoreTable table) {
+ this.table = table;
+ setup(parameters.getContainingTask(), parameters.getStreamConfig(), parameters.getOutput());
+ }
+
+ @Override
+ public void processElement(StreamRecord element) throws Exception {
+ try {
+ DataEvolutionDeletionVectorRewriteTask task =
+ serializer.deserialize(serializer.getVersion(), element.getValue());
+ CommitMessageImpl message = (CommitMessageImpl) task.execute(table);
+ IndexPathFactory paths =
+ table.store()
+ .pathFactory()
+ .indexFileFactory(message.partition(), message.bucket());
+ for (IndexFileMeta file : message.compactIncrement().newIndexFiles()) {
+ outputs.add(paths.toPath(file));
+ }
+ output.collect(new StreamRecord<>(new Committable(Long.MAX_VALUE, message)));
+ } catch (Exception failure) {
+ cleanup();
+ throw failure;
+ }
+ }
+
+ @Override
+ public void finish() throws Exception {
+ super.finish();
+ finished = true;
+ outputs.clear();
+ }
+
+ @Override
+ public void close() throws Exception {
+ try {
+ if (!finished) {
+ cleanup();
+ }
+ } finally {
+ super.close();
+ }
+ }
+
+ private void cleanup() {
+ for (Path output : outputs) {
+ try {
+ table.fileIO().deleteQuietly(output);
+ } catch (RuntimeException failure) {
+ LOG.warn("Failed to delete uncommitted DV index file {}.", output, failure);
+ }
+ }
+ outputs.clear();
+ }
+
+ /** Collects all data outputs before assigning each old DV index file to exactly one task. */
+ private static class Planner extends PrepareCommitOperator {
+
+ private final FileStoreTable table;
+ private final Snapshot snapshot;
+ private final List committables = new ArrayList<>();
+
+ private Planner(
+ StreamOperatorParameters parameters,
+ FileStoreTable table,
+ Snapshot snapshot) {
+ super(parameters, Options.fromMap(table.options()));
+ this.table = table;
+ this.snapshot = snapshot;
+ }
+
+ @Override
+ public void processElement(StreamRecord element) {
+ committables.add(element.getValue());
+ }
+
+ @Override
+ protected List prepareCommit(boolean waitCompaction, long checkpointId)
+ throws IOException {
+ List messages = new ArrayList<>(committables.size());
+ for (Committable committable : committables) {
+ messages.add(committable.commitMessage());
+ output.collect(DATA_COMMITS, new StreamRecord<>(committable));
+ }
+ committables.clear();
+ List tasks = new ArrayList<>();
+ DataEvolutionDeletionVectorRewriteTask.Serializer serializer =
+ new DataEvolutionDeletionVectorRewriteTask.Serializer();
+ for (DataEvolutionDeletionVectorRewriteTask task :
+ new DataEvolutionDeletionVectorRewritePlanner(table, snapshot).plan(messages)) {
+ tasks.add(serializer.serialize(task));
+ }
+ return tasks;
+ }
+ }
+
+ /** Factory for the metadata-only planner. */
+ public static class PlannerFactory extends PrepareCommitOperator.Factory {
+ private final FileStoreTable table;
+ private final Snapshot snapshot;
+
+ public PlannerFactory(FileStoreTable table, Snapshot snapshot) {
+ super(Options.fromMap(table.options()));
+ this.table = table;
+ this.snapshot = snapshot;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public > T createStreamOperator(
+ StreamOperatorParameters parameters) {
+ return (T) new Planner(parameters, table, snapshot);
+ }
+
+ @Override
+ @SuppressWarnings("rawtypes")
+ public Class extends StreamOperator> getStreamOperatorClass(ClassLoader classLoader) {
+ return Planner.class;
+ }
+ }
+
+ /** Factory for the parallel bitmap workers. */
+ public static class Factory extends AbstractStreamOperatorFactory
+ implements OneInputStreamOperatorFactory {
+ private final FileStoreTable table;
+
+ public Factory(FileStoreTable table) {
+ this.table = table;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public > T createStreamOperator(
+ StreamOperatorParameters parameters) {
+ return (T) new DataEvolutionDeletionVectorRewriteOperator(parameters, table);
+ }
+
+ @Override
+ @SuppressWarnings("rawtypes")
+ public Class extends StreamOperator> getStreamOperatorClass(ClassLoader classLoader) {
+ return DataEvolutionDeletionVectorRewriteOperator.class;
+ }
+ }
+}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionRowIdRewriteSink.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionRowIdRewriteSink.java
index 3d013bb85d6a..92de51ba5f40 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionRowIdRewriteSink.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionRowIdRewriteSink.java
@@ -80,14 +80,7 @@ public static DataStreamSink> commitSorted(
private DataStreamSink> commitWritten(
DataStream written, String initialCommitUser) {
- written =
- written.transform(
- "Data Evolution Row ID Rewrite Commit Preparation : "
- + table.name(),
- new CommittableTypeInfo(),
- new DataEvolutionCommitPreparationOperator.Factory(
- table, snapshot, true))
- .forceNonParallel();
+ written = DataEvolutionCommitPreparationOperator.prepare(written, table, snapshot, true);
return doCommit(written, initialCommitUser);
}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionTableCompactSink.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionTableCompactSink.java
index 97fc301b4674..e45a85373674 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionTableCompactSink.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionTableCompactSink.java
@@ -52,13 +52,7 @@ public DataStreamSink> sinkFrom(
DataStream written = doWrite(input, initialCommitUser, null);
if (table.coreOptions().deletionVectorsEnabled()) {
written =
- written.transform(
- "Data Evolution Compact Deletion Vector Rewriter : "
- + table.name(),
- new CommittableTypeInfo(),
- new DataEvolutionCommitPreparationOperator.Factory(
- table, snapshot))
- .forceNonParallel();
+ DataEvolutionCommitPreparationOperator.prepare(written, table, snapshot, false);
}
return doCommit(written, initialCommitUser);
}
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/DataEvolutionDeleteSqlITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/DataEvolutionDeleteSqlITCase.java
index 2115105394ab..fa6acf8a4f40 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/DataEvolutionDeleteSqlITCase.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/DataEvolutionDeleteSqlITCase.java
@@ -65,16 +65,26 @@ public void testDeleteRowsWithoutRewritingDataFiles() throws Exception {
}
@Test
- public void testMaterializeDeletionVectorsProcedure() throws Exception {
+ public void testCompactAndMaterializeDeletionVectorsProcedure() throws Exception {
tEnv.getConfig().set(TableConfigOptions.TABLE_DML_SYNC, true);
createTable();
sql("INSERT INTO T VALUES (1, 'one', 'A'), (2, 'two', 'A'), (3, 'three', 'A')");
+ sql("INSERT INTO T VALUES (7, 'seven', 'A'), (8, 'eight', 'A'), (9, 'nine', 'A')");
sql("INSERT INTO T VALUES (4, 'four', 'B'), (5, 'five', 'B'), (6, 'six', 'B')");
sql("DELETE FROM T WHERE id IN (2, 4)");
FileStoreTable table = paimonTable("T");
assertThat(deletionVectorCardinality(table)).isEqualTo(2L);
+ List rowsWithIds = sql("SELECT id, _ROW_ID FROM `T$row_tracking` ORDER BY id");
+ // Force separate bitmap/old-index work in the same partition through the parallel stage.
+ sql(
+ "CALL sys.compact(`table` => 'default.T', options => "
+ + "'compaction.min.file-num=2,deletion-vector.index-file.target-size=1 b')");
+ assertThat(sql("SELECT id, _ROW_ID FROM `T$row_tracking` ORDER BY id"))
+ .isEqualTo(rowsWithIds);
+ assertThat(deletionVectorCardinality(table)).isEqualTo(2L);
+
sql("CALL sys.materialize_deletion_vectors(`table` => 'default.T')");
table = paimonTable("T");
@@ -84,7 +94,10 @@ public void testMaterializeDeletionVectorsProcedure() throws Exception {
Row.of(1, "one", "A"),
Row.of(3, "three", "A"),
Row.of(5, "five", "B"),
- Row.of(6, "six", "B"));
+ Row.of(6, "six", "B"),
+ Row.of(7, "seven", "A"),
+ Row.of(8, "eight", "A"),
+ Row.of(9, "nine", "A"));
}
@Test
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/DataEvolutionDeletionVectorRewriteOperatorTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/DataEvolutionDeletionVectorRewriteOperatorTest.java
new file mode 100644
index 000000000000..19b6fb2e521b
--- /dev/null
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/DataEvolutionDeletionVectorRewriteOperatorTest.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.flink.sink;
+
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.Snapshot;
+import org.apache.paimon.append.dataevolution.DataEvolutionDeletionVectorRewritePlanner;
+import org.apache.paimon.append.dataevolution.DataEvolutionDeletionVectorRewriteTask;
+import org.apache.paimon.append.dataevolution.DataEvolutionNormalCompactTask;
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.deletionvectors.BitmapDeletionVector;
+import org.apache.paimon.deletionvectors.DeletionVectorsIndexFile;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.index.IndexFileMeta;
+import org.apache.paimon.io.CompactIncrement;
+import org.apache.paimon.io.DataFileMeta;
+import org.apache.paimon.io.DataIncrement;
+import org.apache.paimon.schema.Schema;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.TableTestBase;
+import org.apache.paimon.table.sink.CommitMessage;
+import org.apache.paimon.table.sink.CommitMessageImpl;
+import org.apache.paimon.table.sink.TableCommitImpl;
+import org.apache.paimon.types.DataTypes;
+
+import org.apache.flink.api.common.RuntimeExecutionMode;
+import org.apache.flink.api.common.functions.RichMapFunction;
+import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.RestartStrategyOptions;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.sink.v2.DiscardingSink;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+
+import static org.apache.paimon.table.BucketMode.UNAWARE_BUCKET;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Tests attempt-scoped cleanup of distributed deletion-vector rewrites. */
+public class DataEvolutionDeletionVectorRewriteOperatorTest extends TableTestBase {
+
+ private static final AtomicInteger LAST_ATTEMPT = new AtomicInteger();
+
+ private FileStoreTable table;
+ private DeletionVectorsIndexFile index;
+ private List oldIndexes;
+ private List tasks;
+
+ @BeforeEach
+ public void prepareRewrites() throws Exception {
+ createTableDefault();
+ table = getTableDefault();
+ try (TableCommitImpl commit = table.newCommit("insert")) {
+ commit.commit(writeDataDefault(2, 4));
+ }
+ List files =
+ table.store().newScan().plan().files().stream()
+ .map(entry -> entry.file())
+ .sorted(Comparator.comparingLong(DataFileMeta::nonNullFirstRowId))
+ .collect(Collectors.toList());
+ assertThat(files).hasSize(4);
+ index = table.store().newIndexFileHandler().dvIndex(BinaryRow.EMPTY_ROW, UNAWARE_BUCKET);
+ oldIndexes = new ArrayList<>();
+ for (int file : new int[] {0, 2}) {
+ BitmapDeletionVector bitmap = new BitmapDeletionVector();
+ bitmap.delete(0);
+ List added =
+ index.writeWithRolling(
+ Collections.singletonMap(files.get(file).fileName(), bitmap));
+ oldIndexes.addAll(added);
+ try (TableCommitImpl commit = table.newCommit("delete-" + file)) {
+ commit.commit(
+ Collections.singletonList(
+ new CommitMessageImpl(
+ BinaryRow.EMPTY_ROW,
+ UNAWARE_BUCKET,
+ null,
+ DataIncrement.indexIncrement(added),
+ CompactIncrement.emptyIncrement())));
+ }
+ }
+ Snapshot snapshot = table.latestSnapshot().get();
+ List compact = new ArrayList<>();
+ for (int first : new int[] {0, 2}) {
+ compact.add(
+ new DataEvolutionNormalCompactTask(
+ BinaryRow.EMPTY_ROW,
+ files.subList(first, first + 2),
+ Collections.emptyList(),
+ false)
+ .doCompact(table, "compact"));
+ }
+ tasks = new ArrayList<>();
+ DataEvolutionDeletionVectorRewriteTask.Serializer serializer =
+ new DataEvolutionDeletionVectorRewriteTask.Serializer();
+ for (DataEvolutionDeletionVectorRewriteTask task :
+ new DataEvolutionDeletionVectorRewritePlanner(
+ table.copy(
+ Collections.singletonMap(
+ CoreOptions.DELETION_VECTOR_INDEX_FILE_TARGET_SIZE
+ .key(),
+ "1 b")),
+ snapshot)
+ .plan(compact)) {
+ tasks.add(serializer.serialize(task));
+ }
+ assertThat(tasks).hasSizeGreaterThan(1);
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"failure", "cancel", "success"})
+ public void testAttemptCleanup(String outcome) throws Exception {
+ Set before = indexFiles();
+ try (OneInputStreamOperatorTestHarness harness =
+ new OneInputStreamOperatorTestHarness<>(
+ new DataEvolutionDeletionVectorRewriteOperator.Factory(table))) {
+ harness.setup(new CommittableTypeInfo().createSerializer(harness.getExecutionConfig()));
+ harness.open();
+ harness.processElement(new StreamRecord<>(tasks.get(0)));
+ assertThat(indexFiles()).hasSizeGreaterThan(before.size());
+ if ("failure".equals(outcome)) {
+ assertThatThrownBy(() -> harness.processElement(new StreamRecord<>(new byte[0])))
+ .isInstanceOf(Exception.class);
+ } else if ("success".equals(outcome)) {
+ harness.getOperator().finish();
+ } else {
+ // Harness.close() finishes normally; cancellation closes without finishing.
+ harness.getOperator().close();
+ }
+ }
+ if ("success".equals(outcome)) {
+ assertThat(indexFiles()).hasSizeGreaterThan(before.size());
+ } else {
+ assertThat(indexFiles()).isEqualTo(before);
+ }
+ }
+
+ @Test
+ public void testFailedWorkerAttemptCleansEarlierOutputsOnRestart() throws Exception {
+ index.delete(oldIndexes.get(1));
+ Set before = indexFiles();
+ long snapshot = table.latestSnapshot().get().id();
+ LAST_ATTEMPT.set(0);
+ Configuration configuration = new Configuration();
+ configuration.set(RestartStrategyOptions.RESTART_STRATEGY, "fixed-delay");
+ configuration.set(RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS, 1);
+ configuration.set(RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_DELAY, Duration.ZERO);
+ StreamExecutionEnvironment env =
+ StreamExecutionEnvironment.getExecutionEnvironment(configuration);
+ env.setRuntimeMode(RuntimeExecutionMode.BATCH);
+ env.setParallelism(1);
+ env.fromCollection(tasks, PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO)
+ .map(
+ new RichMapFunction() {
+ @Override
+ public byte[] map(byte[] value) {
+ LAST_ATTEMPT.set(
+ getRuntimeContext().getTaskInfo().getAttemptNumber());
+ return value;
+ }
+ })
+ .transform(
+ "DV rewrite",
+ new CommittableTypeInfo(),
+ new DataEvolutionDeletionVectorRewriteOperator.Factory(table))
+ .sinkTo(new DiscardingSink<>());
+ assertThatThrownBy(env::execute).hasStackTraceContaining("Unable to read deletion vector");
+ assertThat(LAST_ATTEMPT.get()).isEqualTo(1);
+ assertThat(indexFiles()).isEqualTo(before);
+ assertThat(table.latestSnapshot().get().id()).isEqualTo(snapshot);
+ }
+
+ private Set indexFiles() throws Exception {
+ return Arrays.stream(table.fileIO().listStatus(index.path(oldIndexes.get(0)).getParent()))
+ .map(status -> status.getPath())
+ .collect(Collectors.toSet());
+ }
+
+ @Override
+ protected Schema schemaDefault() {
+ return Schema.newBuilder()
+ .column("id", DataTypes.INT())
+ .option(CoreOptions.BUCKET.key(), "-1")
+ .option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true")
+ .option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true")
+ .option(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true")
+ .option(CoreOptions.WRITE_ONLY.key(), "true")
+ .build();
+ }
+
+ @Override
+ protected InternalRow dataDefault(int time, int size) {
+ return GenericRow.of(time * size);
+ }
+}
diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/DataEvolutionRewriteExecutor.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/DataEvolutionRewriteExecutor.java
index 950291628877..4aea0bd3e8a8 100644
--- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/DataEvolutionRewriteExecutor.java
+++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/DataEvolutionRewriteExecutor.java
@@ -22,6 +22,11 @@
import org.apache.paimon.append.dataevolution.DataEvolutionCompactTask;
import org.apache.paimon.append.dataevolution.DataEvolutionCompactTaskSerializer;
import org.apache.paimon.append.dataevolution.DataEvolutionCompactionCommitPreparation;
+import org.apache.paimon.append.dataevolution.DataEvolutionDeletionVectorRewriteTask;
+import org.apache.paimon.append.dataevolution.DataEvolutionDeletionVectorRewriteTask.Serializer;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.index.IndexFileMeta;
+import org.apache.paimon.index.IndexPathFactory;
import org.apache.paimon.operation.commit.DataEvolutionRowRangeConflictException;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.sink.CommitMessage;
@@ -32,10 +37,13 @@
import org.apache.paimon.utils.ExceptionUtils;
import org.apache.paimon.utils.RetryWaiter;
+import org.apache.spark.TaskContext;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.sql.SparkSession;
+import org.apache.spark.util.TaskCompletionListener;
+import org.apache.spark.util.TaskFailureListener;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -314,16 +322,33 @@ private static Snapshot commitWithMergeConflictRetry(
}
List preparedMessages = new ArrayList<>(attemptMessages);
- List preparationArtifacts =
- new DataEvolutionCompactionCommitPreparation(table, attemptSnapshot)
- .prepare(preparedMessages);
+ DataEvolutionCompactionCommitPreparation preparation =
+ new DataEvolutionCompactionCommitPreparation(table, attemptSnapshot);
+ List preparationArtifacts = new ArrayList<>();
+ try {
+ preparationArtifacts.addAll(
+ rewriteDeletionVectors(
+ table,
+ preparation.planDeletionVectorRewrites(attemptMessages),
+ sparkSession));
+ } catch (Exception failure) {
+ RuntimeException preparationFailure =
+ failure instanceof RuntimeException
+ ? (RuntimeException) failure
+ : new RuntimeException(
+ "Prepare deletion vector rewrites failed.", failure);
+ List staged = new ArrayList<>(retryArtifacts);
+ staged.addAll(preparationArtifacts);
+ abortRetryArtifacts(table, commitUser, staged, preparationFailure);
+ throw preparationFailure;
+ }
preparedMessages.addAll(preparationArtifacts);
List abortMessages = new ArrayList<>(retryArtifacts);
abortMessages.addAll(preparationArtifacts);
try (TableCommitImpl commit = table.newCommit(commitUser)) {
- commitConfigurer.configure(commit);
- commitConfigurer.prepareMessages(attemptSnapshot, preparedMessages);
try {
+ commitConfigurer.configure(commit);
+ commitConfigurer.prepareMessages(attemptSnapshot, preparedMessages);
commit.commit(preparedMessages);
} catch (RuntimeException conflict) {
if (commitConfigurer == RANGE_SORT_COMMIT) {
@@ -365,6 +390,102 @@ private static Snapshot commitWithMergeConflictRetry(
}
}
+ private static List rewriteDeletionVectors(
+ FileStoreTable table,
+ List tasks,
+ SparkSession sparkSession) {
+ if (tasks.isEmpty()) {
+ return new ArrayList<>();
+ }
+ Serializer serializer = new Serializer();
+ List serializedTasks = new ArrayList<>(tasks.size());
+ try {
+ for (DataEvolutionDeletionVectorRewriteTask task : tasks) {
+ serializedTasks.add(serializer.serialize(task));
+ }
+ int parallelism = readParallelism(serializedTasks, sparkSession);
+ LOG.info(
+ "Rewriting deletion vectors for table {} in {} tasks with parallelism {}.",
+ table.fullName(),
+ tasks.size(),
+ parallelism);
+ JavaRDD rewritten =
+ JavaSparkContext.fromSparkContext(sparkSession.sparkContext())
+ .parallelize(serializedTasks, parallelism)
+ .mapPartitions(
+ iterator ->
+ rewriteDeletionVectorPartition(
+ table, iterator, TaskContext.get())
+ .iterator());
+ // Successful tasks cannot observe a later failure elsewhere in the job. Their
+ // uncommitted files, and outputs lost with an executor, need orphan cleanup.
+ List messages = new ArrayList<>(rewritten.collect());
+ return deserializeCommitMessagesAndReleaseSerializedBytes(
+ new CommitMessageSerializer(), messages);
+ } catch (IOException e) {
+ throw new RuntimeException("Serialize deletion vector rewrite messages failed.", e);
+ }
+ }
+
+ static List rewriteDeletionVectorPartition(
+ FileStoreTable table, Iterator iterator, @Nullable TaskContext context)
+ throws IOException {
+ List outputs = new ArrayList<>();
+ Runnable cleanup =
+ () -> {
+ for (Path output : outputs) {
+ try {
+ table.fileIO().deleteQuietly(output);
+ } catch (RuntimeException failure) {
+ LOG.warn(
+ "Failed to delete uncommitted DV index file {}.",
+ output,
+ failure);
+ }
+ }
+ outputs.clear();
+ };
+ if (context != null) {
+ context.addTaskFailureListener(
+ (TaskFailureListener) (failed, failure) -> cleanup.run());
+ context.addTaskCompletionListener(
+ (TaskCompletionListener)
+ completed -> {
+ if (completed.isInterrupted()) {
+ cleanup.run();
+ }
+ });
+ }
+ Serializer taskSerializer = new Serializer();
+ CommitMessageSerializer messageSerializer = new CommitMessageSerializer();
+ List results = new ArrayList<>();
+ try {
+ while (iterator.hasNext()) {
+ if (context != null) {
+ context.killTaskIfInterrupted();
+ }
+ DataEvolutionDeletionVectorRewriteTask task =
+ taskSerializer.deserialize(taskSerializer.getVersion(), iterator.next());
+ CommitMessageImpl message = (CommitMessageImpl) task.execute(table);
+ IndexPathFactory paths =
+ table.store()
+ .pathFactory()
+ .indexFileFactory(message.partition(), message.bucket());
+ for (IndexFileMeta file : message.compactIncrement().newIndexFiles()) {
+ outputs.add(paths.toPath(file));
+ }
+ results.add(messageSerializer.serialize(message));
+ }
+ if (context != null) {
+ context.killTaskIfInterrupted();
+ }
+ return results;
+ } catch (IOException | RuntimeException failure) {
+ cleanup.run();
+ throw failure;
+ }
+ }
+
private static List retryArtifacts(
List compactMessages, List rewrittenMessages) {
checkArgument(
diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CompactProcedureTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CompactProcedureTestBase.scala
index 8671762798e1..89cf55890dfd 100644
--- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CompactProcedureTestBase.scala
+++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CompactProcedureTestBase.scala
@@ -20,13 +20,16 @@ package org.apache.paimon.spark.procedure
import org.apache.paimon.{CoreOptions, Snapshot}
import org.apache.paimon.Snapshot.CommitKind
-import org.apache.paimon.append.dataevolution.DataEvolutionNormalCompactTask
-import org.apache.paimon.data.BinaryRow
+import org.apache.paimon.append.dataevolution.{DataEvolutionCompactTask, DataEvolutionDeletionVectorRewritePlanner, DataEvolutionDeletionVectorRewriteTask, DataEvolutionNormalCompactTask}
+import org.apache.paimon.data.{BinaryRow, BinaryString, GenericRow}
import org.apache.paimon.deletionvectors.DeletionVectorsIndexFile.DELETION_VECTORS_INDEX
import org.apache.paimon.format.blob.BlobFileFormat
-import org.apache.paimon.fs.Path
-import org.apache.paimon.io.DataFileMeta
+import org.apache.paimon.fs.{Path, SeekableInputStream}
+import org.apache.paimon.fs.local.LocalFileIO
+import org.apache.paimon.index.IndexFileMeta
+import org.apache.paimon.io.{CompactIncrement, DataFileMeta}
import org.apache.paimon.manifest.FileSource
+import org.apache.paimon.operation.LocalOrphanFilesClean
import org.apache.paimon.operation.commit.DataEvolutionRowRangeConflictException
import org.apache.paimon.partition.PartitionPredicate
import org.apache.paimon.spark.PaimonSparkTestBase
@@ -34,12 +37,14 @@ import org.apache.paimon.spark.catalyst.analysis.PaimonRelation
import org.apache.paimon.spark.commands.{DataEvolutionCompactMergeConflictRewriter, DataEvolutionPaimonWriter, PaimonSparkWriter}
import org.apache.paimon.spark.commands.CompactRowIdRangeIndex
import org.apache.paimon.spark.utils.SparkProcedureUtils
-import org.apache.paimon.table.FileStoreTable
-import org.apache.paimon.table.sink.CommitMessageImpl
+import org.apache.paimon.table.{AppendOnlyFileStoreTable, FileStoreTable}
+import org.apache.paimon.table.BucketMode.UNAWARE_BUCKET
+import org.apache.paimon.table.sink.{CommitMessage, CommitMessageImpl, TableCommitImpl}
import org.apache.paimon.table.source.{DataSplit, IncrementalSplit}
import org.apache.paimon.table.source.snapshot.SnapshotReader
import org.apache.paimon.utils.Range
+import org.apache.spark.{SparkException, TaskContext}
import org.apache.spark.api.java.JavaSparkContext
import org.apache.spark.scheduler.{SparkListener, SparkListenerStageSubmitted}
import org.apache.spark.sql.{Dataset, Row}
@@ -50,10 +55,12 @@ import org.assertj.core.api.Assertions
import org.assertj.core.api.Assertions.assertThatThrownBy
import org.scalatest.time.Span
+import java.io.IOException
import java.lang.reflect.{InvocationHandler, Method, Proxy}
import java.time.LocalDate
import java.time.format.DateTimeFormatter
import java.util
+import java.util.concurrent.{CountDownLatch, TimeUnit}
import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger, AtomicLong, AtomicReference}
import scala.collection.JavaConverters._
@@ -3001,7 +3008,9 @@ abstract class CompactProcedureTestBase extends PaimonSparkTestBase with StreamT
Assertions.assertThat(deletionVectorIndexFileCount(table)).isEqualTo(1)
val snapshotBefore = lastSnapshotId(table)
- executeDataEvolutionCompaction(table, candidateFilesPerBatch = 2)
+ executeDataEvolutionCompaction(
+ table.copy(Map("deletion-vector.index-file.target-size" -> "1 b").asJava),
+ candidateFilesPerBatch = 2)
Assertions.assertThat(lastSnapshotId(table)).isEqualTo(snapshotBefore + 2)
Assertions
@@ -3015,6 +3024,229 @@ abstract class CompactProcedureTestBase extends PaimonSparkTestBase with StreamT
}
}
+ test("Paimon Procedure: clean outputs of failed deletion vector rewrite partitions") {
+ withTable("T") {
+ sql("""
+ |CREATE TABLE T (id INT, value STRING)
+ |TBLPROPERTIES (
+ | 'bucket' = '-1', 'row-tracking.enabled' = 'true',
+ | 'data-evolution.enabled' = 'true', 'deletion-vectors.enabled' = 'true')
+ |""".stripMargin)
+ sql("INSERT INTO T VALUES (0, 'zero')")
+ sql("INSERT INTO T VALUES (1, 'one')")
+ sql("DELETE FROM T WHERE id = 0")
+ val table = loadTable("T").copy(Map("deletion-vector.index-file.target-size" -> "1 b").asJava)
+ val snapshot = table.latestSnapshot().get()
+ val compact = new DataEvolutionNormalCompactTask(
+ BinaryRow.EMPTY_ROW,
+ normalDataFiles(table).asJava,
+ util.Collections.emptyList[Range](),
+ false).doCompact(table, "failed-dv-rewrite")
+ val tasks = new DataEvolutionDeletionVectorRewritePlanner(table, snapshot)
+ .plan(util.Collections.singletonList(compact))
+ val serializer = new DataEvolutionDeletionVectorRewriteTask.Serializer()
+ val taskBytes = serializer.serialize(tasks.get(0))
+ val handler = table.store().newIndexFileHandler()
+ val oldIndex = handler.scan(snapshot, DELETION_VECTORS_INDEX, BinaryRow.EMPTY_ROW, 0).get(0)
+ val directory = handler.dvIndex(BinaryRow.EMPTY_ROW, 0).path(oldIndex).getParent
+ def indexFiles(): Set[Path] = table.fileIO().listStatus(directory).map(_.getPath).toSet
+ val filesBefore = indexFiles()
+
+ // The first rewrite writes a file; deserializing the following task then fails.
+ assertThatThrownBy(
+ () =>
+ DataEvolutionRewriteExecutor.rewriteDeletionVectorPartition(
+ table,
+ util.Arrays.asList(taskBytes, Array.emptyByteArray).iterator(),
+ null))
+ .isInstanceOf(classOf[java.io.IOException])
+ Assertions.assertThat(indexFiles()).isEqualTo(filesBefore)
+
+ Seq("success", "failure", "cancel").foreach {
+ outcome =>
+ val module = Class.forName("org.apache.spark.TaskContext$").getField("MODULE$").get(null)
+ val context = module.getClass.getMethod("empty").invoke(module).asInstanceOf[TaskContext]
+ val messages = DataEvolutionRewriteExecutor.rewriteDeletionVectorPartition(
+ table,
+ util.Collections.singletonList(taskBytes).iterator(),
+ context)
+ assert(!messages.isEmpty)
+ val outputs = indexFiles() -- filesBefore
+ assert(outputs.nonEmpty)
+ if (outcome == "failure") {
+ context.getClass
+ .getMethod("markTaskFailed", classOf[Throwable])
+ .invoke(context, new IllegalStateException("Injected failure"))
+ } else if (outcome == "cancel") {
+ context.getClass.getMethod("markInterrupted", classOf[String]).invoke(context, "cancel")
+ }
+ context.getClass.getMethod("markTaskCompleted", classOf[Option[_]]).invoke(context, None)
+ if (outcome == "success") {
+ Assertions.assertThat(indexFiles()).isEqualTo(filesBefore ++ outputs)
+ outputs.foreach(table.fileIO().deleteQuietly)
+ } else {
+ Assertions.assertThat(indexFiles()).isEqualTo(filesBefore)
+ }
+ }
+ Assertions.assertThat(lastSnapshotId(table)).isEqualTo(snapshot.id())
+ Assertions.assertThat(deletionVectorCardinality(table)).isEqualTo(1)
+ checkAnswer(sql("SELECT id, value FROM T"), Seq(Row(1, "one")))
+ }
+ }
+
+ test("Paimon Procedure: retry distributed deletion vector preparation") {
+ withTable("T") {
+ sql("""
+ |CREATE TABLE T (id INT, value STRING)
+ |TBLPROPERTIES (
+ | 'bucket' = '-1', 'row-tracking.enabled' = 'true',
+ | 'data-evolution.enabled' = 'true', 'deletion-vectors.enabled' = 'true',
+ | 'commit.min-retry-wait' = '1 ms', 'commit.max-retry-wait' = '1 ms')
+ |""".stripMargin)
+ sql("INSERT INTO T VALUES (0, 'zero')")
+ sql("INSERT INTO T VALUES (1, 'one')")
+ sql("DELETE FROM T WHERE id = 0")
+ val table = loadTable("T").copy(Map("deletion-vector.index-file.target-size" -> "1 b").asJava)
+ val snapshot = table.latestSnapshot().get()
+ val task = new DataEvolutionNormalCompactTask(
+ BinaryRow.EMPTY_ROW,
+ normalDataFiles(table).asJava,
+ util.Collections.emptyList[Range](),
+ false)
+ val attempts = new AtomicInteger()
+ val abortedIndexes = new util.ArrayList[IndexFileMeta]()
+ val configurer = new DataEvolutionRewriteExecutor.CommitConfigurer {
+ override def configure(commit: TableCommitImpl): Unit = {}
+
+ override def prepareMessages(base: Snapshot, messages: util.List[CommitMessage]): Unit = {
+ if (attempts.getAndIncrement() == 0) {
+ messages.asScala.foreach(
+ message =>
+ abortedIndexes.addAll(
+ message.asInstanceOf[CommitMessageImpl].compactIncrement().newIndexFiles()))
+ throw new DataEvolutionRowRangeConflictException("Injected DV preparation conflict.")
+ }
+ }
+ }
+ DataEvolutionRewriteExecutor.execute(
+ table,
+ snapshot,
+ _ =>
+ new DataEvolutionRewriteExecutor.PlanResult(util.Collections.singletonList(task), true),
+ new JavaSparkContext(spark.sparkContext),
+ spark,
+ configurer,
+ (_, _, _, messages) => util.Optional.of(messages)
+ )
+
+ Assertions.assertThat(attempts.get()).isEqualTo(2)
+ assert(!abortedIndexes.isEmpty)
+ val index = table.store().newIndexFileHandler().dvIndex(BinaryRow.EMPTY_ROW, -1)
+ abortedIndexes.asScala.foreach(file => assert(!index.exists(file)))
+ Assertions.assertThat(deletionVectorCardinality(table)).isEqualTo(1)
+ checkAnswer(sql("SELECT id, value FROM T"), Seq(Row(1, "one")))
+ }
+ }
+
+ test("Paimon Procedure: recover outputs after a distributed DV rewrite job fails") {
+ withTable("T") {
+ sql("""
+ |CREATE TABLE T (id INT, value STRING)
+ |TBLPROPERTIES (
+ | 'bucket' = '-1', 'row-tracking.enabled' = 'true',
+ | 'data-evolution.enabled' = 'true', 'deletion-vectors.enabled' = 'true')
+ |""".stripMargin)
+ (0 until 4).foreach(id => sql(s"INSERT INTO T VALUES ($id, 'value-$id')"))
+ sql("DELETE FROM T WHERE id = 0")
+ sql("DELETE FROM T WHERE id = 2")
+ val table = loadTable("T").copy(Map("deletion-vector.index-file.target-size" -> "1 b").asJava)
+ val snapshot = table.latestSnapshot().get()
+ val files = normalDataFiles(table).sortBy(_.nonNullFirstRowId())
+ val compact: Seq[DataEvolutionCompactTask] = Seq(0, 2).map(
+ first =>
+ new DataEvolutionNormalCompactTask(
+ BinaryRow.EMPTY_ROW,
+ files.slice(first, first + 2).asJava,
+ util.Collections.emptyList[Range](),
+ false))
+ val handler = table.store().newIndexFileHandler()
+ val index = handler.dvIndex(BinaryRow.EMPTY_ROW, UNAWARE_BUCKET)
+ val oldIndexes =
+ handler.scan(snapshot, DELETION_VECTORS_INDEX, BinaryRow.EMPTY_ROW, UNAWARE_BUCKET)
+ Assertions.assertThat(oldIndexes.size()).isEqualTo(2)
+ val unreadable = oldIndexes.asScala.find(_.dvRanges().containsKey(files(2).fileName())).get
+ val readable = oldIndexes.asScala.find(_.dvRanges().containsKey(files.head.fileName())).get
+ val directory = index.path(unreadable).getParent
+ def indexFiles(): Set[Path] = table.fileIO().listStatus(directory).map(_.getPath).toSet
+ val before = indexFiles()
+ // Advance the head so the rewriter supplies a staged retry artifact before DV preparation.
+ sql("INSERT INTO T VALUES (4, 'four')")
+ val latest = table.latestSnapshot().get().id()
+ val retryFile = new AtomicReference[Path]()
+ val failingTable = new AppendOnlyFileStoreTable(
+ new DvRewriteFailureFileIO(index.path(unreadable).toString, index.path(readable).toString),
+ table.location(),
+ table.schema(),
+ table.catalogEnvironment())
+ DvRewriteFailure.firstTaskCompleted = new CountDownLatch(1)
+ try {
+ withSQLConf("spark.sql.shuffle.partitions" -> "2") {
+ assertThatThrownBy(
+ () =>
+ DataEvolutionRewriteExecutor.execute(
+ failingTable,
+ snapshot,
+ _ => new DataEvolutionRewriteExecutor.PlanResult(compact.asJava, true),
+ new JavaSparkContext(spark.sparkContext),
+ spark,
+ new DataEvolutionRewriteExecutor.CommitConfigurer {
+ override def configure(commit: TableCommitImpl): Unit = {}
+ },
+ (_, _, _, messages) => {
+ val rewritten = new util.ArrayList[CommitMessage](messages)
+ val write = table.newBatchWriteBuilder().newWrite()
+ try {
+ write.write(
+ GenericRow.of(Int.box(99), BinaryString.fromString("retry-artifact")))
+ val artifacts = write.prepareCommit()
+ val artifact = artifacts.get(0).asInstanceOf[CommitMessageImpl]
+ retryFile.set(
+ table
+ .store()
+ .pathFactory()
+ .createDataFilePathFactory(artifact.partition(), artifact.bucket())
+ .toPath(artifact.newFilesIncrement().newFiles().get(0)))
+ rewritten.add(
+ new CommitMessageImpl(
+ artifact.partition(),
+ UNAWARE_BUCKET,
+ null,
+ artifact.newFilesIncrement(),
+ CompactIncrement.emptyIncrement()))
+ } finally {
+ write.close()
+ }
+ assert(table.fileIO().exists(retryFile.get()))
+ util.Optional.of(rewritten)
+ }
+ ))
+ .hasStackTraceContaining(classOf[SparkException].getName)
+ .hasStackTraceContaining("Injected DV read failure")
+ }
+ } finally {
+ DvRewriteFailure.firstTaskCompleted.countDown()
+ }
+ assert(retryFile.get() != null && !table.fileIO().exists(retryFile.get()))
+ Assertions.assertThat(lastSnapshotId(table)).isEqualTo(latest)
+ val orphanIndexes = indexFiles() -- before
+ assert(orphanIndexes.nonEmpty)
+ // The other partition already succeeded. Exercise the documented orphan-file recovery path.
+ new LocalOrphanFilesClean(table, System.currentTimeMillis() + 60000).clean()
+ Assertions.assertThat(indexFiles()).isEqualTo(before)
+ checkAnswer(sql("SELECT id FROM T"), Seq(Row(1), Row(3), Row(4)))
+ }
+ }
+
private def createCompactMergeRaceTable(): FileStoreTable = {
sql("""
|CREATE TABLE T (id INT, value INT, picture BINARY)
@@ -3156,3 +3388,27 @@ abstract class CompactProcedureTestBase extends PaimonSparkTestBase with StreamT
.sum
}
}
+
+private object DvRewriteFailure {
+ @volatile var firstTaskCompleted = new CountDownLatch(1)
+}
+
+private class DvRewriteFailureFileIO(unreadable: String, readable: String) extends LocalFileIO {
+ override def newInputStream(path: Path): SeekableInputStream = {
+ if (path.toString == readable && TaskContext.get() != null) {
+ TaskContext.get().addTaskCompletionListener[Unit] {
+ completed =>
+ if (!completed.isInterrupted()) {
+ DvRewriteFailure.firstTaskCompleted.countDown()
+ }
+ }
+ }
+ if (path.toString == unreadable) {
+ if (!DvRewriteFailure.firstTaskCompleted.await(30, TimeUnit.SECONDS)) {
+ throw new IOException("Timed out waiting for the successful DV rewrite task")
+ }
+ throw new IOException("Injected DV read failure")
+ }
+ super.newInputStream(path)
+ }
+}