diff --git a/docs/docs/multimodal-table/data-evolution-maintenance.mdx b/docs/docs/multimodal-table/data-evolution-maintenance.mdx index 6e47487ccd50..ab0f2d258afb 100644 --- a/docs/docs/multimodal-table/data-evolution-maintenance.mdx +++ b/docs/docs/multimodal-table/data-evolution-maintenance.mdx @@ -89,6 +89,20 @@ Ordinary compaction **does not physically remove rows hidden by deletion vectors**. Removing their positions would change the alignment with untouched column files. +### Recovering Files After a Failed Rewrite + +Flink and Spark rewrite DV indexes on workers. A failed worker attempt cleans +up the new DV files it has tracked. Files from tasks that already succeeded, +or from a lost worker process, can remain unreferenced if the overall rewrite +fails before committing. + +After the failed job and its retries have stopped, use +[Remove Orphan Files](../maintenance/manage-snapshots#remove-orphan-files) +to reclaim these files. The default retention window is one day. If setting +`older_than`, choose a cutoff that keeps files from any active writes or +maintenance jobs. Snapshot expiration alone does not reclaim files that were +never committed. + ### Incremental Column-Version Compaction To reduce repeated rewrites of large base files when only a few columns diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactDeletionVectorRewriter.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactDeletionVectorRewriter.java index 474eeaaa3864..b60a7c3d8b21 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactDeletionVectorRewriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactDeletionVectorRewriter.java @@ -19,57 +19,14 @@ package org.apache.paimon.append.dataevolution; import org.apache.paimon.Snapshot; -import org.apache.paimon.data.BinaryRow; -import org.apache.paimon.deletionvectors.Bitmap64DeletionVector; -import org.apache.paimon.deletionvectors.BitmapDeletionVector; -import org.apache.paimon.deletionvectors.DeletionVector; -import org.apache.paimon.deletionvectors.append.AppendDeleteFileMaintainer; -import org.apache.paimon.deletionvectors.append.BaseAppendDeleteFileMaintainer; -import org.apache.paimon.index.IndexFileHandler; -import org.apache.paimon.index.IndexFileMeta; -import org.apache.paimon.io.CompactIncrement; -import org.apache.paimon.io.DataFileMeta; -import org.apache.paimon.io.DataIncrement; -import org.apache.paimon.manifest.FileKind; -import org.apache.paimon.manifest.IndexManifestEntry; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.sink.CommitMessage; -import org.apache.paimon.table.sink.CommitMessageImpl; -import org.apache.paimon.utils.Pair; -import org.apache.paimon.utils.Range; -import org.apache.paimon.utils.RangeHelper; +import org.apache.paimon.table.sink.TableCommit; import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.stream.Collectors; -import static org.apache.paimon.deletionvectors.DeletionVectorsIndexFile.DELETION_VECTORS_INDEX; -import static org.apache.paimon.format.blob.BlobFileFormat.isBlobFile; -import static org.apache.paimon.table.BucketMode.UNAWARE_BUCKET; -import static org.apache.paimon.types.VectorType.isVectorStoreFile; -import static org.apache.paimon.utils.DataEvolutionUtils.checkContiguousRowRange; -import static org.apache.paimon.utils.DataEvolutionUtils.retrieveAnchorFile; -import static org.apache.paimon.utils.Preconditions.checkState; - -/** - * Rewrites deletion vectors for non-materialized data-evolution compaction. - * - *

Compaction tasks rewrite data files without applying deletion vectors. This class consumes all - * compaction messages before commit, moves old anchor-file DVs to new compacted files, and returns - * index-only compact messages for the rewritten DV index files. - * - *

Unlike append compaction, data-evolution compaction does not pack compact tasks by deletion - * vector index files. Data-evolution compaction is batch-oriented, and packing by DV may create a - * huge compact task when many files contain only a few deleted rows. This is especially risky when - * blob compaction is involved. Instead, compact tasks are planned normally and DVs are rewritten - * centrally from compact commit messages. This is acceptable because most DVs only need to be - * renamed to the new anchor file and are not fully rewritten. - */ +/** Local execution of the same DV rewrite tasks used by distributed compaction. */ public class DataEvolutionCompactDeletionVectorRewriter { private final FileStoreTable table; @@ -81,231 +38,22 @@ public DataEvolutionCompactDeletionVectorRewriter(FileStoreTable table, Snapshot } public List rewriteDeletionVectors(List messages) { - if (!table.coreOptions().deletionVectorsEnabled() || messages.isEmpty()) { - return Collections.emptyList(); - } - - Map maintainerByParts = - collectMaintainers(snapshot, messages); - if (maintainerByParts.isEmpty()) { - return Collections.emptyList(); - } - List result = new ArrayList<>(); - for (Map.Entry entry : - maintainerByParts.entrySet()) { - List indexChanges = entry.getValue().persist(); - if (!indexChanges.isEmpty()) { - result.add(toCommitMessage(entry.getKey(), indexChanges)); + try { + for (DataEvolutionDeletionVectorRewriteTask task : + new DataEvolutionDeletionVectorRewritePlanner(table, snapshot).plan(messages)) { + result.add(task.execute(table)); } - } - return result; - } - - private Map collectMaintainers( - Snapshot snapshot, List messages) { - Map result = createMaintainers(snapshot, messages); - RangeHelper rangeHelper = new RangeHelper<>(DataFileMeta::nonNullRowIdRange); - - for (CommitMessage message : messages) { - CommitMessageImpl commitMessage = (CommitMessageImpl) message; - CompactIncrement compactIncrement = commitMessage.compactIncrement(); - List before = normalFiles(compactIncrement.compactBefore()); - List after = normalFiles(compactIncrement.compactAfter()); - - // Skip blob-only tasks and index-only commit messages. - if (before.isEmpty() && after.isEmpty()) { - continue; + } catch (RuntimeException failure) { + if (!result.isEmpty()) { + try (TableCommit commit = table.newCommit("data-evolution-dv-rewrite-abort")) { + commit.abort(result); + } catch (Exception cleanupFailure) { + failure.addSuppressed(cleanupFailure); + } } - - AppendDeleteFileMaintainer maintainer = result.get(commitMessage.partition()); - if (isMaterialized(compactIncrement)) { - removeMaterializedDeletionVectors(maintainer, rangeHelper, before); - continue; - } - - Range beforeRange = checkContiguousRowRange(before); - Range afterRange = checkContiguousRowRange(after); - checkState( - beforeRange.equals(afterRange), - "Non-materialized data evolution compaction should keep the same row-id " - + "range, but compact before range is %s and compact after range is %s.", - beforeRange, - afterRange); - TreeMap outputFiles = new TreeMap<>(); - for (DataFileMeta file : after) { - outputFiles.put(file.nonNullFirstRowId(), file); - } - Map rewritten = new LinkedHashMap<>(); - - // Redistribute old deletion positions across the output file boundaries. - for (List previousRowGroup : rangeHelper.mergeOverlappingRanges(before)) { - DataFileMeta oldAnchor = retrieveAnchorFile(previousRowGroup, file -> file); - moveDeletionVector( - maintainer, - outputFiles, - rewritten, - oldAnchor.fileName(), - oldAnchor.nonNullRowIdRange()); - } - rewritten.forEach(maintainer::notifyNewDeletionVector); + throw failure; } return result; } - - private Map createMaintainers( - Snapshot snapshot, List messages) { - Map> entriesByPartition = new LinkedHashMap<>(); - for (CommitMessage message : messages) { - CommitMessageImpl commitMessage = (CommitMessageImpl) message; - CompactIncrement increment = commitMessage.compactIncrement(); - checkNoCompactIndexChanges(increment); - checkState( - commitMessage.bucket() == UNAWARE_BUCKET - && commitMessage.totalBuckets() == null, - "Data evolution compaction should only produce unaware-bucket commit " - + "messages, but got bucket %s and total buckets %s.", - commitMessage.bucket(), - commitMessage.totalBuckets()); - List before = normalFiles(increment.compactBefore()); - List after = normalFiles(increment.compactAfter()); - if (before.isEmpty() && after.isEmpty()) { - continue; - } - entriesByPartition.computeIfAbsent( - commitMessage.partition(), ignored -> new ArrayList<>()); - } - if (entriesByPartition.isEmpty()) { - return Collections.emptyMap(); - } - - IndexFileHandler handler = table.store().newIndexFileHandler(); - Set> buckets = - entriesByPartition.keySet().stream() - .map(partition -> Pair.of(partition, UNAWARE_BUCKET)) - .collect(Collectors.toSet()); - // Filter before decoding DV ranges and scan only once for all affected partitions. - for (IndexManifestEntry entry : - handler.scanBucketEntries(snapshot, DELETION_VECTORS_INDEX, buckets)) { - entriesByPartition.get(entry.partition()).add(entry); - } - - Map result = new LinkedHashMap<>(); - entriesByPartition.forEach( - (partition, entries) -> - result.put( - partition, - BaseAppendDeleteFileMaintainer.forUnawareAppend( - handler, partition, entries))); - return result; - } - - private void removeMaterializedDeletionVectors( - AppendDeleteFileMaintainer maintainer, - RangeHelper rangeHelper, - List before) { - for (List previousRowGroup : rangeHelper.mergeOverlappingRanges(before)) { - DataFileMeta oldAnchor = retrieveAnchorFile(previousRowGroup, file -> file); - maintainer.notifyRemovedDeletionVector(oldAnchor.fileName()); - } - } - - private boolean isMaterialized(CompactIncrement compactIncrement) { - List before = normalFiles(compactIncrement.compactBefore()); - if (before.isEmpty()) { - return false; - } - return compactIncrement.compactAfter().stream().allMatch(file -> file.firstRowId() == null); - } - - /** - * Moves old anchor-file deletion vectors to the new files, merging or splitting them at the - * output row-id boundaries. - */ - private void moveDeletionVector( - AppendDeleteFileMaintainer maintainer, - TreeMap outputFiles, - Map rewritten, - String oldFileName, - Range oldRange) { - DeletionVector old = maintainer.getDeletionVector(oldFileName); - if (old == null || old.isEmpty()) { - return; - } - - // Fast path for renaming deletion vectors only. - DataFileMeta firstOutput = outputFiles.floorEntry(oldRange.from).getValue(); - if (oldRange.equals(firstOutput.nonNullRowIdRange())) { - rewritten - .computeIfAbsent(firstOutput.fileName(), ignored -> newDeletionVector()) - .merge(old); - } else { - old.forEachDeletedPosition( - position -> { - long absolutePosition = oldRange.from + position; - DataFileMeta output = outputFiles.floorEntry(absolutePosition).getValue(); - Range newRange = output.nonNullRowIdRange(); - long newPosition = absolutePosition - newRange.from; - checkState( - newPosition >= 0 && newPosition < newRange.count(), - "Cannot move deletion position %s from old range %s to new range %s.", - absolutePosition, - oldRange, - newRange); - rewritten - .computeIfAbsent(output.fileName(), ignored -> newDeletionVector()) - .delete(newPosition); - }); - } - maintainer.notifyRemovedDeletionVector(oldFileName); - } - - private List normalFiles(List files) { - return files.stream().filter(this::isNormalFile).collect(Collectors.toList()); - } - - private boolean isNormalFile(DataFileMeta fileMeta) { - return !isBlobFile(fileMeta.fileName()) && !isVectorStoreFile(fileMeta.fileName()); - } - - private DeletionVector newDeletionVector() { - return table.coreOptions().deletionVectorBitmap64() - ? new Bitmap64DeletionVector() - : new BitmapDeletionVector(); - } - - private CommitMessage toCommitMessage( - BinaryRow partition, List indexChanges) { - List newIndexFiles = new ArrayList<>(); - List deletedIndexFiles = new ArrayList<>(); - for (IndexManifestEntry entry : indexChanges) { - if (entry.kind() == FileKind.ADD) { - newIndexFiles.add(entry.indexFile()); - } else { - deletedIndexFiles.add(entry.indexFile()); - } - } - - return new CommitMessageImpl( - partition, - UNAWARE_BUCKET, - null, - DataIncrement.emptyIncrement(), - new CompactIncrement( - Collections.emptyList(), - Collections.emptyList(), - Collections.emptyList(), - newIndexFiles, - deletedIndexFiles)); - } - - private void checkNoCompactIndexChanges(CompactIncrement compactIncrement) { - checkState( - compactIncrement.newIndexFiles().isEmpty() - && compactIncrement.deletedIndexFiles().isEmpty(), - "Data evolution compaction should not produce index changes before deletion vector rewrite, " - + "but got new index files %s and deleted index files %s.", - compactIncrement.newIndexFiles(), - compactIncrement.deletedIndexFiles()); - } } diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactionCommitPreparation.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactionCommitPreparation.java index 238f98e2b2e9..75ab8067f52d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactionCommitPreparation.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactionCommitPreparation.java @@ -48,4 +48,9 @@ public List prepare(List messages) { .rewriteDeletionVectors(messages)); return result; } + + public List planDeletionVectorRewrites( + List messages) { + return new DataEvolutionDeletionVectorRewritePlanner(table, snapshot).plan(messages); + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionDeletionVectorRewritePlanner.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionDeletionVectorRewritePlanner.java new file mode 100644 index 000000000000..1bdc46d993d6 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionDeletionVectorRewritePlanner.java @@ -0,0 +1,251 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.append.dataevolution; + +import org.apache.paimon.Snapshot; +import org.apache.paimon.append.dataevolution.DataEvolutionDeletionVectorRewriteTask.IndexRewrite; +import org.apache.paimon.append.dataevolution.DataEvolutionDeletionVectorRewriteTask.Rewrite; +import org.apache.paimon.append.dataevolution.DataEvolutionDeletionVectorRewriteTask.Source; +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.index.DeletionVectorMeta; +import org.apache.paimon.index.IndexFileHandler; +import org.apache.paimon.io.CompactIncrement; +import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.manifest.IndexManifestEntry; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.sink.CommitMessage; +import org.apache.paimon.table.sink.CommitMessageImpl; +import org.apache.paimon.table.source.DeletionFile; +import org.apache.paimon.utils.Pair; +import org.apache.paimon.utils.Range; +import org.apache.paimon.utils.RangeHelper; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.apache.paimon.deletionvectors.DeletionVectorsIndexFile.DELETION_VECTORS_INDEX; +import static org.apache.paimon.format.blob.BlobFileFormat.isBlobFile; +import static org.apache.paimon.table.BucketMode.UNAWARE_BUCKET; +import static org.apache.paimon.types.VectorType.isVectorStoreFile; +import static org.apache.paimon.utils.DataEvolutionUtils.checkContiguousRowRange; +import static org.apache.paimon.utils.DataEvolutionUtils.retrieveAnchorFile; +import static org.apache.paimon.utils.Preconditions.checkState; + +/** + * Plans bitmap work without reading bitmap contents or changing data-compaction boundaries. + * + *

All compact messages are collected first, so one old index file has a single owner even when + * its anchors are replaced by different data tasks. + */ +public class DataEvolutionDeletionVectorRewritePlanner { + + private static final int MAX_DELETE_ONLY_FILES_PER_TASK = 128; + + private final FileStoreTable table; + private final Snapshot snapshot; + + public DataEvolutionDeletionVectorRewritePlanner(FileStoreTable table, Snapshot snapshot) { + this.table = table; + this.snapshot = snapshot; + } + + public List plan(List messages) { + if (!table.coreOptions().deletionVectorsEnabled() || messages.isEmpty()) { + return Collections.emptyList(); + } + Map partitions = new LinkedHashMap<>(); + RangeHelper ranges = new RangeHelper<>(DataFileMeta::nonNullRowIdRange); + for (CommitMessage message : messages) { + CommitMessageImpl commit = (CommitMessageImpl) message; + CompactIncrement increment = commit.compactIncrement(); + checkState( + commit.bucket() == UNAWARE_BUCKET && commit.totalBuckets() == null, + "DV rewrite requires unaware-bucket compact messages."); + checkState( + increment.newIndexFiles().isEmpty() && increment.deletedIndexFiles().isEmpty(), + "DV rewrite input must not contain prepared index changes."); + List before = normalFiles(increment.compactBefore()); + List after = normalFiles(increment.compactAfter()); + if (before.isEmpty() && after.isEmpty()) { + continue; + } + PartitionPlan partition = + partitions.computeIfAbsent(commit.partition(), ignored -> new PartitionPlan()); + Map anchors = new LinkedHashMap<>(); + for (List group : ranges.mergeOverlappingRanges(before)) { + DataFileMeta anchor = retrieveAnchorFile(group, file -> file); + checkState( + partition.removed.add(anchor.fileName()), + "Anchor %s belongs to multiple DV rewrite messages.", + anchor.fileName()); + anchors.put(anchor.fileName(), anchor.nonNullRowIdRange()); + } + boolean materialized = + !before.isEmpty() + && increment.compactAfter().stream() + .allMatch(file -> file.firstRowId() == null); + if (!materialized) { + checkState( + checkContiguousRowRange(before).equals(checkContiguousRowRange(after)), + "DV rewrite must preserve the input row-ID range."); + Map outputs = new LinkedHashMap<>(); + after.forEach(file -> outputs.put(file.fileName(), file.nonNullRowIdRange())); + partition.specifications.add(new Specification(anchors, outputs)); + } + } + if (partitions.isEmpty()) { + return Collections.emptyList(); + } + IndexFileHandler handler = table.store().newIndexFileHandler(); + Set> buckets = + partitions.keySet().stream() + .map(partition -> Pair.of(partition, UNAWARE_BUCKET)) + .collect(Collectors.toSet()); + for (IndexManifestEntry entry : + handler.scanBucketEntries(snapshot, DELETION_VECTORS_INDEX, buckets)) { + PartitionPlan partition = partitions.get(entry.partition()); + Set removed = new LinkedHashSet<>(); + String path = null; + for (DeletionVectorMeta dv : entry.indexFile().dvRanges().values()) { + if (partition.removed.contains(dv.dataFileName())) { + if (path == null) { + path = handler.filePath(entry).toString(); + } + checkState( + partition.deletions.put( + dv.dataFileName(), + new DeletionFile( + path, + dv.offset(), + dv.length(), + dv.cardinality())) + == null, + "Duplicate DV for anchor %s.", + dv.dataFileName()); + removed.add(dv.dataFileName()); + } + } + if (!removed.isEmpty()) { + partition.indexes.add(new IndexRewrite(entry.indexFile(), removed)); + } + } + + List result = new ArrayList<>(); + long target = table.coreOptions().dvIndexFileTargetSize().getBytes(); + partitions.forEach( + (partition, plan) -> { + Batch batch = new Batch(partition, target, result); + for (Specification spec : plan.specifications) { + List sources = new ArrayList<>(); + long weight = 0; + for (Map.Entry anchor : spec.anchors.entrySet()) { + DeletionFile deletion = plan.deletions.get(anchor.getKey()); + if (deletion != null) { + sources.add( + new Source(anchor.getKey(), anchor.getValue(), deletion)); + weight += deletion.length(); + } + } + if (!sources.isEmpty()) { + // Keep all contributors to an output anchor together, even across old + // indexes. + batch.ensureCapacity(weight); + batch.task.rewrites.add(new Rewrite(sources, spec.outputs)); + } + } + for (IndexRewrite index : plan.indexes) { + long weight = 0; + for (DeletionVectorMeta dv : index.file.dvRanges().values()) { + if (!index.removed.contains(dv.dataFileName())) { + weight += dv.length(); + } + } + batch.ensureCapacity(weight); + batch.task.indexes.add(index); + } + batch.finish(); + }); + return result; + } + + private static List normalFiles(List files) { + return files.stream() + .filter(file -> !isBlobFile(file.fileName()) && !isVectorStoreFile(file.fileName())) + .collect(Collectors.toList()); + } + + private static class PartitionPlan { + private final Set removed = new LinkedHashSet<>(); + private final Map deletions = new LinkedHashMap<>(); + private final List specifications = new ArrayList<>(); + private final List indexes = new ArrayList<>(); + } + + private static class Specification { + private final Map anchors; + private final Map outputs; + + private Specification(Map anchors, Map outputs) { + this.anchors = anchors; + this.outputs = outputs; + } + } + + private static class Batch { + private final BinaryRow partition; + private final long target; + private final List result; + private DataEvolutionDeletionVectorRewriteTask task; + private long weight; + + private Batch( + BinaryRow partition, + long target, + List result) { + this.partition = partition; + this.target = target; + this.result = result; + this.task = new DataEvolutionDeletionVectorRewriteTask(partition); + } + + private void ensureCapacity(long nextWeight) { + if (task.workCount() > 0 + && (weight + nextWeight > target + || (nextWeight == 0 + && task.indexes.size() >= MAX_DELETE_ONLY_FILES_PER_TASK))) { + finish(); + } + weight += nextWeight; + } + + private void finish() { + if (task.workCount() > 0) { + result.add(task); + task = new DataEvolutionDeletionVectorRewriteTask(partition); + weight = 0; + } + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionDeletionVectorRewriteTask.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionDeletionVectorRewriteTask.java new file mode 100644 index 000000000000..05b34647956a --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionDeletionVectorRewriteTask.java @@ -0,0 +1,323 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.append.dataevolution; + +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.serializer.VersionedSerializer; +import org.apache.paimon.deletionvectors.Bitmap64DeletionVector; +import org.apache.paimon.deletionvectors.BitmapDeletionVector; +import org.apache.paimon.deletionvectors.DeletionVector; +import org.apache.paimon.deletionvectors.DeletionVectorsIndexFile; +import org.apache.paimon.index.IndexFileMeta; +import org.apache.paimon.index.IndexFileMetaSerializer; +import org.apache.paimon.io.CompactIncrement; +import org.apache.paimon.io.DataIncrement; +import org.apache.paimon.io.DataInputDeserializer; +import org.apache.paimon.io.DataOutputSerializer; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.sink.CommitMessage; +import org.apache.paimon.table.sink.CommitMessageImpl; +import org.apache.paimon.table.source.DeletionFile; +import org.apache.paimon.utils.Range; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Set; +import java.util.TreeMap; + +import static org.apache.paimon.table.BucketMode.UNAWARE_BUCKET; +import static org.apache.paimon.utils.Preconditions.checkArgument; +import static org.apache.paimon.utils.Preconditions.checkState; +import static org.apache.paimon.utils.SerializationUtils.deserializeBinaryRow; +import static org.apache.paimon.utils.SerializationUtils.readCount; +import static org.apache.paimon.utils.SerializationUtils.serializeBinaryRow; + +/** + * A partition-local batch of complete anchor rewrites and uniquely owned old index files. + * + *

New anchor bitmaps share rolling output files. Retained entries are written separately for + * each old index, preserving the conflict boundaries of subsequent deletes. + */ +public class DataEvolutionDeletionVectorRewriteTask { + + final BinaryRow partition; + final List rewrites = new ArrayList<>(); + final List indexes = new ArrayList<>(); + + DataEvolutionDeletionVectorRewriteTask(BinaryRow partition) { + this.partition = partition; + } + + public BinaryRow partition() { + return partition; + } + + public int workCount() { + return rewrites.size() + indexes.size(); + } + + public CommitMessage execute(FileStoreTable table) { + DeletionVectorsIndexFile index = + table.store().newIndexFileHandler().dvIndex(partition, UNAWARE_BUCKET); + boolean bitmap64 = table.coreOptions().deletionVectorBitmap64(); + Iterator> bitmaps = + new Iterator>() { + private int rewritePosition; + private Iterator> current = + Collections.emptyIterator(); + + @Override + public boolean hasNext() { + while (!current.hasNext()) { + if (rewritePosition < rewrites.size()) { + current = + rewrite(rewrites.get(rewritePosition++), index, bitmap64) + .entrySet() + .iterator(); + } else { + return false; + } + } + return true; + } + + @Override + public Map.Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return current.next(); + } + }; + List added = new ArrayList<>(); + try { + added.addAll(index.writeWithRolling(bitmaps)); + for (IndexRewrite old : indexes) { + Map retained = + index.toDeletionFiles(Collections.singletonList(old.file)); + old.removed.forEach(retained::remove); + added.addAll(index.writeWithRolling(index.readDeletionVector(retained))); + } + } catch (RuntimeException failure) { + for (IndexFileMeta file : added) { + try { + table.fileIO().deleteQuietly(index.path(file)); + } catch (RuntimeException cleanupFailure) { + failure.addSuppressed(cleanupFailure); + } + } + throw failure; + } + List deleted = new ArrayList<>(indexes.size()); + indexes.forEach(old -> deleted.add(old.file)); + return new CommitMessageImpl( + partition, + UNAWARE_BUCKET, + null, + DataIncrement.emptyIncrement(), + new CompactIncrement( + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyList(), + added, + deleted)); + } + + private static Map rewrite( + Rewrite rewrite, DeletionVectorsIndexFile index, boolean bitmap64) { + Map> byIndex = new LinkedHashMap<>(); + Map inputRanges = new LinkedHashMap<>(); + for (Source source : rewrite.sources) { + byIndex.computeIfAbsent(source.deletion.path(), ignored -> new LinkedHashMap<>()) + .put(source.anchor, source.deletion); + inputRanges.put(source.anchor, source.range); + } + TreeMap> outputs = new TreeMap<>(); + rewrite.outputs.entrySet().forEach(output -> outputs.put(output.getValue().from, output)); + Map result = new LinkedHashMap<>(); + for (Map inputs : byIndex.values()) { + index.readDeletionVector(inputs) + .forEach( + (anchor, bitmap) -> + move( + bitmap, + inputRanges.get(anchor), + outputs, + result, + bitmap64)); + } + return result; + } + + static void move( + DeletionVector bitmap, + Range input, + TreeMap> outputs, + Map result, + boolean bitmap64) { + if (bitmap.isEmpty()) { + return; + } + Map.Entry first = outputs.floorEntry(input.from).getValue(); + if (input.equals(first.getValue())) { + // Disjoint input ranges make this output exclusive to the current anchor. + result.put(first.getKey(), bitmap); + return; + } + bitmap.forEachDeletedPosition( + position -> { + long rowId = input.from + position; + Map.Entry output = outputs.floorEntry(rowId).getValue(); + long offset = rowId - output.getValue().from; + checkState( + offset >= 0 && offset < output.getValue().count(), + "Deleted row ID %s is outside output range %s.", + rowId, + output.getValue()); + result.computeIfAbsent(output.getKey(), ignored -> newBitmap(bitmap64)) + .delete(offset); + }); + } + + private static DeletionVector newBitmap(boolean bitmap64) { + return bitmap64 ? new Bitmap64DeletionVector() : new BitmapDeletionVector(); + } + + static class Source { + final String anchor; + final Range range; + final DeletionFile deletion; + + Source(String anchor, Range range, DeletionFile deletion) { + this.anchor = anchor; + this.range = range; + this.deletion = deletion; + } + } + + static class Rewrite { + final List sources; + final Map outputs; + + Rewrite(List sources, Map outputs) { + this.sources = sources; + this.outputs = outputs; + } + } + + static class IndexRewrite { + final IndexFileMeta file; + final Set removed; + + IndexRewrite(IndexFileMeta file, Set removed) { + this.file = file; + this.removed = removed; + } + } + + /** Runtime-only task format shared by Flink and Spark. */ + public static class Serializer + implements VersionedSerializer { + + private final IndexFileMetaSerializer indexes = new IndexFileMetaSerializer(); + + @Override + public int getVersion() { + return 1; + } + + @Override + public byte[] serialize(DataEvolutionDeletionVectorRewriteTask task) throws IOException { + DataOutputSerializer out = new DataOutputSerializer(1024); + serializeBinaryRow(task.partition, out); + out.writeInt(task.rewrites.size()); + for (Rewrite rewrite : task.rewrites) { + out.writeInt(rewrite.sources.size()); + for (Source source : rewrite.sources) { + out.writeUTF(source.anchor); + out.writeLong(source.range.from); + out.writeLong(source.range.to); + DeletionFile.serialize(out, source.deletion); + } + out.writeInt(rewrite.outputs.size()); + for (Map.Entry output : rewrite.outputs.entrySet()) { + out.writeUTF(output.getKey()); + out.writeLong(output.getValue().from); + out.writeLong(output.getValue().to); + } + } + out.writeInt(task.indexes.size()); + for (IndexRewrite index : task.indexes) { + indexes.serializeList(Collections.singletonList(index.file), out); + out.writeInt(index.removed.size()); + for (String anchor : index.removed) { + out.writeUTF(anchor); + } + } + return out.getCopyOfBuffer(); + } + + @Override + public DataEvolutionDeletionVectorRewriteTask deserialize(int version, byte[] bytes) + throws IOException { + checkArgument( + version == getVersion(), "Unsupported DV rewrite task version %s.", version); + DataInputDeserializer in = new DataInputDeserializer(bytes); + DataEvolutionDeletionVectorRewriteTask task = + new DataEvolutionDeletionVectorRewriteTask(deserializeBinaryRow(in)); + int rewriteCount = readCount(in, "DV rewrites"); + for (int i = 0; i < rewriteCount; i++) { + int sourceCount = readCount(in, "DV sources"); + List sources = new ArrayList<>(); + for (int j = 0; j < sourceCount; j++) { + sources.add( + new Source( + in.readUTF(), + new Range(in.readLong(), in.readLong()), + DeletionFile.deserialize(in))); + } + int outputCount = readCount(in, "DV outputs"); + Map outputs = new LinkedHashMap<>(); + for (int j = 0; j < outputCount; j++) { + outputs.put(in.readUTF(), new Range(in.readLong(), in.readLong())); + } + task.rewrites.add(new Rewrite(sources, outputs)); + } + int indexCount = readCount(in, "DV index rewrites"); + for (int i = 0; i < indexCount; i++) { + List files = indexes.deserializeList(in); + checkArgument(files.size() == 1, "Expected one owned DV index file."); + int removedCount = readCount(in, "Removed DV anchors"); + Set removed = new HashSet<>(); + for (int j = 0; j < removedCount; j++) { + removed.add(in.readUTF()); + } + task.indexes.add(new IndexRewrite(files.get(0), removed)); + } + return task; + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorIndexFileWriter.java b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorIndexFileWriter.java index 6ce9e1c3875e..aebabc5f6545 100644 --- a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorIndexFileWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorIndexFileWriter.java @@ -25,7 +25,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -65,13 +64,22 @@ public IndexFileMeta writeSingleFile(Map input) throws I public List writeWithRolling(Map input) throws IOException { - if (input.isEmpty()) { - return Collections.emptyList(); - } + return writeWithRolling(input.entrySet().iterator()); + } + + /** Streams complete anchor bitmaps across work units into shared rolling files. */ + public List writeWithRolling( + Iterator> iterator) throws IOException { List result = new ArrayList<>(); - Iterator> iterator = input.entrySet().iterator(); - while (iterator.hasNext()) { - result.add(tryWriter(iterator)); + try { + while (iterator.hasNext()) { + result.add(tryWriter(iterator)); + } + } catch (IOException | RuntimeException e) { + for (IndexFileMeta file : result) { + fileIO.deleteQuietly(indexPathFactory.toPath(file)); + } + throw e; } return result; } @@ -87,8 +95,20 @@ private IndexFileMeta tryWriter(Iterator> iter break; } } - } finally { + } catch (IOException | RuntimeException e) { + try { + writer.close(); + } catch (IOException closeError) { + e.addSuppressed(closeError); + } + fileIO.deleteQuietly(indexPathFactory.toPath(writer.result())); + throw e; + } + try { writer.close(); + } catch (IOException e) { + fileIO.deleteQuietly(indexPathFactory.toPath(writer.result())); + throw e; } return writer.result(); } diff --git a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFile.java b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFile.java index 04b37e2deb12..b53334719f83 100644 --- a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFile.java @@ -33,6 +33,7 @@ import java.io.InputStream; import java.io.UncheckedIOException; import java.util.HashMap; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -177,6 +178,14 @@ public List writeWithRolling(Map input) { } } + public List writeWithRolling(Iterator> input) { + try { + return createWriter().writeWithRolling(input); + } catch (IOException e) { + throw new RuntimeException("Failed to write deletion vectors.", e); + } + } + private DeletionVectorIndexFileWriter createWriter() { return new DeletionVectorIndexFileWriter( this.fileIO, this.pathFactory, this.targetSizePerIndexFile); diff --git a/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFileTest.java b/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFileTest.java index c1a8e2a0fed2..0d3bbcd608b7 100644 --- a/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFileTest.java @@ -38,6 +38,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -54,6 +55,41 @@ public class DeletionVectorsIndexFileTest { @TempDir java.nio.file.Path tempPath; + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testStreamingRewriteFailureRemovesOutputs(boolean rolling) throws IOException { + DeletionVectorsIndexFile index = + deletionVectorsIndexFile( + getPathFactory(), MemorySize.ofBytes(rolling ? 1 : Long.MAX_VALUE), false); + DeletionVector bitmap = new BitmapDeletionVector(); + bitmap.delete(1); + Iterator> input = + new Iterator>() { + private int position; + + @Override + public boolean hasNext() { + if (position == 2) { + throw new IllegalStateException("Injected bitmap read failure"); + } + return true; + } + + @Override + public Map.Entry next() { + return Collections.singletonMap("file-" + position++, bitmap) + .entrySet() + .iterator() + .next(); + } + }; + assertThatThrownBy(() -> index.writeWithRolling(input)) + .hasMessage("Injected bitmap read failure"); + try (Stream files = Files.list(tempPath)) { + assertThat(files).isEmpty(); + } + } + @ParameterizedTest @ValueSource(booleans = {false, true}) public void testReadDvIndex(boolean bitmap64) { diff --git a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionDeletionVectorTest.java b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionDeletionVectorTest.java index e1de0d660b02..507f33137d92 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionDeletionVectorTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionDeletionVectorTest.java @@ -27,6 +27,8 @@ import org.apache.paimon.append.dataevolution.DataEvolutionCompactTaskSerializer; import org.apache.paimon.append.dataevolution.DataEvolutionCompactionCommitPreparation; import org.apache.paimon.append.dataevolution.DataEvolutionDeletionVectorMaterializeCoordinator; +import org.apache.paimon.append.dataevolution.DataEvolutionDeletionVectorRewritePlanner; +import org.apache.paimon.append.dataevolution.DataEvolutionDeletionVectorRewriteTask; import org.apache.paimon.append.dataevolution.DataEvolutionNormalCompactTask; import org.apache.paimon.append.dataevolution.DataEvolutionRangeSortPlan; import org.apache.paimon.append.dataevolution.DataEvolutionRowIdReassigner; @@ -39,6 +41,7 @@ import org.apache.paimon.data.serializer.InternalRowSerializer; import org.apache.paimon.deletionvectors.BitmapDeletionVector; import org.apache.paimon.deletionvectors.DeletionVector; +import org.apache.paimon.deletionvectors.DeletionVectorsIndexFile; import org.apache.paimon.deletionvectors.append.BaseAppendDeleteFileMaintainer; import org.apache.paimon.format.blob.BlobFileFormat; import org.apache.paimon.fs.FileIO; @@ -129,6 +132,178 @@ public class DataEvolutionDeletionVectorTest extends DataEvolutionTestBase { new DvSpec(new Range(5, 9), 6), new DvSpec(new Range(10, 14), 10, 12)); + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testDistributedDvRewriteWithSharedOldIndexes(boolean splitTasks) throws Exception { + FileStoreTable table = createPartitionedReassignTable("distributed_dv", false); + for (int first = 0; first < 18; first += 3) { + writePartitionRows(table, "a", first, first + 1, first + 2); + } + BinaryRow partition = partition(table, "a"); + // The first index is shared by both compact messages and an untouched anchor. + commitDeletionVectors( + table, + partition, + Arrays.asList( + new DvSpec(new Range(0, 2), 1), + new DvSpec(new Range(6, 8), 7), + new DvSpec(new Range(12, 14), 13))); + commitDeletionVectors( + table, + partition, + Arrays.asList( + new DvSpec(new Range(3, 5), 4), + new DvSpec(new Range(9, 11), 10), + new DvSpec(new Range(15, 17), 16))); + List rows = readPartitionedRows(table); + List oldIndexes = liveDeletionVectorIndexFileNames(table); + assertThat(oldIndexes).hasSize(2); + Snapshot snapshot = table.latestSnapshot().get(); + List files = currentDataFiles(table, partition); + List messages = new ArrayList<>(); + for (long first : new long[] {0, 6}) { + List inputs = + files.stream() + .filter( + file -> + file.nonNullFirstRowId() >= first + && file.nonNullFirstRowId() < first + 6) + .collect(Collectors.toList()); + messages.add( + new DataEvolutionNormalCompactTask( + partition, inputs, Collections.emptyList(), false) + .doCompact(table, "distributed-dv")); + } + FileStoreTable rewriteTable = + splitTasks + ? table.copy( + Collections.singletonMap( + CoreOptions.DELETION_VECTOR_INDEX_FILE_TARGET_SIZE.key(), + "1 b")) + : table; + List tasks = + new DataEvolutionDeletionVectorRewritePlanner(rewriteTable, snapshot) + .plan(messages); + if (splitTasks) { + assertThat(tasks.size()).isGreaterThan(1); + } else { + assertThat(tasks).hasSize(1); + } + // Reverse execution order after transport, so no task can depend on another's bitmap + // output. + Collections.reverse(tasks); + DataEvolutionDeletionVectorRewriteTask.Serializer serializer = + new DataEvolutionDeletionVectorRewriteTask.Serializer(); + List deleted = new ArrayList<>(); + List added = new ArrayList<>(); + for (DataEvolutionDeletionVectorRewriteTask task : tasks) { + CommitMessage rewritten = + serializer + .deserialize(serializer.getVersion(), serializer.serialize(task)) + .execute(rewriteTable); + CompactIncrement increment = ((CommitMessageImpl) rewritten).compactIncrement(); + increment.deletedIndexFiles().forEach(file -> deleted.add(file.fileName())); + added.addAll(increment.newIndexFiles()); + messages.add(rewritten); + } + assertThat(deleted).containsExactlyInAnyOrderElementsOf(oldIndexes); + if (!splitTasks) { + assertThat(added).hasSize(3); + } + commit(table, messages); + assertThat(readPartitionedRows(table)).containsExactlyInAnyOrderElementsOf(rows); + assertThat(liveDeletionVectorDataFileNames(table)).hasSize(4); + + // Prepare both deletes from the same snapshot. Retained anchors from different old indexes + // must remain independently updatable after compaction. + CommitMessage firstDelete = + prepareDeletionVectors( + table, + partition, + Collections.singletonList(new DvSpec(new Range(12, 14), 14))); + CommitMessage secondDelete = + prepareDeletionVectors( + table, + partition, + Collections.singletonList(new DvSpec(new Range(15, 17), 17))); + commit(table, Collections.singletonList(firstDelete)); + commit(table, Collections.singletonList(secondDelete)); + assertThat(readPartitionedRows(table)) + .containsExactlyInAnyOrderElementsOf( + rows.stream() + .filter(row -> !row.startsWith("a|14|") && !row.startsWith("a|17|")) + .collect(Collectors.toList())); + } + + @Test + public void testFailedDvRewriteRemovesPreviousTaskOutputs() throws Exception { + FileStoreTable table = + createPartitionedReassignTable("failed_dv_rewrite", false) + .copy( + Collections.singletonMap( + CoreOptions.DELETION_VECTOR_INDEX_FILE_TARGET_SIZE.key(), + "1 b")); + for (int first = 0; first < 12; first += 3) { + writePartitionRows(table, "a", first, first + 1, first + 2); + } + BinaryRow partition = partition(table, "a"); + for (long first : new long[] {0, 6}) { + commitDeletionVectors( + table, + partition, + Collections.singletonList(new DvSpec(new Range(first, first + 2), first + 1))); + } + Snapshot snapshot = table.latestSnapshot().get(); + List files = currentDataFiles(table, partition); + List messages = new ArrayList<>(); + for (long first : new long[] {0, 6}) { + List inputs = + files.stream() + .filter( + file -> + file.nonNullFirstRowId() >= first + && file.nonNullFirstRowId() < first + 6) + .collect(Collectors.toList()); + messages.add( + new DataEvolutionNormalCompactTask( + partition, inputs, Collections.emptyList(), false) + .doCompact(table, "failed-dv-rewrite")); + } + assertThat(new DataEvolutionDeletionVectorRewritePlanner(table, snapshot).plan(messages)) + .hasSizeGreaterThan(1); + + IndexFileHandler handler = table.store().newIndexFileHandler(); + DeletionVectorsIndexFile index = handler.dvIndex(partition, UNAWARE_BUCKET); + List oldIndexes = + handler.scan(snapshot, DELETION_VECTORS_INDEX, partition, UNAWARE_BUCKET); + String unreadableAnchor = anchorFilesByRange(table, partition).get(new Range(6, 8)); + IndexFileMeta unreadable = + oldIndexes.stream() + .filter(file -> file.dvRanges().containsKey(unreadableAnchor)) + .findFirst() + .get(); + table.fileIO().delete(index.path(unreadable), false); + Path indexDirectory = index.path(unreadable).getParent(); + Set filesBefore = + Arrays.stream(table.fileIO().listStatus(indexDirectory)) + .map(status -> status.getPath()) + .collect(Collectors.toSet()); + + assertThatThrownBy( + () -> + new DataEvolutionCompactDeletionVectorRewriter(table, snapshot) + .rewriteDeletionVectors(messages)) + .isInstanceOf(RuntimeException.class) + .hasMessageContaining("Unable to read deletion vector"); + + assertThat( + Arrays.stream(table.fileIO().listStatus(indexDirectory)) + .map(status -> status.getPath()) + .collect(Collectors.toSet())) + .isEqualTo(filesBefore); + assertThat(table.latestSnapshot().get().id()).isEqualTo(snapshot.id()); + } + @Test public void testCompactScansDeletionVectorIndexesOnceAcrossPartitions() throws Exception { FileStoreTable table = createPartitionedReassignTable("compact_dv_scan", false); @@ -2112,6 +2287,14 @@ private void commitDeletionVectors(FileStoreTable table, List deletionVe private void commitDeletionVectors( FileStoreTable table, BinaryRow partition, List deletionVectorSpecs) throws Exception { + commit( + table, + Collections.singletonList( + prepareDeletionVectors(table, partition, deletionVectorSpecs))); + } + + private CommitMessage prepareDeletionVectors( + FileStoreTable table, BinaryRow partition, List deletionVectorSpecs) { BaseAppendDeleteFileMaintainer maintainer = BaseAppendDeleteFileMaintainer.forUnawareAppend( table.store().newIndexFileHandler(), @@ -2137,20 +2320,17 @@ private void commitDeletionVectors( } } - commit( - table, - Collections.singletonList( - new CommitMessageImpl( - partition, - UNAWARE_BUCKET, - null, - new DataIncrement( - Collections.emptyList(), - Collections.emptyList(), - Collections.emptyList(), - newIndexFiles, - deletedIndexFiles), - CompactIncrement.emptyIncrement()))); + return new CommitMessageImpl( + partition, + UNAWARE_BUCKET, + null, + new DataIncrement( + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyList(), + newIndexFiles, + deletedIndexFiles), + CompactIncrement.emptyIncrement()); } private static void commit(FileStoreTable table, List commitMessages) diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionCommitPreparationOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionCommitPreparationOperator.java index f2c0aa7bffdb..3ca97ef9f20c 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionCommitPreparationOperator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionCommitPreparationOperator.java @@ -19,12 +19,13 @@ package org.apache.paimon.flink.sink; import org.apache.paimon.Snapshot; -import org.apache.paimon.append.dataevolution.DataEvolutionCompactionCommitPreparation; import org.apache.paimon.options.Options; import org.apache.paimon.table.FileStoreTable; -import org.apache.paimon.table.sink.CommitMessage; import org.apache.paimon.table.sink.CommitMessageImpl; +import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; import org.apache.flink.streaming.api.operators.StreamOperator; import org.apache.flink.streaming.api.operators.StreamOperatorFactory; import org.apache.flink.streaming.api.operators.StreamOperatorParameters; @@ -34,22 +35,50 @@ import java.util.ArrayList; import java.util.List; -/** Prepares deletion-vector and index changes for a data-evolution rewrite commit. */ +/** Prepares commits after all distributed DV rewrites finish. */ public class DataEvolutionCommitPreparationOperator extends PrepareCommitOperator { - private final FileStoreTable table; private final Snapshot snapshot; private final boolean materializeDvRowIdCheck; private final List committables; + static DataStream prepare( + DataStream written, + FileStoreTable table, + Snapshot snapshot, + boolean materializeDvRowIdCheck) { + SingleOutputStreamOperator planned = + written.transform( + "Data Evolution DV Rewrite Planner : " + table.name(), + PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO, + new DataEvolutionDeletionVectorRewriteOperator.PlannerFactory( + table, snapshot)) + .forceNonParallel(); + DataStream rewritten = + planned.rebalance() + .transform( + "Data Evolution DV Rewriter : " + table.name(), + new CommittableTypeInfo(), + new DataEvolutionDeletionVectorRewriteOperator.Factory(table)) + .setParallelism(written.getParallelism()); + return rewritten + .union( + planned.getSideOutput( + DataEvolutionDeletionVectorRewriteOperator.DATA_COMMITS)) + .transform( + "Data Evolution Commit Preparation : " + table.name(), + new CommittableTypeInfo(), + new Factory(table, snapshot, materializeDvRowIdCheck)) + .forceNonParallel(); + } + private DataEvolutionCommitPreparationOperator( StreamOperatorParameters parameters, FileStoreTable table, Snapshot snapshot, boolean materializeDvRowIdCheck) { super(parameters, Options.fromMap(table.options())); - this.table = table; this.snapshot = snapshot; this.materializeDvRowIdCheck = materializeDvRowIdCheck; this.committables = new ArrayList<>(); @@ -69,14 +98,6 @@ protected List prepareCommit(boolean waitCompaction, long checkpoin return toCommit; } - List messages = new ArrayList<>(toCommit.size()); - for (Committable committable : toCommit) { - messages.add(committable.commitMessage()); - } - for (CommitMessage message : - new DataEvolutionCompactionCommitPreparation(table, snapshot).prepare(messages)) { - toCommit.add(new Committable(toCommit.get(0).checkpointId(), message)); - } if (!materializeDvRowIdCheck) { return toCommit; } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionDeletionVectorRewriteOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionDeletionVectorRewriteOperator.java new file mode 100644 index 000000000000..5e181049feac --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionDeletionVectorRewriteOperator.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.sink; + +import org.apache.paimon.Snapshot; +import org.apache.paimon.append.dataevolution.DataEvolutionDeletionVectorRewritePlanner; +import org.apache.paimon.append.dataevolution.DataEvolutionDeletionVectorRewriteTask; +import org.apache.paimon.fs.Path; +import org.apache.paimon.index.IndexFileMeta; +import org.apache.paimon.index.IndexPathFactory; +import org.apache.paimon.options.Options; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.sink.CommitMessage; +import org.apache.paimon.table.sink.CommitMessageImpl; + +import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.AbstractStreamOperatorFactory; +import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory; +import org.apache.flink.streaming.api.operators.StreamOperator; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.util.OutputTag; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** Executes partition-local DV batches in parallel, emitting only commit metadata. */ +public class DataEvolutionDeletionVectorRewriteOperator extends AbstractStreamOperator + implements OneInputStreamOperator { + + private static final Logger LOG = + LoggerFactory.getLogger(DataEvolutionDeletionVectorRewriteOperator.class); + + static final OutputTag DATA_COMMITS = + new OutputTag<>("data-evolution-data-commits", new CommittableTypeInfo()); + + private final FileStoreTable table; + private final DataEvolutionDeletionVectorRewriteTask.Serializer serializer = + new DataEvolutionDeletionVectorRewriteTask.Serializer(); + private final List outputs = new ArrayList<>(); + private boolean finished; + + private DataEvolutionDeletionVectorRewriteOperator( + StreamOperatorParameters parameters, FileStoreTable table) { + this.table = table; + setup(parameters.getContainingTask(), parameters.getStreamConfig(), parameters.getOutput()); + } + + @Override + public void processElement(StreamRecord element) throws Exception { + try { + DataEvolutionDeletionVectorRewriteTask task = + serializer.deserialize(serializer.getVersion(), element.getValue()); + CommitMessageImpl message = (CommitMessageImpl) task.execute(table); + IndexPathFactory paths = + table.store() + .pathFactory() + .indexFileFactory(message.partition(), message.bucket()); + for (IndexFileMeta file : message.compactIncrement().newIndexFiles()) { + outputs.add(paths.toPath(file)); + } + output.collect(new StreamRecord<>(new Committable(Long.MAX_VALUE, message))); + } catch (Exception failure) { + cleanup(); + throw failure; + } + } + + @Override + public void finish() throws Exception { + super.finish(); + finished = true; + outputs.clear(); + } + + @Override + public void close() throws Exception { + try { + if (!finished) { + cleanup(); + } + } finally { + super.close(); + } + } + + private void cleanup() { + for (Path output : outputs) { + try { + table.fileIO().deleteQuietly(output); + } catch (RuntimeException failure) { + LOG.warn("Failed to delete uncommitted DV index file {}.", output, failure); + } + } + outputs.clear(); + } + + /** Collects all data outputs before assigning each old DV index file to exactly one task. */ + private static class Planner extends PrepareCommitOperator { + + private final FileStoreTable table; + private final Snapshot snapshot; + private final List committables = new ArrayList<>(); + + private Planner( + StreamOperatorParameters parameters, + FileStoreTable table, + Snapshot snapshot) { + super(parameters, Options.fromMap(table.options())); + this.table = table; + this.snapshot = snapshot; + } + + @Override + public void processElement(StreamRecord element) { + committables.add(element.getValue()); + } + + @Override + protected List prepareCommit(boolean waitCompaction, long checkpointId) + throws IOException { + List messages = new ArrayList<>(committables.size()); + for (Committable committable : committables) { + messages.add(committable.commitMessage()); + output.collect(DATA_COMMITS, new StreamRecord<>(committable)); + } + committables.clear(); + List tasks = new ArrayList<>(); + DataEvolutionDeletionVectorRewriteTask.Serializer serializer = + new DataEvolutionDeletionVectorRewriteTask.Serializer(); + for (DataEvolutionDeletionVectorRewriteTask task : + new DataEvolutionDeletionVectorRewritePlanner(table, snapshot).plan(messages)) { + tasks.add(serializer.serialize(task)); + } + return tasks; + } + } + + /** Factory for the metadata-only planner. */ + public static class PlannerFactory extends PrepareCommitOperator.Factory { + private final FileStoreTable table; + private final Snapshot snapshot; + + public PlannerFactory(FileStoreTable table, Snapshot snapshot) { + super(Options.fromMap(table.options())); + this.table = table; + this.snapshot = snapshot; + } + + @Override + @SuppressWarnings("unchecked") + public > T createStreamOperator( + StreamOperatorParameters parameters) { + return (T) new Planner(parameters, table, snapshot); + } + + @Override + @SuppressWarnings("rawtypes") + public Class getStreamOperatorClass(ClassLoader classLoader) { + return Planner.class; + } + } + + /** Factory for the parallel bitmap workers. */ + public static class Factory extends AbstractStreamOperatorFactory + implements OneInputStreamOperatorFactory { + private final FileStoreTable table; + + public Factory(FileStoreTable table) { + this.table = table; + } + + @Override + @SuppressWarnings("unchecked") + public > T createStreamOperator( + StreamOperatorParameters parameters) { + return (T) new DataEvolutionDeletionVectorRewriteOperator(parameters, table); + } + + @Override + @SuppressWarnings("rawtypes") + public Class getStreamOperatorClass(ClassLoader classLoader) { + return DataEvolutionDeletionVectorRewriteOperator.class; + } + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionRowIdRewriteSink.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionRowIdRewriteSink.java index 3d013bb85d6a..92de51ba5f40 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionRowIdRewriteSink.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionRowIdRewriteSink.java @@ -80,14 +80,7 @@ public static DataStreamSink commitSorted( private DataStreamSink commitWritten( DataStream written, String initialCommitUser) { - written = - written.transform( - "Data Evolution Row ID Rewrite Commit Preparation : " - + table.name(), - new CommittableTypeInfo(), - new DataEvolutionCommitPreparationOperator.Factory( - table, snapshot, true)) - .forceNonParallel(); + written = DataEvolutionCommitPreparationOperator.prepare(written, table, snapshot, true); return doCommit(written, initialCommitUser); } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionTableCompactSink.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionTableCompactSink.java index 97fc301b4674..e45a85373674 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionTableCompactSink.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionTableCompactSink.java @@ -52,13 +52,7 @@ public DataStreamSink sinkFrom( DataStream written = doWrite(input, initialCommitUser, null); if (table.coreOptions().deletionVectorsEnabled()) { written = - written.transform( - "Data Evolution Compact Deletion Vector Rewriter : " - + table.name(), - new CommittableTypeInfo(), - new DataEvolutionCommitPreparationOperator.Factory( - table, snapshot)) - .forceNonParallel(); + DataEvolutionCommitPreparationOperator.prepare(written, table, snapshot, false); } return doCommit(written, initialCommitUser); } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/DataEvolutionDeleteSqlITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/DataEvolutionDeleteSqlITCase.java index 2115105394ab..fa6acf8a4f40 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/DataEvolutionDeleteSqlITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/DataEvolutionDeleteSqlITCase.java @@ -65,16 +65,26 @@ public void testDeleteRowsWithoutRewritingDataFiles() throws Exception { } @Test - public void testMaterializeDeletionVectorsProcedure() throws Exception { + public void testCompactAndMaterializeDeletionVectorsProcedure() throws Exception { tEnv.getConfig().set(TableConfigOptions.TABLE_DML_SYNC, true); createTable(); sql("INSERT INTO T VALUES (1, 'one', 'A'), (2, 'two', 'A'), (3, 'three', 'A')"); + sql("INSERT INTO T VALUES (7, 'seven', 'A'), (8, 'eight', 'A'), (9, 'nine', 'A')"); sql("INSERT INTO T VALUES (4, 'four', 'B'), (5, 'five', 'B'), (6, 'six', 'B')"); sql("DELETE FROM T WHERE id IN (2, 4)"); FileStoreTable table = paimonTable("T"); assertThat(deletionVectorCardinality(table)).isEqualTo(2L); + List rowsWithIds = sql("SELECT id, _ROW_ID FROM `T$row_tracking` ORDER BY id"); + // Force separate bitmap/old-index work in the same partition through the parallel stage. + sql( + "CALL sys.compact(`table` => 'default.T', options => " + + "'compaction.min.file-num=2,deletion-vector.index-file.target-size=1 b')"); + assertThat(sql("SELECT id, _ROW_ID FROM `T$row_tracking` ORDER BY id")) + .isEqualTo(rowsWithIds); + assertThat(deletionVectorCardinality(table)).isEqualTo(2L); + sql("CALL sys.materialize_deletion_vectors(`table` => 'default.T')"); table = paimonTable("T"); @@ -84,7 +94,10 @@ public void testMaterializeDeletionVectorsProcedure() throws Exception { Row.of(1, "one", "A"), Row.of(3, "three", "A"), Row.of(5, "five", "B"), - Row.of(6, "six", "B")); + Row.of(6, "six", "B"), + Row.of(7, "seven", "A"), + Row.of(8, "eight", "A"), + Row.of(9, "nine", "A")); } @Test diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/DataEvolutionDeletionVectorRewriteOperatorTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/DataEvolutionDeletionVectorRewriteOperatorTest.java new file mode 100644 index 000000000000..19b6fb2e521b --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/DataEvolutionDeletionVectorRewriteOperatorTest.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.sink; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.Snapshot; +import org.apache.paimon.append.dataevolution.DataEvolutionDeletionVectorRewritePlanner; +import org.apache.paimon.append.dataevolution.DataEvolutionDeletionVectorRewriteTask; +import org.apache.paimon.append.dataevolution.DataEvolutionNormalCompactTask; +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.deletionvectors.BitmapDeletionVector; +import org.apache.paimon.deletionvectors.DeletionVectorsIndexFile; +import org.apache.paimon.fs.Path; +import org.apache.paimon.index.IndexFileMeta; +import org.apache.paimon.io.CompactIncrement; +import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.io.DataIncrement; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.TableTestBase; +import org.apache.paimon.table.sink.CommitMessage; +import org.apache.paimon.table.sink.CommitMessageImpl; +import org.apache.paimon.table.sink.TableCommitImpl; +import org.apache.paimon.types.DataTypes; + +import org.apache.flink.api.common.RuntimeExecutionMode; +import org.apache.flink.api.common.functions.RichMapFunction; +import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.RestartStrategyOptions; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.api.functions.sink.v2.DiscardingSink; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +import static org.apache.paimon.table.BucketMode.UNAWARE_BUCKET; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests attempt-scoped cleanup of distributed deletion-vector rewrites. */ +public class DataEvolutionDeletionVectorRewriteOperatorTest extends TableTestBase { + + private static final AtomicInteger LAST_ATTEMPT = new AtomicInteger(); + + private FileStoreTable table; + private DeletionVectorsIndexFile index; + private List oldIndexes; + private List tasks; + + @BeforeEach + public void prepareRewrites() throws Exception { + createTableDefault(); + table = getTableDefault(); + try (TableCommitImpl commit = table.newCommit("insert")) { + commit.commit(writeDataDefault(2, 4)); + } + List files = + table.store().newScan().plan().files().stream() + .map(entry -> entry.file()) + .sorted(Comparator.comparingLong(DataFileMeta::nonNullFirstRowId)) + .collect(Collectors.toList()); + assertThat(files).hasSize(4); + index = table.store().newIndexFileHandler().dvIndex(BinaryRow.EMPTY_ROW, UNAWARE_BUCKET); + oldIndexes = new ArrayList<>(); + for (int file : new int[] {0, 2}) { + BitmapDeletionVector bitmap = new BitmapDeletionVector(); + bitmap.delete(0); + List added = + index.writeWithRolling( + Collections.singletonMap(files.get(file).fileName(), bitmap)); + oldIndexes.addAll(added); + try (TableCommitImpl commit = table.newCommit("delete-" + file)) { + commit.commit( + Collections.singletonList( + new CommitMessageImpl( + BinaryRow.EMPTY_ROW, + UNAWARE_BUCKET, + null, + DataIncrement.indexIncrement(added), + CompactIncrement.emptyIncrement()))); + } + } + Snapshot snapshot = table.latestSnapshot().get(); + List compact = new ArrayList<>(); + for (int first : new int[] {0, 2}) { + compact.add( + new DataEvolutionNormalCompactTask( + BinaryRow.EMPTY_ROW, + files.subList(first, first + 2), + Collections.emptyList(), + false) + .doCompact(table, "compact")); + } + tasks = new ArrayList<>(); + DataEvolutionDeletionVectorRewriteTask.Serializer serializer = + new DataEvolutionDeletionVectorRewriteTask.Serializer(); + for (DataEvolutionDeletionVectorRewriteTask task : + new DataEvolutionDeletionVectorRewritePlanner( + table.copy( + Collections.singletonMap( + CoreOptions.DELETION_VECTOR_INDEX_FILE_TARGET_SIZE + .key(), + "1 b")), + snapshot) + .plan(compact)) { + tasks.add(serializer.serialize(task)); + } + assertThat(tasks).hasSizeGreaterThan(1); + } + + @ParameterizedTest + @ValueSource(strings = {"failure", "cancel", "success"}) + public void testAttemptCleanup(String outcome) throws Exception { + Set before = indexFiles(); + try (OneInputStreamOperatorTestHarness harness = + new OneInputStreamOperatorTestHarness<>( + new DataEvolutionDeletionVectorRewriteOperator.Factory(table))) { + harness.setup(new CommittableTypeInfo().createSerializer(harness.getExecutionConfig())); + harness.open(); + harness.processElement(new StreamRecord<>(tasks.get(0))); + assertThat(indexFiles()).hasSizeGreaterThan(before.size()); + if ("failure".equals(outcome)) { + assertThatThrownBy(() -> harness.processElement(new StreamRecord<>(new byte[0]))) + .isInstanceOf(Exception.class); + } else if ("success".equals(outcome)) { + harness.getOperator().finish(); + } else { + // Harness.close() finishes normally; cancellation closes without finishing. + harness.getOperator().close(); + } + } + if ("success".equals(outcome)) { + assertThat(indexFiles()).hasSizeGreaterThan(before.size()); + } else { + assertThat(indexFiles()).isEqualTo(before); + } + } + + @Test + public void testFailedWorkerAttemptCleansEarlierOutputsOnRestart() throws Exception { + index.delete(oldIndexes.get(1)); + Set before = indexFiles(); + long snapshot = table.latestSnapshot().get().id(); + LAST_ATTEMPT.set(0); + Configuration configuration = new Configuration(); + configuration.set(RestartStrategyOptions.RESTART_STRATEGY, "fixed-delay"); + configuration.set(RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS, 1); + configuration.set(RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_DELAY, Duration.ZERO); + StreamExecutionEnvironment env = + StreamExecutionEnvironment.getExecutionEnvironment(configuration); + env.setRuntimeMode(RuntimeExecutionMode.BATCH); + env.setParallelism(1); + env.fromCollection(tasks, PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO) + .map( + new RichMapFunction() { + @Override + public byte[] map(byte[] value) { + LAST_ATTEMPT.set( + getRuntimeContext().getTaskInfo().getAttemptNumber()); + return value; + } + }) + .transform( + "DV rewrite", + new CommittableTypeInfo(), + new DataEvolutionDeletionVectorRewriteOperator.Factory(table)) + .sinkTo(new DiscardingSink<>()); + assertThatThrownBy(env::execute).hasStackTraceContaining("Unable to read deletion vector"); + assertThat(LAST_ATTEMPT.get()).isEqualTo(1); + assertThat(indexFiles()).isEqualTo(before); + assertThat(table.latestSnapshot().get().id()).isEqualTo(snapshot); + } + + private Set indexFiles() throws Exception { + return Arrays.stream(table.fileIO().listStatus(index.path(oldIndexes.get(0)).getParent())) + .map(status -> status.getPath()) + .collect(Collectors.toSet()); + } + + @Override + protected Schema schemaDefault() { + return Schema.newBuilder() + .column("id", DataTypes.INT()) + .option(CoreOptions.BUCKET.key(), "-1") + .option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true") + .option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true") + .option(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true") + .option(CoreOptions.WRITE_ONLY.key(), "true") + .build(); + } + + @Override + protected InternalRow dataDefault(int time, int size) { + return GenericRow.of(time * size); + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/DataEvolutionRewriteExecutor.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/DataEvolutionRewriteExecutor.java index 950291628877..4aea0bd3e8a8 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/DataEvolutionRewriteExecutor.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/DataEvolutionRewriteExecutor.java @@ -22,6 +22,11 @@ import org.apache.paimon.append.dataevolution.DataEvolutionCompactTask; import org.apache.paimon.append.dataevolution.DataEvolutionCompactTaskSerializer; import org.apache.paimon.append.dataevolution.DataEvolutionCompactionCommitPreparation; +import org.apache.paimon.append.dataevolution.DataEvolutionDeletionVectorRewriteTask; +import org.apache.paimon.append.dataevolution.DataEvolutionDeletionVectorRewriteTask.Serializer; +import org.apache.paimon.fs.Path; +import org.apache.paimon.index.IndexFileMeta; +import org.apache.paimon.index.IndexPathFactory; import org.apache.paimon.operation.commit.DataEvolutionRowRangeConflictException; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.sink.CommitMessage; @@ -32,10 +37,13 @@ import org.apache.paimon.utils.ExceptionUtils; import org.apache.paimon.utils.RetryWaiter; +import org.apache.spark.TaskContext; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.function.FlatMapFunction; import org.apache.spark.sql.SparkSession; +import org.apache.spark.util.TaskCompletionListener; +import org.apache.spark.util.TaskFailureListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -314,16 +322,33 @@ private static Snapshot commitWithMergeConflictRetry( } List preparedMessages = new ArrayList<>(attemptMessages); - List preparationArtifacts = - new DataEvolutionCompactionCommitPreparation(table, attemptSnapshot) - .prepare(preparedMessages); + DataEvolutionCompactionCommitPreparation preparation = + new DataEvolutionCompactionCommitPreparation(table, attemptSnapshot); + List preparationArtifacts = new ArrayList<>(); + try { + preparationArtifacts.addAll( + rewriteDeletionVectors( + table, + preparation.planDeletionVectorRewrites(attemptMessages), + sparkSession)); + } catch (Exception failure) { + RuntimeException preparationFailure = + failure instanceof RuntimeException + ? (RuntimeException) failure + : new RuntimeException( + "Prepare deletion vector rewrites failed.", failure); + List staged = new ArrayList<>(retryArtifacts); + staged.addAll(preparationArtifacts); + abortRetryArtifacts(table, commitUser, staged, preparationFailure); + throw preparationFailure; + } preparedMessages.addAll(preparationArtifacts); List abortMessages = new ArrayList<>(retryArtifacts); abortMessages.addAll(preparationArtifacts); try (TableCommitImpl commit = table.newCommit(commitUser)) { - commitConfigurer.configure(commit); - commitConfigurer.prepareMessages(attemptSnapshot, preparedMessages); try { + commitConfigurer.configure(commit); + commitConfigurer.prepareMessages(attemptSnapshot, preparedMessages); commit.commit(preparedMessages); } catch (RuntimeException conflict) { if (commitConfigurer == RANGE_SORT_COMMIT) { @@ -365,6 +390,102 @@ private static Snapshot commitWithMergeConflictRetry( } } + private static List rewriteDeletionVectors( + FileStoreTable table, + List tasks, + SparkSession sparkSession) { + if (tasks.isEmpty()) { + return new ArrayList<>(); + } + Serializer serializer = new Serializer(); + List serializedTasks = new ArrayList<>(tasks.size()); + try { + for (DataEvolutionDeletionVectorRewriteTask task : tasks) { + serializedTasks.add(serializer.serialize(task)); + } + int parallelism = readParallelism(serializedTasks, sparkSession); + LOG.info( + "Rewriting deletion vectors for table {} in {} tasks with parallelism {}.", + table.fullName(), + tasks.size(), + parallelism); + JavaRDD rewritten = + JavaSparkContext.fromSparkContext(sparkSession.sparkContext()) + .parallelize(serializedTasks, parallelism) + .mapPartitions( + iterator -> + rewriteDeletionVectorPartition( + table, iterator, TaskContext.get()) + .iterator()); + // Successful tasks cannot observe a later failure elsewhere in the job. Their + // uncommitted files, and outputs lost with an executor, need orphan cleanup. + List messages = new ArrayList<>(rewritten.collect()); + return deserializeCommitMessagesAndReleaseSerializedBytes( + new CommitMessageSerializer(), messages); + } catch (IOException e) { + throw new RuntimeException("Serialize deletion vector rewrite messages failed.", e); + } + } + + static List rewriteDeletionVectorPartition( + FileStoreTable table, Iterator iterator, @Nullable TaskContext context) + throws IOException { + List outputs = new ArrayList<>(); + Runnable cleanup = + () -> { + for (Path output : outputs) { + try { + table.fileIO().deleteQuietly(output); + } catch (RuntimeException failure) { + LOG.warn( + "Failed to delete uncommitted DV index file {}.", + output, + failure); + } + } + outputs.clear(); + }; + if (context != null) { + context.addTaskFailureListener( + (TaskFailureListener) (failed, failure) -> cleanup.run()); + context.addTaskCompletionListener( + (TaskCompletionListener) + completed -> { + if (completed.isInterrupted()) { + cleanup.run(); + } + }); + } + Serializer taskSerializer = new Serializer(); + CommitMessageSerializer messageSerializer = new CommitMessageSerializer(); + List results = new ArrayList<>(); + try { + while (iterator.hasNext()) { + if (context != null) { + context.killTaskIfInterrupted(); + } + DataEvolutionDeletionVectorRewriteTask task = + taskSerializer.deserialize(taskSerializer.getVersion(), iterator.next()); + CommitMessageImpl message = (CommitMessageImpl) task.execute(table); + IndexPathFactory paths = + table.store() + .pathFactory() + .indexFileFactory(message.partition(), message.bucket()); + for (IndexFileMeta file : message.compactIncrement().newIndexFiles()) { + outputs.add(paths.toPath(file)); + } + results.add(messageSerializer.serialize(message)); + } + if (context != null) { + context.killTaskIfInterrupted(); + } + return results; + } catch (IOException | RuntimeException failure) { + cleanup.run(); + throw failure; + } + } + private static List retryArtifacts( List compactMessages, List rewrittenMessages) { checkArgument( diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CompactProcedureTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CompactProcedureTestBase.scala index 8671762798e1..89cf55890dfd 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CompactProcedureTestBase.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CompactProcedureTestBase.scala @@ -20,13 +20,16 @@ package org.apache.paimon.spark.procedure import org.apache.paimon.{CoreOptions, Snapshot} import org.apache.paimon.Snapshot.CommitKind -import org.apache.paimon.append.dataevolution.DataEvolutionNormalCompactTask -import org.apache.paimon.data.BinaryRow +import org.apache.paimon.append.dataevolution.{DataEvolutionCompactTask, DataEvolutionDeletionVectorRewritePlanner, DataEvolutionDeletionVectorRewriteTask, DataEvolutionNormalCompactTask} +import org.apache.paimon.data.{BinaryRow, BinaryString, GenericRow} import org.apache.paimon.deletionvectors.DeletionVectorsIndexFile.DELETION_VECTORS_INDEX import org.apache.paimon.format.blob.BlobFileFormat -import org.apache.paimon.fs.Path -import org.apache.paimon.io.DataFileMeta +import org.apache.paimon.fs.{Path, SeekableInputStream} +import org.apache.paimon.fs.local.LocalFileIO +import org.apache.paimon.index.IndexFileMeta +import org.apache.paimon.io.{CompactIncrement, DataFileMeta} import org.apache.paimon.manifest.FileSource +import org.apache.paimon.operation.LocalOrphanFilesClean import org.apache.paimon.operation.commit.DataEvolutionRowRangeConflictException import org.apache.paimon.partition.PartitionPredicate import org.apache.paimon.spark.PaimonSparkTestBase @@ -34,12 +37,14 @@ import org.apache.paimon.spark.catalyst.analysis.PaimonRelation import org.apache.paimon.spark.commands.{DataEvolutionCompactMergeConflictRewriter, DataEvolutionPaimonWriter, PaimonSparkWriter} import org.apache.paimon.spark.commands.CompactRowIdRangeIndex import org.apache.paimon.spark.utils.SparkProcedureUtils -import org.apache.paimon.table.FileStoreTable -import org.apache.paimon.table.sink.CommitMessageImpl +import org.apache.paimon.table.{AppendOnlyFileStoreTable, FileStoreTable} +import org.apache.paimon.table.BucketMode.UNAWARE_BUCKET +import org.apache.paimon.table.sink.{CommitMessage, CommitMessageImpl, TableCommitImpl} import org.apache.paimon.table.source.{DataSplit, IncrementalSplit} import org.apache.paimon.table.source.snapshot.SnapshotReader import org.apache.paimon.utils.Range +import org.apache.spark.{SparkException, TaskContext} import org.apache.spark.api.java.JavaSparkContext import org.apache.spark.scheduler.{SparkListener, SparkListenerStageSubmitted} import org.apache.spark.sql.{Dataset, Row} @@ -50,10 +55,12 @@ import org.assertj.core.api.Assertions import org.assertj.core.api.Assertions.assertThatThrownBy import org.scalatest.time.Span +import java.io.IOException import java.lang.reflect.{InvocationHandler, Method, Proxy} import java.time.LocalDate import java.time.format.DateTimeFormatter import java.util +import java.util.concurrent.{CountDownLatch, TimeUnit} import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger, AtomicLong, AtomicReference} import scala.collection.JavaConverters._ @@ -3001,7 +3008,9 @@ abstract class CompactProcedureTestBase extends PaimonSparkTestBase with StreamT Assertions.assertThat(deletionVectorIndexFileCount(table)).isEqualTo(1) val snapshotBefore = lastSnapshotId(table) - executeDataEvolutionCompaction(table, candidateFilesPerBatch = 2) + executeDataEvolutionCompaction( + table.copy(Map("deletion-vector.index-file.target-size" -> "1 b").asJava), + candidateFilesPerBatch = 2) Assertions.assertThat(lastSnapshotId(table)).isEqualTo(snapshotBefore + 2) Assertions @@ -3015,6 +3024,229 @@ abstract class CompactProcedureTestBase extends PaimonSparkTestBase with StreamT } } + test("Paimon Procedure: clean outputs of failed deletion vector rewrite partitions") { + withTable("T") { + sql(""" + |CREATE TABLE T (id INT, value STRING) + |TBLPROPERTIES ( + | 'bucket' = '-1', 'row-tracking.enabled' = 'true', + | 'data-evolution.enabled' = 'true', 'deletion-vectors.enabled' = 'true') + |""".stripMargin) + sql("INSERT INTO T VALUES (0, 'zero')") + sql("INSERT INTO T VALUES (1, 'one')") + sql("DELETE FROM T WHERE id = 0") + val table = loadTable("T").copy(Map("deletion-vector.index-file.target-size" -> "1 b").asJava) + val snapshot = table.latestSnapshot().get() + val compact = new DataEvolutionNormalCompactTask( + BinaryRow.EMPTY_ROW, + normalDataFiles(table).asJava, + util.Collections.emptyList[Range](), + false).doCompact(table, "failed-dv-rewrite") + val tasks = new DataEvolutionDeletionVectorRewritePlanner(table, snapshot) + .plan(util.Collections.singletonList(compact)) + val serializer = new DataEvolutionDeletionVectorRewriteTask.Serializer() + val taskBytes = serializer.serialize(tasks.get(0)) + val handler = table.store().newIndexFileHandler() + val oldIndex = handler.scan(snapshot, DELETION_VECTORS_INDEX, BinaryRow.EMPTY_ROW, 0).get(0) + val directory = handler.dvIndex(BinaryRow.EMPTY_ROW, 0).path(oldIndex).getParent + def indexFiles(): Set[Path] = table.fileIO().listStatus(directory).map(_.getPath).toSet + val filesBefore = indexFiles() + + // The first rewrite writes a file; deserializing the following task then fails. + assertThatThrownBy( + () => + DataEvolutionRewriteExecutor.rewriteDeletionVectorPartition( + table, + util.Arrays.asList(taskBytes, Array.emptyByteArray).iterator(), + null)) + .isInstanceOf(classOf[java.io.IOException]) + Assertions.assertThat(indexFiles()).isEqualTo(filesBefore) + + Seq("success", "failure", "cancel").foreach { + outcome => + val module = Class.forName("org.apache.spark.TaskContext$").getField("MODULE$").get(null) + val context = module.getClass.getMethod("empty").invoke(module).asInstanceOf[TaskContext] + val messages = DataEvolutionRewriteExecutor.rewriteDeletionVectorPartition( + table, + util.Collections.singletonList(taskBytes).iterator(), + context) + assert(!messages.isEmpty) + val outputs = indexFiles() -- filesBefore + assert(outputs.nonEmpty) + if (outcome == "failure") { + context.getClass + .getMethod("markTaskFailed", classOf[Throwable]) + .invoke(context, new IllegalStateException("Injected failure")) + } else if (outcome == "cancel") { + context.getClass.getMethod("markInterrupted", classOf[String]).invoke(context, "cancel") + } + context.getClass.getMethod("markTaskCompleted", classOf[Option[_]]).invoke(context, None) + if (outcome == "success") { + Assertions.assertThat(indexFiles()).isEqualTo(filesBefore ++ outputs) + outputs.foreach(table.fileIO().deleteQuietly) + } else { + Assertions.assertThat(indexFiles()).isEqualTo(filesBefore) + } + } + Assertions.assertThat(lastSnapshotId(table)).isEqualTo(snapshot.id()) + Assertions.assertThat(deletionVectorCardinality(table)).isEqualTo(1) + checkAnswer(sql("SELECT id, value FROM T"), Seq(Row(1, "one"))) + } + } + + test("Paimon Procedure: retry distributed deletion vector preparation") { + withTable("T") { + sql(""" + |CREATE TABLE T (id INT, value STRING) + |TBLPROPERTIES ( + | 'bucket' = '-1', 'row-tracking.enabled' = 'true', + | 'data-evolution.enabled' = 'true', 'deletion-vectors.enabled' = 'true', + | 'commit.min-retry-wait' = '1 ms', 'commit.max-retry-wait' = '1 ms') + |""".stripMargin) + sql("INSERT INTO T VALUES (0, 'zero')") + sql("INSERT INTO T VALUES (1, 'one')") + sql("DELETE FROM T WHERE id = 0") + val table = loadTable("T").copy(Map("deletion-vector.index-file.target-size" -> "1 b").asJava) + val snapshot = table.latestSnapshot().get() + val task = new DataEvolutionNormalCompactTask( + BinaryRow.EMPTY_ROW, + normalDataFiles(table).asJava, + util.Collections.emptyList[Range](), + false) + val attempts = new AtomicInteger() + val abortedIndexes = new util.ArrayList[IndexFileMeta]() + val configurer = new DataEvolutionRewriteExecutor.CommitConfigurer { + override def configure(commit: TableCommitImpl): Unit = {} + + override def prepareMessages(base: Snapshot, messages: util.List[CommitMessage]): Unit = { + if (attempts.getAndIncrement() == 0) { + messages.asScala.foreach( + message => + abortedIndexes.addAll( + message.asInstanceOf[CommitMessageImpl].compactIncrement().newIndexFiles())) + throw new DataEvolutionRowRangeConflictException("Injected DV preparation conflict.") + } + } + } + DataEvolutionRewriteExecutor.execute( + table, + snapshot, + _ => + new DataEvolutionRewriteExecutor.PlanResult(util.Collections.singletonList(task), true), + new JavaSparkContext(spark.sparkContext), + spark, + configurer, + (_, _, _, messages) => util.Optional.of(messages) + ) + + Assertions.assertThat(attempts.get()).isEqualTo(2) + assert(!abortedIndexes.isEmpty) + val index = table.store().newIndexFileHandler().dvIndex(BinaryRow.EMPTY_ROW, -1) + abortedIndexes.asScala.foreach(file => assert(!index.exists(file))) + Assertions.assertThat(deletionVectorCardinality(table)).isEqualTo(1) + checkAnswer(sql("SELECT id, value FROM T"), Seq(Row(1, "one"))) + } + } + + test("Paimon Procedure: recover outputs after a distributed DV rewrite job fails") { + withTable("T") { + sql(""" + |CREATE TABLE T (id INT, value STRING) + |TBLPROPERTIES ( + | 'bucket' = '-1', 'row-tracking.enabled' = 'true', + | 'data-evolution.enabled' = 'true', 'deletion-vectors.enabled' = 'true') + |""".stripMargin) + (0 until 4).foreach(id => sql(s"INSERT INTO T VALUES ($id, 'value-$id')")) + sql("DELETE FROM T WHERE id = 0") + sql("DELETE FROM T WHERE id = 2") + val table = loadTable("T").copy(Map("deletion-vector.index-file.target-size" -> "1 b").asJava) + val snapshot = table.latestSnapshot().get() + val files = normalDataFiles(table).sortBy(_.nonNullFirstRowId()) + val compact: Seq[DataEvolutionCompactTask] = Seq(0, 2).map( + first => + new DataEvolutionNormalCompactTask( + BinaryRow.EMPTY_ROW, + files.slice(first, first + 2).asJava, + util.Collections.emptyList[Range](), + false)) + val handler = table.store().newIndexFileHandler() + val index = handler.dvIndex(BinaryRow.EMPTY_ROW, UNAWARE_BUCKET) + val oldIndexes = + handler.scan(snapshot, DELETION_VECTORS_INDEX, BinaryRow.EMPTY_ROW, UNAWARE_BUCKET) + Assertions.assertThat(oldIndexes.size()).isEqualTo(2) + val unreadable = oldIndexes.asScala.find(_.dvRanges().containsKey(files(2).fileName())).get + val readable = oldIndexes.asScala.find(_.dvRanges().containsKey(files.head.fileName())).get + val directory = index.path(unreadable).getParent + def indexFiles(): Set[Path] = table.fileIO().listStatus(directory).map(_.getPath).toSet + val before = indexFiles() + // Advance the head so the rewriter supplies a staged retry artifact before DV preparation. + sql("INSERT INTO T VALUES (4, 'four')") + val latest = table.latestSnapshot().get().id() + val retryFile = new AtomicReference[Path]() + val failingTable = new AppendOnlyFileStoreTable( + new DvRewriteFailureFileIO(index.path(unreadable).toString, index.path(readable).toString), + table.location(), + table.schema(), + table.catalogEnvironment()) + DvRewriteFailure.firstTaskCompleted = new CountDownLatch(1) + try { + withSQLConf("spark.sql.shuffle.partitions" -> "2") { + assertThatThrownBy( + () => + DataEvolutionRewriteExecutor.execute( + failingTable, + snapshot, + _ => new DataEvolutionRewriteExecutor.PlanResult(compact.asJava, true), + new JavaSparkContext(spark.sparkContext), + spark, + new DataEvolutionRewriteExecutor.CommitConfigurer { + override def configure(commit: TableCommitImpl): Unit = {} + }, + (_, _, _, messages) => { + val rewritten = new util.ArrayList[CommitMessage](messages) + val write = table.newBatchWriteBuilder().newWrite() + try { + write.write( + GenericRow.of(Int.box(99), BinaryString.fromString("retry-artifact"))) + val artifacts = write.prepareCommit() + val artifact = artifacts.get(0).asInstanceOf[CommitMessageImpl] + retryFile.set( + table + .store() + .pathFactory() + .createDataFilePathFactory(artifact.partition(), artifact.bucket()) + .toPath(artifact.newFilesIncrement().newFiles().get(0))) + rewritten.add( + new CommitMessageImpl( + artifact.partition(), + UNAWARE_BUCKET, + null, + artifact.newFilesIncrement(), + CompactIncrement.emptyIncrement())) + } finally { + write.close() + } + assert(table.fileIO().exists(retryFile.get())) + util.Optional.of(rewritten) + } + )) + .hasStackTraceContaining(classOf[SparkException].getName) + .hasStackTraceContaining("Injected DV read failure") + } + } finally { + DvRewriteFailure.firstTaskCompleted.countDown() + } + assert(retryFile.get() != null && !table.fileIO().exists(retryFile.get())) + Assertions.assertThat(lastSnapshotId(table)).isEqualTo(latest) + val orphanIndexes = indexFiles() -- before + assert(orphanIndexes.nonEmpty) + // The other partition already succeeded. Exercise the documented orphan-file recovery path. + new LocalOrphanFilesClean(table, System.currentTimeMillis() + 60000).clean() + Assertions.assertThat(indexFiles()).isEqualTo(before) + checkAnswer(sql("SELECT id FROM T"), Seq(Row(1), Row(3), Row(4))) + } + } + private def createCompactMergeRaceTable(): FileStoreTable = { sql(""" |CREATE TABLE T (id INT, value INT, picture BINARY) @@ -3156,3 +3388,27 @@ abstract class CompactProcedureTestBase extends PaimonSparkTestBase with StreamT .sum } } + +private object DvRewriteFailure { + @volatile var firstTaskCompleted = new CountDownLatch(1) +} + +private class DvRewriteFailureFileIO(unreadable: String, readable: String) extends LocalFileIO { + override def newInputStream(path: Path): SeekableInputStream = { + if (path.toString == readable && TaskContext.get() != null) { + TaskContext.get().addTaskCompletionListener[Unit] { + completed => + if (!completed.isInterrupted()) { + DvRewriteFailure.firstTaskCompleted.countDown() + } + } + } + if (path.toString == unreadable) { + if (!DvRewriteFailure.firstTaskCompleted.await(30, TimeUnit.SECONDS)) { + throw new IOException("Timed out waiting for the successful DV rewrite task") + } + throw new IOException("Injected DV read failure") + } + super.newInputStream(path) + } +}