Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions docs/docs/multimodal-table/data-evolution-maintenance.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,20 @@ Ordinary compaction **does not physically remove rows hidden by deletion
vectors**. Removing their positions would change the alignment with untouched
column files.

### Recovering Files After a Failed Rewrite

Flink and Spark rewrite DV indexes on workers. A failed worker attempt cleans
up the new DV files it has tracked. Files from tasks that already succeeded,
or from a lost worker process, can remain unreferenced if the overall rewrite
fails before committing.

After the failed job and its retries have stopped, use
[Remove Orphan Files](../maintenance/manage-snapshots#remove-orphan-files)
to reclaim these files. The default retention window is one day. If setting
`older_than`, choose a cutoff that keeps files from any active writes or
maintenance jobs. Snapshot expiration alone does not reclaim files that were
never committed.

### Incremental Column-Version Compaction

To reduce repeated rewrites of large base files when only a few columns
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,57 +19,14 @@
package org.apache.paimon.append.dataevolution;

import org.apache.paimon.Snapshot;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.deletionvectors.Bitmap64DeletionVector;
import org.apache.paimon.deletionvectors.BitmapDeletionVector;
import org.apache.paimon.deletionvectors.DeletionVector;
import org.apache.paimon.deletionvectors.append.AppendDeleteFileMaintainer;
import org.apache.paimon.deletionvectors.append.BaseAppendDeleteFileMaintainer;
import org.apache.paimon.index.IndexFileHandler;
import org.apache.paimon.index.IndexFileMeta;
import org.apache.paimon.io.CompactIncrement;
import org.apache.paimon.io.DataFileMeta;
import org.apache.paimon.io.DataIncrement;
import org.apache.paimon.manifest.FileKind;
import org.apache.paimon.manifest.IndexManifestEntry;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.sink.CommitMessage;
import org.apache.paimon.table.sink.CommitMessageImpl;
import org.apache.paimon.utils.Pair;
import org.apache.paimon.utils.Range;
import org.apache.paimon.utils.RangeHelper;
import org.apache.paimon.table.sink.TableCommit;

import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.stream.Collectors;

import static org.apache.paimon.deletionvectors.DeletionVectorsIndexFile.DELETION_VECTORS_INDEX;
import static org.apache.paimon.format.blob.BlobFileFormat.isBlobFile;
import static org.apache.paimon.table.BucketMode.UNAWARE_BUCKET;
import static org.apache.paimon.types.VectorType.isVectorStoreFile;
import static org.apache.paimon.utils.DataEvolutionUtils.checkContiguousRowRange;
import static org.apache.paimon.utils.DataEvolutionUtils.retrieveAnchorFile;
import static org.apache.paimon.utils.Preconditions.checkState;

/**
* Rewrites deletion vectors for non-materialized data-evolution compaction.
*
* <p>Compaction tasks rewrite data files without applying deletion vectors. This class consumes all
* compaction messages before commit, moves old anchor-file DVs to new compacted files, and returns
* index-only compact messages for the rewritten DV index files.
*
* <p>Unlike append compaction, data-evolution compaction does not pack compact tasks by deletion
* vector index files. Data-evolution compaction is batch-oriented, and packing by DV may create a
* huge compact task when many files contain only a few deleted rows. This is especially risky when
* blob compaction is involved. Instead, compact tasks are planned normally and DVs are rewritten
* centrally from compact commit messages. This is acceptable because most DVs only need to be
* renamed to the new anchor file and are not fully rewritten.
*/
/** Local execution of the same DV rewrite tasks used by distributed compaction. */
public class DataEvolutionCompactDeletionVectorRewriter {

private final FileStoreTable table;
Expand All @@ -81,231 +38,22 @@ public DataEvolutionCompactDeletionVectorRewriter(FileStoreTable table, Snapshot
}

public List<CommitMessage> rewriteDeletionVectors(List<CommitMessage> messages) {
if (!table.coreOptions().deletionVectorsEnabled() || messages.isEmpty()) {
return Collections.emptyList();
}

Map<BinaryRow, AppendDeleteFileMaintainer> maintainerByParts =
collectMaintainers(snapshot, messages);
if (maintainerByParts.isEmpty()) {
return Collections.emptyList();
}

List<CommitMessage> result = new ArrayList<>();
for (Map.Entry<BinaryRow, AppendDeleteFileMaintainer> entry :
maintainerByParts.entrySet()) {
List<IndexManifestEntry> indexChanges = entry.getValue().persist();
if (!indexChanges.isEmpty()) {
result.add(toCommitMessage(entry.getKey(), indexChanges));
try {
for (DataEvolutionDeletionVectorRewriteTask task :
new DataEvolutionDeletionVectorRewritePlanner(table, snapshot).plan(messages)) {
result.add(task.execute(table));
}
}
return result;
}

private Map<BinaryRow, AppendDeleteFileMaintainer> collectMaintainers(
Snapshot snapshot, List<CommitMessage> messages) {
Map<BinaryRow, AppendDeleteFileMaintainer> result = createMaintainers(snapshot, messages);
RangeHelper<DataFileMeta> rangeHelper = new RangeHelper<>(DataFileMeta::nonNullRowIdRange);

for (CommitMessage message : messages) {
CommitMessageImpl commitMessage = (CommitMessageImpl) message;
CompactIncrement compactIncrement = commitMessage.compactIncrement();
List<DataFileMeta> before = normalFiles(compactIncrement.compactBefore());
List<DataFileMeta> after = normalFiles(compactIncrement.compactAfter());

// Skip blob-only tasks and index-only commit messages.
if (before.isEmpty() && after.isEmpty()) {
continue;
} catch (RuntimeException failure) {
if (!result.isEmpty()) {
try (TableCommit commit = table.newCommit("data-evolution-dv-rewrite-abort")) {
commit.abort(result);
} catch (Exception cleanupFailure) {
failure.addSuppressed(cleanupFailure);
}
}

AppendDeleteFileMaintainer maintainer = result.get(commitMessage.partition());
if (isMaterialized(compactIncrement)) {
removeMaterializedDeletionVectors(maintainer, rangeHelper, before);
continue;
}

Range beforeRange = checkContiguousRowRange(before);
Range afterRange = checkContiguousRowRange(after);
checkState(
beforeRange.equals(afterRange),
"Non-materialized data evolution compaction should keep the same row-id "
+ "range, but compact before range is %s and compact after range is %s.",
beforeRange,
afterRange);
TreeMap<Long, DataFileMeta> outputFiles = new TreeMap<>();
for (DataFileMeta file : after) {
outputFiles.put(file.nonNullFirstRowId(), file);
}
Map<String, DeletionVector> rewritten = new LinkedHashMap<>();

// Redistribute old deletion positions across the output file boundaries.
for (List<DataFileMeta> previousRowGroup : rangeHelper.mergeOverlappingRanges(before)) {
DataFileMeta oldAnchor = retrieveAnchorFile(previousRowGroup, file -> file);
moveDeletionVector(
maintainer,
outputFiles,
rewritten,
oldAnchor.fileName(),
oldAnchor.nonNullRowIdRange());
}
rewritten.forEach(maintainer::notifyNewDeletionVector);
throw failure;
}
return result;
}

private Map<BinaryRow, AppendDeleteFileMaintainer> createMaintainers(
Snapshot snapshot, List<CommitMessage> messages) {
Map<BinaryRow, List<IndexManifestEntry>> entriesByPartition = new LinkedHashMap<>();
for (CommitMessage message : messages) {
CommitMessageImpl commitMessage = (CommitMessageImpl) message;
CompactIncrement increment = commitMessage.compactIncrement();
checkNoCompactIndexChanges(increment);
checkState(
commitMessage.bucket() == UNAWARE_BUCKET
&& commitMessage.totalBuckets() == null,
"Data evolution compaction should only produce unaware-bucket commit "
+ "messages, but got bucket %s and total buckets %s.",
commitMessage.bucket(),
commitMessage.totalBuckets());
List<DataFileMeta> before = normalFiles(increment.compactBefore());
List<DataFileMeta> after = normalFiles(increment.compactAfter());
if (before.isEmpty() && after.isEmpty()) {
continue;
}
entriesByPartition.computeIfAbsent(
commitMessage.partition(), ignored -> new ArrayList<>());
}
if (entriesByPartition.isEmpty()) {
return Collections.emptyMap();
}

IndexFileHandler handler = table.store().newIndexFileHandler();
Set<Pair<BinaryRow, Integer>> buckets =
entriesByPartition.keySet().stream()
.map(partition -> Pair.of(partition, UNAWARE_BUCKET))
.collect(Collectors.toSet());
// Filter before decoding DV ranges and scan only once for all affected partitions.
for (IndexManifestEntry entry :
handler.scanBucketEntries(snapshot, DELETION_VECTORS_INDEX, buckets)) {
entriesByPartition.get(entry.partition()).add(entry);
}

Map<BinaryRow, AppendDeleteFileMaintainer> result = new LinkedHashMap<>();
entriesByPartition.forEach(
(partition, entries) ->
result.put(
partition,
BaseAppendDeleteFileMaintainer.forUnawareAppend(
handler, partition, entries)));
return result;
}

private void removeMaterializedDeletionVectors(
AppendDeleteFileMaintainer maintainer,
RangeHelper<DataFileMeta> rangeHelper,
List<DataFileMeta> before) {
for (List<DataFileMeta> previousRowGroup : rangeHelper.mergeOverlappingRanges(before)) {
DataFileMeta oldAnchor = retrieveAnchorFile(previousRowGroup, file -> file);
maintainer.notifyRemovedDeletionVector(oldAnchor.fileName());
}
}

private boolean isMaterialized(CompactIncrement compactIncrement) {
List<DataFileMeta> before = normalFiles(compactIncrement.compactBefore());
if (before.isEmpty()) {
return false;
}
return compactIncrement.compactAfter().stream().allMatch(file -> file.firstRowId() == null);
}

/**
* Moves old anchor-file deletion vectors to the new files, merging or splitting them at the
* output row-id boundaries.
*/
private void moveDeletionVector(
AppendDeleteFileMaintainer maintainer,
TreeMap<Long, DataFileMeta> outputFiles,
Map<String, DeletionVector> rewritten,
String oldFileName,
Range oldRange) {
DeletionVector old = maintainer.getDeletionVector(oldFileName);
if (old == null || old.isEmpty()) {
return;
}

// Fast path for renaming deletion vectors only.
DataFileMeta firstOutput = outputFiles.floorEntry(oldRange.from).getValue();
if (oldRange.equals(firstOutput.nonNullRowIdRange())) {
rewritten
.computeIfAbsent(firstOutput.fileName(), ignored -> newDeletionVector())
.merge(old);
} else {
old.forEachDeletedPosition(
position -> {
long absolutePosition = oldRange.from + position;
DataFileMeta output = outputFiles.floorEntry(absolutePosition).getValue();
Range newRange = output.nonNullRowIdRange();
long newPosition = absolutePosition - newRange.from;
checkState(
newPosition >= 0 && newPosition < newRange.count(),
"Cannot move deletion position %s from old range %s to new range %s.",
absolutePosition,
oldRange,
newRange);
rewritten
.computeIfAbsent(output.fileName(), ignored -> newDeletionVector())
.delete(newPosition);
});
}
maintainer.notifyRemovedDeletionVector(oldFileName);
}

private List<DataFileMeta> normalFiles(List<DataFileMeta> files) {
return files.stream().filter(this::isNormalFile).collect(Collectors.toList());
}

private boolean isNormalFile(DataFileMeta fileMeta) {
return !isBlobFile(fileMeta.fileName()) && !isVectorStoreFile(fileMeta.fileName());
}

private DeletionVector newDeletionVector() {
return table.coreOptions().deletionVectorBitmap64()
? new Bitmap64DeletionVector()
: new BitmapDeletionVector();
}

private CommitMessage toCommitMessage(
BinaryRow partition, List<IndexManifestEntry> indexChanges) {
List<IndexFileMeta> newIndexFiles = new ArrayList<>();
List<IndexFileMeta> deletedIndexFiles = new ArrayList<>();
for (IndexManifestEntry entry : indexChanges) {
if (entry.kind() == FileKind.ADD) {
newIndexFiles.add(entry.indexFile());
} else {
deletedIndexFiles.add(entry.indexFile());
}
}

return new CommitMessageImpl(
partition,
UNAWARE_BUCKET,
null,
DataIncrement.emptyIncrement(),
new CompactIncrement(
Collections.emptyList(),
Collections.emptyList(),
Collections.emptyList(),
newIndexFiles,
deletedIndexFiles));
}

private void checkNoCompactIndexChanges(CompactIncrement compactIncrement) {
checkState(
compactIncrement.newIndexFiles().isEmpty()
&& compactIncrement.deletedIndexFiles().isEmpty(),
"Data evolution compaction should not produce index changes before deletion vector rewrite, "
+ "but got new index files %s and deleted index files %s.",
compactIncrement.newIndexFiles(),
compactIncrement.deletedIndexFiles());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -48,4 +48,9 @@ public List<CommitMessage> prepare(List<CommitMessage> messages) {
.rewriteDeletionVectors(messages));
return result;
}

public List<DataEvolutionDeletionVectorRewriteTask> planDeletionVectorRewrites(
List<CommitMessage> messages) {
return new DataEvolutionDeletionVectorRewritePlanner(table, snapshot).plan(messages);
}
}
Loading
Loading