Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions docs/source/user_guide/append_only_table.rst
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,13 @@ If a table does not have a primary key defined, it is an append table.
Ordinary append writes accept ``INSERT`` rows only and do not apply upserts
or deletes by key.

Fixed-bucket append tables preserve append order by default
(``bucket-append-ordered = true``). Set it to ``false`` to allow unordered writes
and custom compaction of files that are not contiguous in append order.
In unordered mode, ``write-only = true`` skips restoring historical files and
sequence numbers; with ``write-only = false``, historical files are restored for
compaction even if the write context requests ignoring previous files.

The experimental real-time append deduplicate mode is an exception. With
``realtime.deduplicate-key-fields`` configured, it also accepts ``UPDATE_AFTER``,
``DELETE`` and ``UPDATE_BEFORE``, using deletion vectors to hide superseded
Expand Down
19 changes: 19 additions & 0 deletions docs/source/user_guide/compaction.rst
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,25 @@ size, so that a file already close to the target is not compacted twice.
primary-key tables, which makes the derived threshold about 179 MiB and 89 MiB
respectively. Adjust ``target-file-size`` to move the threshold.

Custom Compaction
~~~~~~~~~~~~~~~~~
``FileStoreWrite::CompactFiles(partition, bucket, file_names)`` asynchronously
rewrites selected files, accepting partition values, a bucket, and distinct data
file basenames. As with ``Compact``, each partition-bucket restores the latest
snapshot when its writer is initialized and reuses that state. Files need not be
contiguous; compaction thresholds do not apply.

Supports fixed-bucket append tables with or without deletion vectors. Requires
``bucket-append-ordered = false``, ``write-only = false`` and a dedicated writer
with the full schema. Real-time mode and data evolution are unsupported.

Use ``PrepareCommit`` to collect results and background errors, following the
usual ``Compact`` waiting rules. Commit or abort the returned messages through
``FileStoreCommit``.

Do not update deletion vectors or run another compaction for the same
partition-bucket until the custom compaction result is committed or discarded.

Dictionary Passthrough
~~~~~~~~~~~~~~~~~~~~~~
An append-only compaction rewrite copies rows into the new file without
Expand Down
4 changes: 2 additions & 2 deletions include/paimon/defs.h
Original file line number Diff line number Diff line change
Expand Up @@ -720,8 +720,8 @@ struct PAIMON_EXPORT Options {
/// "write-only" - If set to "true", compactions and snapshot expiration will be skipped. This
/// option is used along with dedicated compact jobs. Default value is "false".
static const char WRITE_ONLY[];
/// "bucket-append-ordered" - Whether append writes in fixed bucket mode are ordered. This
/// option is used by commit conflict checks. Default value is "false".
/// "bucket-append-ordered" - Whether append writes in fixed bucket mode preserve append order.
/// Set to "false" to allow unordered writes and custom compaction. Default value is "true".
static const char BUCKET_APPEND_ORDERED[];
/// "write.sequence-number-init-mode" - Specify how to initialize the next sequence number for
/// primary key table writers. Values can be: "scan", "snapshot". Default value is "scan".
Expand Down
12 changes: 12 additions & 0 deletions include/paimon/file_store_write.h
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,18 @@ class PAIMON_EXPORT FileStoreWrite {
virtual Status Compact(const std::map<std::string, std::string>& partition, int32_t bucket,
bool full_compaction) = 0;

/// Asynchronously compacts requested files from the writer's restored state, as with Compact().
/// Supports fixed-bucket append tables with or without deletion vectors.
/// Requires bucket-append-ordered=false, write-only=false and the full write schema.
/// Use a dedicated writer. Real-time mode and data evolution are unsupported.
/// Accepts complete partition values and distinct data file basenames.
/// Files need not be contiguous.
/// Collect results and background errors through PrepareCommit(), as with Compact().
/// Do not update deletion vectors or compact the same partition-bucket until commit or discard.
/// TODO(xinyu.lxy): Consider extracting this API into a separate interface.
virtual Status CompactFiles(const std::map<std::string, std::string>& partition, int32_t bucket,
const std::vector<std::string>& file_names);

/// Generate a list of commit messages with the latest generated data file meta
/// information of the current snapshot.
///
Expand Down
10 changes: 10 additions & 0 deletions src/paimon/core/append/append_only_writer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
#include "paimon/common/utils/checked_cast.h"
#include "paimon/common/utils/long_counter.h"
#include "paimon/common/utils/scope_guard.h"
#include "paimon/core/append/bucketed_append_compact_manager.h"
#include "paimon/core/io/append_data_file_writer_factory.h"
#include "paimon/core/io/blob_data_file_writer_factory.h"
#include "paimon/core/io/compact_increment.h"
Expand Down Expand Up @@ -110,6 +111,15 @@ Status AppendOnlyWriter::Write(std::unique_ptr<RecordBatch>&& batch) {
return writer_->Write(batch->GetData());
}

Status AppendOnlyWriter::CompactFiles(const std::vector<std::string>& file_names) {
auto manager = std::dynamic_pointer_cast<BucketedAppendCompactManager>(compact_manager_);
if (!manager) {
return Status::Invalid("custom compaction requires a bucketed append compact manager");
}
PAIMON_RETURN_NOT_OK(Sync());
return manager->CompactFiles(file_names);
}

Result<CommitIncrement> AppendOnlyWriter::PrepareCommit(bool wait_compaction) {
PAIMON_RETURN_NOT_OK(
Flush(/*wait_for_latest_compaction=*/false, /*forced_full_compaction=*/false));
Expand Down
1 change: 1 addition & 0 deletions src/paimon/core/append/append_only_writer.h
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ class AppendOnlyWriter : public BatchWriter {
Status Compact(bool full_compaction) override {
return Flush(/*wait_for_latest_compaction=*/true, full_compaction);
}
Status CompactFiles(const std::vector<std::string>& file_names);

uint64_t GetMemoryUsage() const override {
// used for spill, AppendOnlyWriter do not support spill, so return 0 to avoid triggering
Expand Down
66 changes: 57 additions & 9 deletions src/paimon/core/append/bucketed_append_compact_manager.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
#include "paimon/core/append/bucketed_append_compact_manager.h"

#include <cassert>
#include <unordered_set>

#include "paimon/common/executor/future.h"

Expand Down Expand Up @@ -57,6 +58,9 @@ void BucketedAppendCompactManager::RequestCancelCompaction() {
}

Status BucketedAppendCompactManager::TriggerCompaction(bool full_compaction) {
if (custom_compaction_) {
return Status::OK();
}
if (full_compaction) {
PAIMON_RETURN_NOT_OK(TriggerFullCompaction());
} else {
Expand All @@ -65,6 +69,42 @@ Status BucketedAppendCompactManager::TriggerCompaction(bool full_compaction) {
return Status::OK();
}

Status BucketedAppendCompactManager::CompactFiles(const std::vector<std::string>& file_names) {
if (task_future_.valid()) {
return Status::Invalid("a compaction task is still running");
}
custom_compaction_ = true;
if (file_names.empty()) {
return Status::Invalid("custom compaction requires at least one input file");
}
std::unordered_set<std::string> names(file_names.begin(), file_names.end());
if (names.size() != file_names.size()) {
return Status::Invalid("custom compaction requires distinct file names");
}
auto candidates = to_compact_;
std::vector<std::shared_ptr<DataFileMeta>> selected;
std::vector<std::shared_ptr<DataFileMeta>> retained;
while (!candidates.empty()) {
std::shared_ptr<DataFileMeta> file = candidates.top();
candidates.pop();
if (names.erase(file->file_name) > 0) {
selected.push_back(file);
} else {
retained.push_back(file);
}
}
if (!names.empty()) {
return Status::Invalid("custom compaction input file is not active in this writer: ",
*names.begin());
}
to_compact_.swap(candidates);
for (const auto& file : retained) {
to_compact_.push(file);
}
SubmitCompaction(selected);
return Status::OK();
}

Status BucketedAppendCompactManager::TriggerFullCompaction() {
if (task_future_.valid()) {
return Status::Invalid(
Expand Down Expand Up @@ -100,17 +140,21 @@ void BucketedAppendCompactManager::TriggerCompactionWithBestEffort() {
}
std::optional<std::vector<std::shared_ptr<DataFileMeta>>> picked = PickCompactBefore();
if (picked) {
cancellation_controller_->Reset();
compacting_ = picked.value();
auto compact_task = std::make_shared<AutoCompactTask>(
reporter_, dv_maintainer_, compacting_.value(), rewriter_, materialize_dv_on_commit_);
task_future_ =
Via(executor_.get(), [compact_task]() -> Result<std::shared_ptr<CompactResult>> {
return compact_task->Execute();
});
SubmitCompaction(picked.value());
}
}

void BucketedAppendCompactManager::SubmitCompaction(
const std::vector<std::shared_ptr<DataFileMeta>>& files) {
cancellation_controller_->Reset();
compacting_ = files;
auto compact_task = std::make_shared<AutoCompactTask>(reporter_, dv_maintainer_, files,
rewriter_, materialize_dv_on_commit_);
task_future_ = Via(executor_.get(), [compact_task]() -> Result<std::shared_ptr<CompactResult>> {
return compact_task->Execute();
});
}

std::optional<std::vector<std::shared_ptr<DataFileMeta>>>
BucketedAppendCompactManager::PickCompactBefore() {
if (to_compact_.empty()) {
Expand Down Expand Up @@ -162,7 +206,11 @@ BucketedAppendCompactManager::GetCompactionResult(bool blocking) {
InnerGetCompactionResult(blocking));
if (result) {
std::shared_ptr<CompactResult> compact_result = result.value();
if (!compact_result->After().empty()) {
if (custom_compaction_) {
for (const auto& file : compact_result->After()) {
to_compact_.push(file);
}
} else if (!compact_result->After().empty()) {
// if the last compacted file is still small,
// add it back to the head
std::shared_ptr<DataFileMeta> last_file = compact_result->After().back();
Expand Down
7 changes: 6 additions & 1 deletion src/paimon/core/append/bucketed_append_compact_manager.h
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
#include <memory>
#include <optional>
#include <queue>
#include <string>
#include <vector>

#include "paimon/common/executor/future.h"
Expand Down Expand Up @@ -89,6 +90,8 @@ class BucketedAppendCompactManager : public CompactFutureManager {

Status TriggerCompaction(bool full_compaction) override;

Status CompactFiles(const std::vector<std::string>& file_names);

bool ShouldWaitForLatestCompaction() const override {
return false;
}
Expand Down Expand Up @@ -166,7 +169,7 @@ class BucketedAppendCompactManager : public CompactFutureManager {
bool materialize_dv_on_commit_;
};

/// A `CompactTask` impl for append-only table auto-compaction.
/// A `CompactTask` impl for selected append-only files.
///
/// This task accepts an already-picked candidate to perform one-time rewrite. And for the
/// rest of input files, it is the duty of `AppendOnlyWriter` to invoke the next time
Expand Down Expand Up @@ -202,6 +205,7 @@ class BucketedAppendCompactManager : public CompactFutureManager {
std::optional<std::vector<std::shared_ptr<DataFileMeta>>> PickCompactBefore();
Status TriggerFullCompaction();
void TriggerCompactionWithBestEffort();
void SubmitCompaction(const std::vector<std::shared_ptr<DataFileMeta>>& files);

std::shared_ptr<Executor> executor_;
std::shared_ptr<BucketedDvMaintainer> dv_maintainer_;
Expand All @@ -217,6 +221,7 @@ class BucketedAppendCompactManager : public CompactFutureManager {
// If true, compaction leaves DV file generation to commit using the latest snapshot;
// otherwise, compaction generates its DV files before commit.
bool materialize_dv_on_commit_;
bool custom_compaction_ = false;
};

} // namespace paimon
58 changes: 58 additions & 0 deletions src/paimon/core/append/bucketed_append_compact_manager_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
#include <vector>

#include "gtest/gtest.h"
#include "paimon/common/utils/scope_guard.h"
#include "paimon/core/deletionvectors/bitmap_deletion_vector.h"
#include "paimon/core/deletionvectors/bucketed_dv_maintainer.h"
#include "paimon/core/deletionvectors/deletion_vectors_index_file.h"
Expand Down Expand Up @@ -311,6 +312,63 @@ TEST_F(BucketedAppendCompactManagerTest, TestPick) {
NewFile(2601, 2610), NewFile(2611, 2620), NewFile(2621, 2630)});
}

TEST_F(BucketedAppendCompactManagerTest, TestCompactFilesUsesAsyncResultLifecycle) {
auto cancellation_controller = std::make_shared<CancellationController>();
auto files = GenerateDataFileMeta();
std::vector<std::shared_ptr<DataFileMeta>> selected = {files[0], files[2]};
std::vector<std::string> file_names = {files[2]->file_name, files[0]->file_name};
std::vector<std::shared_ptr<DataFileMeta>> outputs = {
std::make_shared<DataFileMeta>(*files[0]), std::make_shared<DataFileMeta>(*files[2])};
outputs[0]->file_name = "compacted1";
outputs[0]->file_size = 2048;
outputs[1]->file_name = "compacted3";
std::vector<std::shared_ptr<DataFileMeta>> rewritten;
std::promise<void> release;
auto released = release.get_future().share();
auto rewriter = [&rewritten, released,
outputs](const std::vector<std::shared_ptr<DataFileMeta>>& inputs)
-> Result<std::vector<std::shared_ptr<DataFileMeta>>> {
released.wait();
rewritten = inputs;
return outputs;
};
BucketedAppendCompactManager manager(
executor_, files, /*dv_maintainer=*/nullptr, /*min_file_num=*/100,
/*target_file_size=*/1024, /*compaction_file_size=*/700,
/*force_rewrite_all_files=*/false, rewriter, /*reporter=*/nullptr, cancellation_controller,
/*materialize_dv_on_commit=*/false);
std::future<Status> submission;
// Unblock the rewrite before an assertion failure joins the submission future.
ScopeGuard unblock([&release]() { release.set_value(); });
cancellation_controller->Cancel();
submission = std::async(std::launch::async, [&]() { return manager.CompactFiles(file_names); });
ASSERT_EQ(std::future_status::ready, submission.wait_for(std::chrono::seconds(5)));
ASSERT_OK(submission.get());
ASSERT_FALSE(cancellation_controller->IsCancelled());
ASSERT_OK_AND_ASSIGN(auto pending, manager.GetCompactionResult(/*blocking=*/false));
ASSERT_FALSE(pending);
ASSERT_TRUE(manager.CompactNotCompleted());
ASSERT_NOK_WITH_MSG(manager.CompactFiles(file_names), "still running");
release.set_value();
unblock.Release();
ASSERT_OK_AND_ASSIGN(auto result, manager.GetCompactionResult(/*blocking=*/true));
ASSERT_TRUE(result);
ExpectVectorsEqual(selected, rewritten);
ExpectVectorsEqual(selected, result.value()->Before());
ExpectVectorsEqual(outputs, result.value()->After());
std::vector<std::shared_ptr<DataFileMeta>> active = {outputs[0], files[1], outputs[1]};
ExpectVectorsEqual(active, manager.AllFiles());
ASSERT_NOK_WITH_MSG(manager.CompactFiles(file_names), "not active");
ExpectVectorsEqual(active, manager.AllFiles());
ASSERT_OK(manager.TriggerCompaction(/*full_compaction=*/false));
ASSERT_FALSE(manager.CompactNotCompleted());
ASSERT_OK(manager.CompactFiles({outputs[1]->file_name, outputs[0]->file_name}));
ASSERT_OK_AND_ASSIGN(auto next_result, manager.GetCompactionResult(/*blocking=*/true));
ASSERT_TRUE(next_result);
ExpectVectorsEqual(outputs, next_result.value()->Before());
ExpectVectorsEqual(active, manager.AllFiles());
}

TEST_F(BucketedAppendCompactManagerTest, TestCancelCompactionPropagatesToRewriteLoop) {
auto cancellation_controller = std::make_shared<CancellationController>();
auto exit_signal = std::make_shared<std::promise<void>>();
Expand Down
2 changes: 1 addition & 1 deletion src/paimon/core/core_options.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -463,7 +463,7 @@ struct CoreOptions::Impl {
bool manifest_delete_file_drop_stats = false;
bool write_buffer_spillable = true;
bool write_only = false;
bool bucket_append_ordered = false;
bool bucket_append_ordered = true;
bool deletion_vectors_enabled = false;
bool pk_clustering_override = false;
bool deletion_vectors_bitmap64 = false;
Expand Down
6 changes: 3 additions & 3 deletions src/paimon/core/core_options_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ TEST(CoreOptionsTest, TestDefaultValue) {
ASSERT_EQ(SortEngine::LOSER_TREE, core_options.GetSortEngine());
ASSERT_FALSE(core_options.IgnoreDelete());
ASSERT_FALSE(core_options.WriteOnly());
ASSERT_FALSE(core_options.BucketAppendOrdered());
ASSERT_TRUE(core_options.BucketAppendOrdered());
ASSERT_EQ(CoreOptions::SequenceNumberInitMode::SCAN,
core_options.WriteSequenceNumberInitMode());
ASSERT_EQ(5, core_options.GetCompactionMinFileNum());
Expand Down Expand Up @@ -322,7 +322,7 @@ TEST(CoreOptionsTest, TestFromMap) {
{Options::GLOBAL_INDEX_EXTERNAL_PATH, "FILE:///tmp/global_index/"},
{Options::SCAN_TAG_NAME, "test-tag"},
{Options::WRITE_ONLY, "true"},
{Options::BUCKET_APPEND_ORDERED, "true"},
{Options::BUCKET_APPEND_ORDERED, "false"},
{Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, "snapshot"},
{Options::COMPACTION_MIN_FILE_NUM, "10"},
{Options::COMPACTION_FORCE_REWRITE_ALL_FILES, "true"},
Expand Down Expand Up @@ -504,7 +504,7 @@ TEST(CoreOptionsTest, TestFromMap) {
ASSERT_EQ(375809637, core_options.GetCompactionFileSize(/*has_primary_key=*/true));
ASSERT_EQ(375809637, core_options.GetCompactionFileSize(/*has_primary_key=*/false));
ASSERT_TRUE(core_options.WriteOnly());
ASSERT_TRUE(core_options.BucketAppendOrdered());
ASSERT_FALSE(core_options.BucketAppendOrdered());
ASSERT_EQ(CoreOptions::SequenceNumberInitMode::SNAPSHOT,
core_options.WriteSequenceNumberInitMode());
ASSERT_EQ(10, core_options.GetCompactionMinFileNum());
Expand Down
3 changes: 2 additions & 1 deletion src/paimon/core/operation/abstract_file_store_write.h
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,8 @@ class AbstractFileStoreWrite : public FileStoreWrite {
const RealtimeOffsetMap& committed_offsets,
const RealtimeDataFileMap& committed_data_files);

Result<std::shared_ptr<BatchWriter>> GetWriter(const BinaryRow& partition, int32_t bucket);

Result<std::shared_ptr<RestoreFiles>> ScanExistingFileMetas(const BinaryRow& partition,
int32_t bucket) const;
int32_t GetDefaultBucketNum() const;
Expand Down Expand Up @@ -184,7 +186,6 @@ class AbstractFileStoreWrite : public FileStoreWrite {
void InvalidateRealtimeContext();
Status PoisonRealtimeWrite(const Status& cause);

Result<std::shared_ptr<BatchWriter>> GetWriter(const BinaryRow& partition, int32_t bucket);
Result<std::vector<RealtimeCommitProgress>> PrepareRealtimeCommit();

private:
Expand Down
Loading
Loading