From 14e9c149fd34c0a7bc942b7cb007a8005e3534f2 Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Mon, 16 Nov 2026 18:25:59 +0800 Subject: [PATCH 1/2] feat: support custom compaction for bucketed append tables --- docs/source/user_guide/append_only_table.rst | 7 + docs/source/user_guide/compaction.rst | 19 ++ include/paimon/compact_request.h | 41 ++++ include/paimon/defs.h | 4 +- include/paimon/file_store_write.h | 9 + src/paimon/core/append/append_only_writer.cpp | 10 + src/paimon/core/append/append_only_writer.h | 1 + .../bucketed_append_compact_manager.cpp | 66 ++++++- .../append/bucketed_append_compact_manager.h | 7 +- .../bucketed_append_compact_manager_test.cpp | 59 ++++++ src/paimon/core/core_options.cpp | 2 +- src/paimon/core/core_options_test.cpp | 6 +- .../operation/abstract_file_store_write.h | 3 +- .../append_only_file_store_write.cpp | 63 +++++++ .../operation/append_only_file_store_write.h | 9 + .../append_only_file_store_write_test.cpp | 177 ++++++++++++++++++ .../core/operation/file_store_write.cpp | 4 + .../key_value_file_store_write_test.cpp | 2 + .../table/append_only_file_store_table.cpp | 7 + test/inte/append_compaction_inte_test.cpp | 167 +++++++++++++++++ test/inte/realtime_write_inte_test.cpp | 1 + 21 files changed, 647 insertions(+), 17 deletions(-) create mode 100644 include/paimon/compact_request.h diff --git a/docs/source/user_guide/append_only_table.rst b/docs/source/user_guide/append_only_table.rst index 4ed6061b5..07c73a142 100644 --- a/docs/source/user_guide/append_only_table.rst +++ b/docs/source/user_guide/append_only_table.rst @@ -24,6 +24,13 @@ If a table does not have a primary key defined, it is an append table. Ordinary append writes accept ``INSERT`` rows only and do not apply upserts or deletes by key. +Fixed-bucket append tables preserve append order by default +(``bucket-append-ordered = true``). Set it to ``false`` to allow unordered writes +and custom compaction of files that are not contiguous in append order. +In unordered mode, ``write-only = true`` skips restoring historical files and +sequence numbers; with ``write-only = false``, historical files are restored for +compaction even if the write context requests ignoring previous files. + The experimental real-time append deduplicate mode is an exception. With ``realtime.deduplicate-key-fields`` configured, it also accepts ``UPDATE_AFTER``, ``DELETE`` and ``UPDATE_BEFORE``, using deletion vectors to hide superseded diff --git a/docs/source/user_guide/compaction.rst b/docs/source/user_guide/compaction.rst index 92aa01c81..58dc64b0b 100644 --- a/docs/source/user_guide/compaction.rst +++ b/docs/source/user_guide/compaction.rst @@ -102,6 +102,25 @@ size, so that a file already close to the target is not compacted twice. primary-key tables, which makes the derived threshold about 179 MiB and 89 MiB respectively. Adjust ``target-file-size`` to move the threshold. +Custom Compaction +~~~~~~~~~~~~~~~~~ +``FileStoreWrite::CustomizeCompact`` asynchronously rewrites selected files. +``CompactRequest`` specifies partition values, a bucket, and distinct data file +basenames. As with ``Compact``, each partition-bucket restores the latest snapshot +when its writer is initialized and reuses that state. Files need not be contiguous; +compaction thresholds do not apply. + +Supports fixed-bucket append tables with or without deletion vectors. Requires +``bucket-append-ordered = false``, ``write-only = false`` and a dedicated writer +with the full schema. Real-time mode and data evolution are unsupported. + +Use ``PrepareCommit`` to collect results and background errors, following the +usual ``Compact`` waiting rules. Commit or abort the returned messages through +``FileStoreCommit``. + +Do not update deletion vectors or run another compaction for the same +partition-bucket until the custom compaction result is committed or discarded. + Dictionary Passthrough ~~~~~~~~~~~~~~~~~~~~~~ An append-only compaction rewrite copies rows into the new file without diff --git a/include/paimon/compact_request.h b/include/paimon/compact_request.h new file mode 100644 index 000000000..a8dfa2c4a --- /dev/null +++ b/include/paimon/compact_request.h @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/visibility.h" + +namespace paimon { + +/// Files to rewrite in one partition-bucket of an append-only table. +struct PAIMON_EXPORT CompactRequest { + /// Complete logical partition values; empty for an unpartitioned table. + std::map partition; + int32_t bucket = -1; + /// Nonempty, distinct data file names as recorded in the manifest, without directories. + /// Files need not be contiguous in append sequence order. + std::vector file_names; +}; + +} // namespace paimon diff --git a/include/paimon/defs.h b/include/paimon/defs.h index 16cf6382a..782068808 100644 --- a/include/paimon/defs.h +++ b/include/paimon/defs.h @@ -715,8 +715,8 @@ struct PAIMON_EXPORT Options { /// "write-only" - If set to "true", compactions and snapshot expiration will be skipped. This /// option is used along with dedicated compact jobs. Default value is "false". static const char WRITE_ONLY[]; - /// "bucket-append-ordered" - Whether append writes in fixed bucket mode are ordered. This - /// option is used by commit conflict checks. Default value is "false". + /// "bucket-append-ordered" - Whether append writes in fixed bucket mode preserve append order. + /// Set to "false" to allow unordered writes and custom compaction. Default value is "true". static const char BUCKET_APPEND_ORDERED[]; /// "write.sequence-number-init-mode" - Specify how to initialize the next sequence number for /// primary key table writers. Values can be: "scan", "snapshot". Default value is "scan". diff --git a/include/paimon/file_store_write.h b/include/paimon/file_store_write.h index ae01dd080..48d0881ca 100644 --- a/include/paimon/file_store_write.h +++ b/include/paimon/file_store_write.h @@ -26,6 +26,7 @@ #include #include "paimon/commit_message.h" +#include "paimon/compact_request.h" #include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" @@ -89,6 +90,14 @@ class PAIMON_EXPORT FileStoreWrite { virtual Status Compact(const std::map& partition, int32_t bucket, bool full_compaction) = 0; + /// Asynchronously compacts requested files from the writer's restored state, as with Compact(). + /// Supports fixed-bucket append tables with or without deletion vectors. + /// Requires bucket-append-ordered=false, write-only=false and the full write schema. + /// Use a dedicated writer. Real-time mode and data evolution are unsupported. + /// Collect results and background errors through PrepareCommit(), as with Compact(). + /// Do not update deletion vectors or compact the same partition-bucket until commit or discard. + virtual Status CustomizeCompact(const CompactRequest& request); + /// Generate a list of commit messages with the latest generated data file meta /// information of the current snapshot. /// diff --git a/src/paimon/core/append/append_only_writer.cpp b/src/paimon/core/append/append_only_writer.cpp index 9b3f32c59..711a78b2f 100644 --- a/src/paimon/core/append/append_only_writer.cpp +++ b/src/paimon/core/append/append_only_writer.cpp @@ -37,6 +37,7 @@ #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/long_counter.h" #include "paimon/common/utils/scope_guard.h" +#include "paimon/core/append/bucketed_append_compact_manager.h" #include "paimon/core/io/append_data_file_writer_factory.h" #include "paimon/core/io/blob_data_file_writer_factory.h" #include "paimon/core/io/compact_increment.h" @@ -110,6 +111,15 @@ Status AppendOnlyWriter::Write(std::unique_ptr&& batch) { return writer_->Write(batch->GetData()); } +Status AppendOnlyWriter::CustomizeCompact(const std::vector& file_names) { + auto manager = std::dynamic_pointer_cast(compact_manager_); + if (!manager) { + return Status::Invalid("custom compaction requires a bucketed append compact manager"); + } + PAIMON_RETURN_NOT_OK(Sync()); + return manager->CustomizeCompact(file_names); +} + Result AppendOnlyWriter::PrepareCommit(bool wait_compaction) { PAIMON_RETURN_NOT_OK( Flush(/*wait_for_latest_compaction=*/false, /*forced_full_compaction=*/false)); diff --git a/src/paimon/core/append/append_only_writer.h b/src/paimon/core/append/append_only_writer.h index eb4c2de67..a717ffd06 100644 --- a/src/paimon/core/append/append_only_writer.h +++ b/src/paimon/core/append/append_only_writer.h @@ -70,6 +70,7 @@ class AppendOnlyWriter : public BatchWriter { Status Compact(bool full_compaction) override { return Flush(/*wait_for_latest_compaction=*/true, full_compaction); } + Status CustomizeCompact(const std::vector& file_names); uint64_t GetMemoryUsage() const override { // used for spill, AppendOnlyWriter do not support spill, so return 0 to avoid triggering diff --git a/src/paimon/core/append/bucketed_append_compact_manager.cpp b/src/paimon/core/append/bucketed_append_compact_manager.cpp index e1fd343cc..963e19142 100644 --- a/src/paimon/core/append/bucketed_append_compact_manager.cpp +++ b/src/paimon/core/append/bucketed_append_compact_manager.cpp @@ -19,6 +19,7 @@ #include "paimon/core/append/bucketed_append_compact_manager.h" #include +#include #include "paimon/common/executor/future.h" @@ -57,6 +58,9 @@ void BucketedAppendCompactManager::RequestCancelCompaction() { } Status BucketedAppendCompactManager::TriggerCompaction(bool full_compaction) { + if (custom_compaction_) { + return Status::OK(); + } if (full_compaction) { PAIMON_RETURN_NOT_OK(TriggerFullCompaction()); } else { @@ -65,6 +69,42 @@ Status BucketedAppendCompactManager::TriggerCompaction(bool full_compaction) { return Status::OK(); } +Status BucketedAppendCompactManager::CustomizeCompact(const std::vector& file_names) { + if (task_future_.valid()) { + return Status::Invalid("a compaction task is still running"); + } + custom_compaction_ = true; + if (file_names.empty()) { + return Status::Invalid("custom compaction requires at least one input file"); + } + std::unordered_set names(file_names.begin(), file_names.end()); + if (names.size() != file_names.size()) { + return Status::Invalid("custom compaction requires distinct file names"); + } + auto candidates = to_compact_; + std::vector> selected; + std::vector> retained; + while (!candidates.empty()) { + std::shared_ptr file = candidates.top(); + candidates.pop(); + if (names.erase(file->file_name) > 0) { + selected.push_back(file); + } else { + retained.push_back(file); + } + } + if (!names.empty()) { + return Status::Invalid("custom compaction input file is not active in this writer: ", + *names.begin()); + } + to_compact_.swap(candidates); + for (const auto& file : retained) { + to_compact_.push(file); + } + SubmitCompaction(selected); + return Status::OK(); +} + Status BucketedAppendCompactManager::TriggerFullCompaction() { if (task_future_.valid()) { return Status::Invalid( @@ -100,17 +140,21 @@ void BucketedAppendCompactManager::TriggerCompactionWithBestEffort() { } std::optional>> picked = PickCompactBefore(); if (picked) { - cancellation_controller_->Reset(); - compacting_ = picked.value(); - auto compact_task = std::make_shared( - reporter_, dv_maintainer_, compacting_.value(), rewriter_, materialize_dv_on_commit_); - task_future_ = - Via(executor_.get(), [compact_task]() -> Result> { - return compact_task->Execute(); - }); + SubmitCompaction(picked.value()); } } +void BucketedAppendCompactManager::SubmitCompaction( + const std::vector>& files) { + cancellation_controller_->Reset(); + compacting_ = files; + auto compact_task = std::make_shared(reporter_, dv_maintainer_, files, + rewriter_, materialize_dv_on_commit_); + task_future_ = Via(executor_.get(), [compact_task]() -> Result> { + return compact_task->Execute(); + }); +} + std::optional>> BucketedAppendCompactManager::PickCompactBefore() { if (to_compact_.empty()) { @@ -162,7 +206,11 @@ BucketedAppendCompactManager::GetCompactionResult(bool blocking) { InnerGetCompactionResult(blocking)); if (result) { std::shared_ptr compact_result = result.value(); - if (!compact_result->After().empty()) { + if (custom_compaction_) { + for (const auto& file : compact_result->After()) { + to_compact_.push(file); + } + } else if (!compact_result->After().empty()) { // if the last compacted file is still small, // add it back to the head std::shared_ptr last_file = compact_result->After().back(); diff --git a/src/paimon/core/append/bucketed_append_compact_manager.h b/src/paimon/core/append/bucketed_append_compact_manager.h index 8ad375181..3db0f2c24 100644 --- a/src/paimon/core/append/bucketed_append_compact_manager.h +++ b/src/paimon/core/append/bucketed_append_compact_manager.h @@ -23,6 +23,7 @@ #include #include #include +#include #include #include "paimon/common/executor/future.h" @@ -89,6 +90,8 @@ class BucketedAppendCompactManager : public CompactFutureManager { Status TriggerCompaction(bool full_compaction) override; + Status CustomizeCompact(const std::vector& file_names); + bool ShouldWaitForLatestCompaction() const override { return false; } @@ -166,7 +169,7 @@ class BucketedAppendCompactManager : public CompactFutureManager { bool materialize_dv_on_commit_; }; - /// A `CompactTask` impl for append-only table auto-compaction. + /// A `CompactTask` impl for selected append-only files. /// /// This task accepts an already-picked candidate to perform one-time rewrite. And for the /// rest of input files, it is the duty of `AppendOnlyWriter` to invoke the next time @@ -202,6 +205,7 @@ class BucketedAppendCompactManager : public CompactFutureManager { std::optional>> PickCompactBefore(); Status TriggerFullCompaction(); void TriggerCompactionWithBestEffort(); + void SubmitCompaction(const std::vector>& files); std::shared_ptr executor_; std::shared_ptr dv_maintainer_; @@ -217,6 +221,7 @@ class BucketedAppendCompactManager : public CompactFutureManager { // If true, compaction leaves DV file generation to commit using the latest snapshot; // otherwise, compaction generates its DV files before commit. bool materialize_dv_on_commit_; + bool custom_compaction_ = false; }; } // namespace paimon diff --git a/src/paimon/core/append/bucketed_append_compact_manager_test.cpp b/src/paimon/core/append/bucketed_append_compact_manager_test.cpp index 91fa25650..104b029bc 100644 --- a/src/paimon/core/append/bucketed_append_compact_manager_test.cpp +++ b/src/paimon/core/append/bucketed_append_compact_manager_test.cpp @@ -27,6 +27,7 @@ #include #include "gtest/gtest.h" +#include "paimon/common/utils/scope_guard.h" #include "paimon/core/deletionvectors/bitmap_deletion_vector.h" #include "paimon/core/deletionvectors/bucketed_dv_maintainer.h" #include "paimon/core/deletionvectors/deletion_vectors_index_file.h" @@ -311,6 +312,64 @@ TEST_F(BucketedAppendCompactManagerTest, TestPick) { NewFile(2601, 2610), NewFile(2611, 2620), NewFile(2621, 2630)}); } +TEST_F(BucketedAppendCompactManagerTest, TestCustomizeCompactUsesAsyncResultLifecycle) { + auto cancellation_controller = std::make_shared(); + auto files = GenerateDataFileMeta(); + std::vector> selected = {files[0], files[2]}; + std::vector file_names = {files[2]->file_name, files[0]->file_name}; + std::vector> outputs = { + std::make_shared(*files[0]), std::make_shared(*files[2])}; + outputs[0]->file_name = "compacted1"; + outputs[0]->file_size = 2048; + outputs[1]->file_name = "compacted3"; + std::vector> rewritten; + std::promise release; + auto released = release.get_future().share(); + auto rewriter = [&rewritten, released, + outputs](const std::vector>& inputs) + -> Result>> { + released.wait(); + rewritten = inputs; + return outputs; + }; + BucketedAppendCompactManager manager( + executor_, files, /*dv_maintainer=*/nullptr, /*min_file_num=*/100, + /*target_file_size=*/1024, /*compaction_file_size=*/700, + /*force_rewrite_all_files=*/false, rewriter, /*reporter=*/nullptr, cancellation_controller, + /*materialize_dv_on_commit=*/false); + std::future submission; + // Unblock the rewrite before an assertion failure joins the submission future. + ScopeGuard unblock([&release]() { release.set_value(); }); + cancellation_controller->Cancel(); + submission = + std::async(std::launch::async, [&]() { return manager.CustomizeCompact(file_names); }); + ASSERT_EQ(std::future_status::ready, submission.wait_for(std::chrono::seconds(5))); + ASSERT_OK(submission.get()); + ASSERT_FALSE(cancellation_controller->IsCancelled()); + ASSERT_OK_AND_ASSIGN(auto pending, manager.GetCompactionResult(/*blocking=*/false)); + ASSERT_FALSE(pending); + ASSERT_TRUE(manager.CompactNotCompleted()); + ASSERT_NOK_WITH_MSG(manager.CustomizeCompact(file_names), "still running"); + release.set_value(); + unblock.Release(); + ASSERT_OK_AND_ASSIGN(auto result, manager.GetCompactionResult(/*blocking=*/true)); + ASSERT_TRUE(result); + ExpectVectorsEqual(selected, rewritten); + ExpectVectorsEqual(selected, result.value()->Before()); + ExpectVectorsEqual(outputs, result.value()->After()); + std::vector> active = {outputs[0], files[1], outputs[1]}; + ExpectVectorsEqual(active, manager.AllFiles()); + ASSERT_NOK_WITH_MSG(manager.CustomizeCompact(file_names), "not active"); + ExpectVectorsEqual(active, manager.AllFiles()); + ASSERT_OK(manager.TriggerCompaction(/*full_compaction=*/false)); + ASSERT_FALSE(manager.CompactNotCompleted()); + ASSERT_OK(manager.CustomizeCompact({outputs[1]->file_name, outputs[0]->file_name})); + ASSERT_OK_AND_ASSIGN(auto next_result, manager.GetCompactionResult(/*blocking=*/true)); + ASSERT_TRUE(next_result); + ExpectVectorsEqual(outputs, next_result.value()->Before()); + ExpectVectorsEqual(active, manager.AllFiles()); +} + TEST_F(BucketedAppendCompactManagerTest, TestCancelCompactionPropagatesToRewriteLoop) { auto cancellation_controller = std::make_shared(); auto exit_signal = std::make_shared>(); diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp index 74affc24e..8af83b8bd 100644 --- a/src/paimon/core/core_options.cpp +++ b/src/paimon/core/core_options.cpp @@ -461,7 +461,7 @@ struct CoreOptions::Impl { bool manifest_delete_file_drop_stats = false; bool write_buffer_spillable = true; bool write_only = false; - bool bucket_append_ordered = false; + bool bucket_append_ordered = true; bool deletion_vectors_enabled = false; bool pk_clustering_override = false; bool deletion_vectors_bitmap64 = false; diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp index ebad7d838..01a6cffe7 100644 --- a/src/paimon/core/core_options_test.cpp +++ b/src/paimon/core/core_options_test.cpp @@ -123,7 +123,7 @@ TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_EQ(SortEngine::LOSER_TREE, core_options.GetSortEngine()); ASSERT_FALSE(core_options.IgnoreDelete()); ASSERT_FALSE(core_options.WriteOnly()); - ASSERT_FALSE(core_options.BucketAppendOrdered()); + ASSERT_TRUE(core_options.BucketAppendOrdered()); ASSERT_EQ(CoreOptions::SequenceNumberInitMode::SCAN, core_options.WriteSequenceNumberInitMode()); ASSERT_EQ(5, core_options.GetCompactionMinFileNum()); @@ -320,7 +320,7 @@ TEST(CoreOptionsTest, TestFromMap) { {Options::GLOBAL_INDEX_EXTERNAL_PATH, "FILE:///tmp/global_index/"}, {Options::SCAN_TAG_NAME, "test-tag"}, {Options::WRITE_ONLY, "true"}, - {Options::BUCKET_APPEND_ORDERED, "true"}, + {Options::BUCKET_APPEND_ORDERED, "false"}, {Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, "snapshot"}, {Options::COMPACTION_MIN_FILE_NUM, "10"}, {Options::COMPACTION_FORCE_REWRITE_ALL_FILES, "true"}, @@ -501,7 +501,7 @@ TEST(CoreOptionsTest, TestFromMap) { ASSERT_EQ(375809637, core_options.GetCompactionFileSize(/*has_primary_key=*/true)); ASSERT_EQ(375809637, core_options.GetCompactionFileSize(/*has_primary_key=*/false)); ASSERT_TRUE(core_options.WriteOnly()); - ASSERT_TRUE(core_options.BucketAppendOrdered()); + ASSERT_FALSE(core_options.BucketAppendOrdered()); ASSERT_EQ(CoreOptions::SequenceNumberInitMode::SNAPSHOT, core_options.WriteSequenceNumberInitMode()); ASSERT_EQ(10, core_options.GetCompactionMinFileNum()); diff --git a/src/paimon/core/operation/abstract_file_store_write.h b/src/paimon/core/operation/abstract_file_store_write.h index 8ed3d5266..008d83ce1 100644 --- a/src/paimon/core/operation/abstract_file_store_write.h +++ b/src/paimon/core/operation/abstract_file_store_write.h @@ -152,6 +152,8 @@ class AbstractFileStoreWrite : public FileStoreWrite { const RealtimeOffsetMap& committed_offsets, const RealtimeDataFileMap& committed_data_files); + Result> GetWriter(const BinaryRow& partition, int32_t bucket); + Result> ScanExistingFileMetas(const BinaryRow& partition, int32_t bucket) const; int32_t GetDefaultBucketNum() const; @@ -184,7 +186,6 @@ class AbstractFileStoreWrite : public FileStoreWrite { void InvalidateRealtimeContext(); Status PoisonRealtimeWrite(const Status& cause); - Result> GetWriter(const BinaryRow& partition, int32_t bucket); Result> PrepareRealtimeCommit(); private: diff --git a/src/paimon/core/operation/append_only_file_store_write.cpp b/src/paimon/core/operation/append_only_file_store_write.cpp index 5887ddd0b..295a9c177 100644 --- a/src/paimon/core/operation/append_only_file_store_write.cpp +++ b/src/paimon/core/operation/append_only_file_store_write.cpp @@ -33,6 +33,7 @@ #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/options_utils.h" +#include "paimon/common/utils/path_util.h" #include "paimon/core/append/append_only_writer.h" #include "paimon/core/append/bucketed_append_compact_manager.h" #include "paimon/core/compact/noop_compact_manager.h" @@ -125,6 +126,68 @@ AppendOnlyFileStoreWrite::AppendOnlyFileStoreWrite( AppendOnlyFileStoreWrite::~AppendOnlyFileStoreWrite() = default; +Status AppendOnlyFileStoreWrite::Write(std::unique_ptr&& batch) { + if (custom_compaction_) { + return Status::Invalid("cannot mix custom compaction with Write or Compact"); + } + regular_write_ = true; + return AbstractFileStoreWrite::Write(std::move(batch)); +} + +Status AppendOnlyFileStoreWrite::Compact(const std::map& partition, + int32_t bucket, bool full_compaction) { + if (custom_compaction_) { + return Status::Invalid("cannot mix custom compaction with Write or Compact"); + } + regular_write_ = true; + return AbstractFileStoreWrite::Compact(partition, bucket, full_compaction); +} + +Status AppendOnlyFileStoreWrite::CustomizeCompact(const CompactRequest& request) { + if (realtime_context_ || options_.RealtimeEnabled() || + !options_.GetRealtimeDeduplicateKeyFields().empty()) { + return Status::NotImplemented("custom compaction does not support real-time mode"); + } + if (options_.GetBucket() <= 0 || options_.DataEvolutionEnabled()) { + return Status::NotImplemented( + "custom compaction only supports fixed-bucket append tables without data evolution"); + } + if (options_.BucketAppendOrdered()) { + return Status::Invalid("custom compaction requires bucket-append-ordered=false"); + } + if (options_.WriteOnly()) { + return Status::Invalid("custom compaction requires write-only=false"); + } + if (write_cols_) { + return Status::Invalid("custom compaction requires the full write schema"); + } + if (regular_write_) { + return Status::Invalid("cannot mix custom compaction with Write or Compact"); + } + if (request.bucket < 0 || request.bucket >= options_.GetBucket()) { + return Status::Invalid("custom compaction bucket is out of range"); + } + if (request.file_names.empty()) { + return Status::Invalid("custom compaction requires at least one input file"); + } + if (request.partition.size() != table_schema_->PartitionKeys().size()) { + return Status::Invalid("custom compaction requires exactly the table partition keys"); + } + for (const std::string& name : request.file_names) { + PAIMON_RETURN_NOT_OK(PathUtil::CheckSinglePathComponent("data file", name)); + } + PAIMON_ASSIGN_OR_RAISE(BinaryRow partition, + file_store_path_factory_->ToBinaryRow(request.partition)); + custom_compaction_ = true; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr writer, + GetWriter(partition, request.bucket)); + auto append_writer = std::dynamic_pointer_cast(writer); + if (!append_writer) { + return Status::Invalid("custom compaction requires an append-only writer"); + } + return append_writer->CustomizeCompact(request.file_names); +} + Status AppendOnlyFileStoreWrite::RefreshCommittedSnapshot(int64_t snapshot_id) { if (!realtime_context_) { return Status::Invalid("refresh committed snapshot requires a real-time writer"); diff --git a/src/paimon/core/operation/append_only_file_store_write.h b/src/paimon/core/operation/append_only_file_store_write.h index 9455594b8..66ea1cf1a 100644 --- a/src/paimon/core/operation/append_only_file_store_write.h +++ b/src/paimon/core/operation/append_only_file_store_write.h @@ -18,7 +18,9 @@ #pragma once +#include #include +#include #include #include #include @@ -90,6 +92,11 @@ class AppendOnlyFileStoreWrite : public AbstractFileStoreWrite { Status RefreshCommittedSnapshot(int64_t snapshot_id) override; + Status Write(std::unique_ptr&& batch) override; + Status Compact(const std::map& partition, int32_t bucket, + bool full_compaction) override; + Status CustomizeCompact(const CompactRequest& request) override; + /// Rewrites the given files into new compacted files. /// /// @param partition The partition of the files. @@ -161,6 +168,8 @@ class AppendOnlyFileStoreWrite : public AbstractFileStoreWrite { const std::shared_ptr& plan_factory) const; std::optional> write_cols_; + std::atomic custom_compaction_{false}; + std::atomic regular_write_{false}; std::shared_ptr realtime_context_; std::unique_ptr logger_; }; diff --git a/src/paimon/core/operation/append_only_file_store_write_test.cpp b/src/paimon/core/operation/append_only_file_store_write_test.cpp index bfeaefced..879e32fe9 100644 --- a/src/paimon/core/operation/append_only_file_store_write_test.cpp +++ b/src/paimon/core/operation/append_only_file_store_write_test.cpp @@ -19,6 +19,7 @@ #include "paimon/core/operation/append_only_file_store_write.h" #include +#include #include #include #include @@ -268,6 +269,180 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestWriteWithInvalidBatch) { } } +TEST_F(AppendOnlyFileStoreWriteTest, TestCustomizeCompactValidationAndRestoredSnapshot) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto schema = + arrow::schema({arrow::field("id", arrow::int32()), arrow::field("name", arrow::utf8())}); + std::map options = {{Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "id"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::COMPACTION_MIN_FILE_NUM, "100"}}; + CreateTable(dir->Str(), schema, options); + const std::string path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + auto first = WriteAndPrepare(path, schema, options, R"([[1, "a"], [2, "b"]])", 0); + Commit(path, options, first); + WriteContextBuilder second_builder(path, commit_user_); + ASSERT_OK_AND_ASSIGN(auto second_context, second_builder.SetOptions(options).Finish()); + ASSERT_OK_AND_ASSIGN(auto second_writer, FileStoreWrite::Create(std::move(second_context))); + auto second_batch = MakeBatch(schema, R"([[3, "c"]])"); + second_batch->SetBucket(1); + ASSERT_OK(second_writer->Write(std::move(second_batch))); + ASSERT_OK_AND_ASSIGN(auto second, second_writer->PrepareCommit(false, 1)); + ASSERT_OK(second_writer->Close()); + Commit(path, options, second); + const std::string first_name = OnlyNewFile(first)->file_name; + const std::string second_name = OnlyNewFile(second)->file_name; + auto new_writer = [&](const std::map& opts, + const std::vector& columns = {}) + -> Result> { + WriteContextBuilder builder(path, "compactor"); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, + builder.SetOptions(opts).WithWriteSchema(columns).Finish()); + return FileStoreWrite::Create(std::move(context)); + }; + CompactRequest request{{}, 0, {first_name}}; + ASSERT_OK_AND_ASSIGN(auto ordered, new_writer(options)); + ASSERT_NOK_WITH_MSG(ordered->CustomizeCompact(request), "bucket-append-ordered=false"); + ASSERT_OK(ordered->Close()); + options[Options::BUCKET_APPEND_ORDERED] = "false"; + auto write_only_options = options; + write_only_options[Options::WRITE_ONLY] = "true"; + ASSERT_OK_AND_ASSIGN(auto write_only, new_writer(write_only_options)); + ASSERT_NOK_WITH_MSG(write_only->CustomizeCompact(request), "write-only=false"); + ASSERT_OK(write_only->Close()); + ASSERT_OK_AND_ASSIGN(auto partial, new_writer(options, {"id"})); + ASSERT_NOK_WITH_MSG(partial->CustomizeCompact(request), "full write schema"); + ASSERT_OK(partial->Close()); + auto realtime_options = options; + realtime_options[Options::REALTIME_ENABLED] = "true"; + ASSERT_OK_AND_ASSIGN(auto realtime, new_writer(realtime_options)); + ASSERT_NOK_WITH_MSG(realtime->CustomizeCompact(request), "real-time mode"); + ASSERT_OK(realtime->Close()); + auto unaware_options = options; + unaware_options[Options::BUCKET] = "-1"; + ASSERT_OK_AND_ASSIGN(auto unaware, new_writer(unaware_options)); + ASSERT_NOK_WITH_MSG(unaware->CustomizeCompact(request), "fixed-bucket append"); + ASSERT_OK(unaware->Close()); + ASSERT_OK_AND_ASSIGN(auto compactor, new_writer(options)); + for (int32_t bucket : {-1, 2}) { + request.bucket = bucket; + ASSERT_NOK_WITH_MSG(compactor->CustomizeCompact(request), "out of range"); + } + request.bucket = 0; + request.partition = {{"unknown", "value"}}; + ASSERT_NOK_WITH_MSG(compactor->CustomizeCompact(request), "partition keys"); + request.partition.clear(); + request.file_names.clear(); + ASSERT_NOK_WITH_MSG(compactor->CustomizeCompact(request), "at least one"); + request.file_names = {first_name, first_name}; + ASSERT_NOK_WITH_MSG(compactor->CustomizeCompact(request), "distinct file names"); + request.file_names = {"../" + first_name}; + ASSERT_NOK(compactor->CustomizeCompact(request)); + request.file_names = {"missing.parquet"}; + ASSERT_NOK_WITH_MSG(compactor->CustomizeCompact(request), "not active"); + request.file_names = {first_name}; + request.bucket = 1; + ASSERT_NOK_WITH_MSG(compactor->CustomizeCompact(request), "not active"); + request.bucket = 0; + request.file_names = {second_name}; + ASSERT_NOK_WITH_MSG(compactor->CustomizeCompact(request), "not active"); + ASSERT_OK_AND_ASSIGN(auto restored_snapshot, compactor->GetBaseSnapshotId({}, 0)); + ASSERT_TRUE(restored_snapshot); + auto third = WriteAndPrepare(path, schema, options, R"([[4, "d"]])", 2); + Commit(path, options, third); + const std::string third_name = OnlyNewFile(third)->file_name; + request.file_names = {third_name}; + ASSERT_NOK_WITH_MSG(compactor->CustomizeCompact(request), "not active"); + ASSERT_OK_AND_ASSIGN(auto reused_snapshot, compactor->GetBaseSnapshotId({}, 0)); + ASSERT_EQ(restored_snapshot, reused_snapshot); + request.file_names = {first_name}; + ASSERT_OK(compactor->CustomizeCompact(request)); + ASSERT_OK(compactor->CustomizeCompact({{}, 1, {second_name}})); + ASSERT_OK_AND_ASSIGN(auto messages, compactor->PrepareCommit()); + ASSERT_EQ(2, messages.size()); + std::shared_ptr impl; + for (const auto& message : messages) { + auto candidate = std::dynamic_pointer_cast(message); + ASSERT_TRUE(candidate); + if (candidate->Bucket() == 0) { + impl = candidate; + } + } + ASSERT_TRUE(impl); + ASSERT_EQ(1, impl->GetCompactIncrement().CompactBefore().size()); + ASSERT_EQ(first_name, impl->GetCompactIncrement().CompactBefore()[0]->file_name); + ASSERT_EQ(1, impl->GetCompactIncrement().CompactAfter().size()); + ASSERT_EQ(2, impl->GetCompactIncrement().CompactAfter()[0]->row_count); + ASSERT_NOK_WITH_MSG(compactor->PrepareCommit(), "one-time committing"); + ASSERT_NOK_WITH_MSG(compactor->Write(MakeBatch(schema, R"([[4, "d"]])")), "cannot mix"); + ASSERT_NOK_WITH_MSG(compactor->Compact({}, 0, true), "cannot mix"); + ASSERT_OK(compactor->Close()); + ASSERT_EQ( + 2, + ReadDataFileArray(path, impl->GetCompactIncrement().CompactAfter()[0], options)->length()); + CommitContextBuilder commit_builder(path, "compactor"); + ASSERT_OK_AND_ASSIGN(auto commit_context, commit_builder.SetOptions(options).Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK(commit->Abort(messages)); + auto fs = std::make_shared(); + ASSERT_OK_AND_ASSIGN( + bool exists, + fs->Exists(PathUtil::JoinPath( + path, "bucket-0/" + impl->GetCompactIncrement().CompactAfter()[0]->file_name))); + ASSERT_FALSE(exists); + ASSERT_OK_AND_ASSIGN(auto mixed, new_writer(options)); + ASSERT_OK(mixed->Write(MakeBatch(schema, R"([[4, "d"]])"))); + ASSERT_NOK_WITH_MSG(mixed->CustomizeCompact(request), "cannot mix"); + ASSERT_OK(mixed->Close()); +} + +TEST_F(AppendOnlyFileStoreWriteTest, TestUnorderedWriterRestore) { + struct TestCase { + bool ordered; + bool write_only; + bool ignore_previous; + int64_t expected_seq; + }; + const std::vector test_cases = { + {/*ordered=*/false, /*write_only=*/false, /*ignore_previous=*/false, /*expected_seq=*/2}, + {/*ordered=*/false, /*write_only=*/false, /*ignore_previous=*/true, /*expected_seq=*/2}, + {/*ordered=*/false, /*write_only=*/true, /*ignore_previous=*/false, /*expected_seq=*/0}, + {/*ordered=*/false, /*write_only=*/true, /*ignore_previous=*/true, /*expected_seq=*/0}, + {/*ordered=*/true, /*write_only=*/false, /*ignore_previous=*/false, /*expected_seq=*/2}, + {/*ordered=*/true, /*write_only=*/false, /*ignore_previous=*/true, /*expected_seq=*/0}, + {/*ordered=*/true, /*write_only=*/true, /*ignore_previous=*/false, /*expected_seq=*/2}, + {/*ordered=*/true, /*write_only=*/true, /*ignore_previous=*/true, /*expected_seq=*/0}, + }; + auto schema = arrow::schema({arrow::field("id", arrow::int32())}); + for (const auto& test_case : test_cases) { + SCOPED_TRACE(::testing::Message() + << "ordered=" << test_case.ordered << ", write_only=" << test_case.write_only + << ", ignore_previous=" << test_case.ignore_previous + << ", expected_seq=" << test_case.expected_seq); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::map options = { + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::BUCKET_APPEND_ORDERED, test_case.ordered ? "true" : "false"}, + {Options::WRITE_ONLY, test_case.write_only ? "true" : "false"}}; + CreateTable(dir->Str(), schema, options); + const std::string path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + Commit(path, options, WriteAndPrepare(path, schema, options, "[[1], [2]]", 0)); + WriteContextBuilder builder(path, commit_user_); + ASSERT_OK_AND_ASSIGN(auto context, builder.SetOptions(options) + .WithIgnorePreviousFiles(test_case.ignore_previous) + .Finish()); + ASSERT_OK_AND_ASSIGN(auto writer, FileStoreWrite::Create(std::move(context))); + ASSERT_OK(writer->Write(MakeBatch(schema, "[[3]]"))); + ASSERT_OK_AND_ASSIGN(auto messages, writer->PrepareCommit()); + ASSERT_EQ(test_case.expected_seq, OnlyNewFile(messages)->min_sequence_number); + ASSERT_OK(writer->Close()); + } +} + TEST_F(AppendOnlyFileStoreWriteTest, TestRealtimeWriteTracksExternalOffsetRange) { std::map options = { {"file.format", "parquet"}, @@ -307,6 +482,8 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestRealtimeWriteTracksExternalOffsetRange) ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_NOK_WITH_MSG(file_store_write->CustomizeCompact({{}, 0, {"unused.parquet"}}), + "real-time mode"); ASSERT_OK(file_store_write->Write(MakeBatch(realtime_schema, R"([ [10, 1, "a"], [20, 2, "b"] diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp index c1b889473..50690b1c7 100644 --- a/src/paimon/core/operation/file_store_write.cpp +++ b/src/paimon/core/operation/file_store_write.cpp @@ -20,6 +20,10 @@ namespace paimon { +Status FileStoreWrite::CustomizeCompact(const CompactRequest&) { + return Status::NotImplemented("custom compaction only supports fixed-bucket append tables"); +} + Result> FileStoreWrite::PrepareCommitWithProgress(int64_t) { return Status::Invalid("prepare commit with progress requires a real-time writer"); } diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp index 7602a2763..f75d9368e 100644 --- a/src/paimon/core/operation/key_value_file_store_write_test.cpp +++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp @@ -345,6 +345,8 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriteWithInvalidBatch) { ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); ASSERT_NOK_WITH_MSG(file_store_write->Write(nullptr), "batch is null pointer"); + ASSERT_NOK_WITH_MSG(file_store_write->CustomizeCompact({{}, 0, {"unused.parquet"}}), + "only supports fixed-bucket append tables"); } { arrow::Schema typed_schema(fields); diff --git a/src/paimon/core/table/append_only_file_store_table.cpp b/src/paimon/core/table/append_only_file_store_table.cpp index 7be44d2ed..b8e3906f6 100644 --- a/src/paimon/core/table/append_only_file_store_table.cpp +++ b/src/paimon/core/table/append_only_file_store_table.cpp @@ -76,8 +76,15 @@ Result> AppendOnlyFileStoreTable::CreateWrite( } else if (options.GetBucket() <= 0) { return Status::Invalid( fmt::format("not support bucket {} in append table", options.GetBucket())); + } else if (!options.BucketAppendOrdered()) { + // Unordered write-only writers do not need historical sequence numbers. Compactors must + // restore existing files even if the write context requested ignoring them. + ignore_previous_files = options.WriteOnly(); } if (ctx.GetRealtimeContext()) { + if (!options.BucketAppendOrdered()) { + return Status::Invalid("real-time append write requires bucket-append-ordered=true"); + } if (!ctx.GetWriteSchema().empty()) { return Status::Invalid("real-time append write does not support a custom schema"); } diff --git a/test/inte/append_compaction_inte_test.cpp b/test/inte/append_compaction_inte_test.cpp index ae5dcf086..13c4753f9 100644 --- a/test/inte/append_compaction_inte_test.cpp +++ b/test/inte/append_compaction_inte_test.cpp @@ -16,6 +16,7 @@ * limitations under the License. */ +#include #include #include #include @@ -206,6 +207,172 @@ std::vector GetTestValuesForAppendCompactionInteTest() { INSTANTIATE_TEST_SUITE_P(FileFormat, AppendCompactionInteTest, ::testing::ValuesIn(GetTestValuesForAppendCompactionInteTest())); +TEST_P(AppendCompactionInteTest, TestCustomizeCompactFilesWithGap) { + for (bool with_dv : {false, true}) { + SCOPED_TRACE(with_dv); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto schema = arrow::schema( + {arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::int32()), + arrow::field("f2", arrow::int32()), arrow::field("f3", arrow::float64())}); + std::map options = { + {Options::FILE_FORMAT, GetParam()}, + {Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "f2"}, + {Options::BUCKET_APPEND_ORDERED, "false"}, + {Options::COMPACTION_MIN_FILE_NUM, "100"}, + {Options::DELETION_VECTORS_ENABLED, with_dv ? "true" : "false"}}; + ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(dir->Str(), schema, {"f1"}, {}, + options, /*is_streaming_mode=*/true)); + ASSERT_OK_AND_ASSIGN(auto table_schema, helper->LatestSchema()); + ASSERT_TRUE(table_schema); + auto gen = std::make_shared(*table_schema, pool_); + int64_t identifier = 0; + PrepareSimpleAppendData(gen, with_dv, helper.get(), &identifier); + ASSERT_OK(helper->write_->Close()); + ASSERT_OK_AND_ASSIGN(auto compactor, + TestHelper::Create(helper->table_path_, options, true)); + auto* write = dynamic_cast(compactor->write_.get()); + ASSERT_NE(nullptr, write); + auto partition = BinaryRowGenerator::GenerateRow({10}, pool_.get()); + ASSERT_OK_AND_ASSIGN(auto restored, write->ScanExistingFileMetas(partition, 1)); + auto files = restored->DataFiles(); + std::sort(files.begin(), files.end(), BucketedAppendCompactManager::FileComparator(true)); + ASSERT_EQ(3, files.size()); + // Reverse the inputs and leave the middle file untouched. Neither order nor adjacency + // of the supplied names determines eligibility. + CompactRequest request{{{"f1", "10"}}, 1, {files[2]->file_name, files[0]->file_name}}; + ASSERT_OK(compactor->write_->CustomizeCompact(request)); + ASSERT_OK_AND_ASSIGN(auto messages, compactor->write_->PrepareCommit(true, identifier)); + ASSERT_EQ(1, messages.size()); + auto message = messages[0]; + auto impl = std::dynamic_pointer_cast(message); + ASSERT_TRUE(impl); + ASSERT_TRUE(impl->GetNewFilesIncrement().IsEmpty()); + const auto& increment = impl->GetCompactIncrement(); + ASSERT_EQ(2, increment.CompactBefore().size()); + ASSERT_EQ(files[0]->file_name, increment.CompactBefore()[0]->file_name); + ASSERT_EQ(files[2]->file_name, increment.CompactBefore()[1]->file_name); + ASSERT_EQ(1, increment.CompactAfter().size()); + ASSERT_EQ(with_dv ? 2 : 4, increment.CompactAfter()[0]->row_count); + ASSERT_EQ(with_dv ? 1 : 0, increment.NewIndexFiles().size()); + ASSERT_OK_AND_ASSIGN(auto pending, compactor->write_->PrepareCommit(true, identifier)); + for (const auto& pending_message : pending) { + auto pending_impl = std::dynamic_pointer_cast(pending_message); + ASSERT_TRUE(pending_impl); + ASSERT_TRUE(pending_impl->IsEmpty()); + } + // Closing the executor must leave the returned files available for publication. + ASSERT_OK(compactor->write_->Close()); + ASSERT_OK_AND_ASSIGN(std::string bytes, CommitMessage::Serialize(message, pool_)); + ASSERT_OK_AND_ASSIGN( + message, CommitMessage::Deserialize(CommitMessage::CurrentVersion(), bytes.data(), + bytes.size(), pool_)); + ASSERT_OK(compactor->commit_->Commit({message}, identifier++)); + ASSERT_OK_AND_ASSIGN(auto active, write->ScanExistingFileMetas(partition, 1)); + ASSERT_EQ(2, active->DataFiles().size()); + if (with_dv) { + ASSERT_OK_AND_ASSIGN(auto dvs, write->dv_maintainer_factory_->Create( + partition, 1, active->DeleteVectorsIndex())); + ASSERT_TRUE(dvs->DeletionVectorOf(files[1]->file_name)); + ASSERT_FALSE(dvs->DeletionVectorOf(files[0]->file_name)); + ASSERT_FALSE(dvs->DeletionVectorOf(files[2]->file_name)); + } + ASSERT_OK_AND_ASSIGN(auto splits, helper->NewScan(StartupMode::LatestFull(), std::nullopt)); + auto fields = schema->fields(); + fields.insert(fields.begin(), arrow::field("_VALUE_KIND", arrow::int8())); + const std::string expected = with_dv ? R"([ + [0, "Emily", 10, 0, 13.1], [0, "Tony", 10, 0, 14.1], + [0, "Bob", 10, 0, 12.1], [0, "Alex", 10, 0, 16.1] + ])" + : R"([ + [0, "Bob", 10, 0, 12.1], [0, "Emily", 10, 0, 13.1], + [0, "Tony", 10, 0, 14.1], [0, "David", 10, 0, 17.1], + [0, "Emily", 10, 0, 15.1], [0, "Bob", 10, 0, 12.1], + [0, "Alex", 10, 0, 16.1] + ])"; + bool checked = false; + for (const auto& split : splits) { + auto data_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(data_split); + if (data_split->Partition() == partition && data_split->Bucket() == 1) { + ASSERT_OK_AND_ASSIGN(bool equal, helper->ReadAndCheckResult(arrow::struct_(fields), + {split}, expected)); + ASSERT_TRUE(equal); + checked = true; + } + } + ASSERT_TRUE(checked); + } +} + +TEST_P(AppendCompactionInteTest, TestCustomizeCompactSingleFullyDeletedFile) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto schema = + arrow::schema({arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::int32()), + arrow::field("f2", arrow::int32()), arrow::field("f3", arrow::float64())}); + std::map options = {{Options::FILE_FORMAT, GetParam()}, + {Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "f2"}, + {Options::BUCKET_APPEND_ORDERED, "false"}, + {Options::DELETION_VECTORS_ENABLED, "true"}}; + ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(dir->Str(), schema, {"f1"}, {}, options, + /*is_streaming_mode=*/true)); + ASSERT_OK_AND_ASSIGN(auto table_schema, helper->LatestSchema()); + ASSERT_TRUE(table_schema); + auto gen = std::make_shared(*table_schema, pool_); + int64_t identifier = 0; + PrepareSimpleAppendData(gen, true, helper.get(), &identifier); + ASSERT_OK(helper->write_->Close()); + ASSERT_OK_AND_ASSIGN(auto compactor, TestHelper::Create(helper->table_path_, options, true)); + auto* write = dynamic_cast(compactor->write_.get()); + ASSERT_NE(nullptr, write); + auto partition = BinaryRowGenerator::GenerateRow({10}, pool_.get()); + ASSERT_OK_AND_ASSIGN(auto restored, write->ScanExistingFileMetas(partition, 1)); + auto files = restored->DataFiles(); + std::sort(files.begin(), files.end(), BucketedAppendCompactManager::FileComparator(true)); + ASSERT_EQ(3, files.size()); + ASSERT_EQ(1, files.back()->row_count); + ASSERT_OK(write->CustomizeCompact({{{"f1", "10"}}, 1, {files.back()->file_name}})); + ASSERT_OK_AND_ASSIGN(auto messages, compactor->write_->PrepareCommit(true, identifier)); + ASSERT_EQ(1, messages.size()); + auto message = messages[0]; + auto impl = std::dynamic_pointer_cast(message); + ASSERT_TRUE(impl); + ASSERT_EQ(1, impl->GetCompactIncrement().CompactBefore().size()); + ASSERT_TRUE(impl->GetCompactIncrement().CompactAfter().empty()); + ASSERT_OK(compactor->commit_->Commit({message}, identifier++)); + ASSERT_OK_AND_ASSIGN(auto active, write->ScanExistingFileMetas(partition, 1)); + ASSERT_EQ(2, active->DataFiles().size()); + ASSERT_OK_AND_ASSIGN(auto dvs, write->dv_maintainer_factory_->Create( + partition, 1, active->DeleteVectorsIndex())); + ASSERT_TRUE(dvs->DeletionVectorOf(files[0]->file_name)); + ASSERT_TRUE(dvs->DeletionVectorOf(files[1]->file_name)); + ASSERT_FALSE(dvs->DeletionVectorOf(files[2]->file_name)); + ASSERT_OK(compactor->write_->Close()); + + ASSERT_OK_AND_ASSIGN(auto splits, helper->NewScan(StartupMode::LatestFull(), std::nullopt)); + std::vector> target_splits; + for (const auto& split : splits) { + auto data_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(data_split); + if (data_split->Partition() == partition && data_split->Bucket() == 1) { + target_splits.push_back(split); + } + } + ASSERT_FALSE(target_splits.empty()); + auto fields = schema->fields(); + fields.insert(fields.begin(), arrow::field("_VALUE_KIND", arrow::int8())); + ASSERT_OK_AND_ASSIGN(bool equal, + helper->ReadAndCheckResult(arrow::struct_(fields), target_splits, + R"([ + [0, "Emily", 10, 0, 13.1], [0, "Tony", 10, 0, 14.1], + [0, "Bob", 10, 0, 12.1], [0, "Alex", 10, 0, 16.1] + ])")); + ASSERT_TRUE(equal); +} + TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteFullCompaction) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp index 7aceaa6a7..419a63038 100644 --- a/test/inte/realtime_write_inte_test.cpp +++ b/test/inte/realtime_write_inte_test.cpp @@ -1971,6 +1971,7 @@ TEST_P(RealtimeDeduplicateCompactionInteTest, TestCommitTransfersLatestDvToCompa write_builder.SetOptions(options_).WithStreamingMode(true).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr compactor, FileStoreWrite::Create(std::move(write_context))); + ASSERT_NOK_WITH_MSG(compactor->CustomizeCompact({{}, 0, {"unused.parquet"}}), "real-time mode"); ASSERT_OK_AND_ASSIGN(std::unique_ptr forbidden_batch, MakeBatch({{99, "not-written", "p0"}}, /*partitioned=*/false)); ASSERT_NOK_WITH_MSG(compactor->Write(std::move(forbidden_batch)), "only supports compaction"); From e1496249be8a99ec763bbc0d43b9bbaccc969c4d Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Thu, 19 Nov 2026 12:00:26 +0800 Subject: [PATCH 2/2] fix comments --- docs/source/user_guide/compaction.rst | 10 +-- include/paimon/compact_request.h | 41 ---------- include/paimon/file_store_write.h | 7 +- src/paimon/core/append/append_only_writer.cpp | 4 +- src/paimon/core/append/append_only_writer.h | 2 +- .../bucketed_append_compact_manager.cpp | 2 +- .../append/bucketed_append_compact_manager.h | 2 +- .../bucketed_append_compact_manager_test.cpp | 11 ++- .../append_only_file_store_write.cpp | 20 ++--- .../operation/append_only_file_store_write.h | 3 +- .../append_only_file_store_write_test.cpp | 75 ++++++++++--------- .../core/operation/file_store_write.cpp | 3 +- .../key_value_file_store_write_test.cpp | 2 +- test/inte/append_compaction_inte_test.cpp | 10 +-- test/inte/realtime_write_inte_test.cpp | 2 +- 15 files changed, 81 insertions(+), 113 deletions(-) delete mode 100644 include/paimon/compact_request.h diff --git a/docs/source/user_guide/compaction.rst b/docs/source/user_guide/compaction.rst index 58dc64b0b..b7e488eec 100644 --- a/docs/source/user_guide/compaction.rst +++ b/docs/source/user_guide/compaction.rst @@ -104,11 +104,11 @@ respectively. Adjust ``target-file-size`` to move the threshold. Custom Compaction ~~~~~~~~~~~~~~~~~ -``FileStoreWrite::CustomizeCompact`` asynchronously rewrites selected files. -``CompactRequest`` specifies partition values, a bucket, and distinct data file -basenames. As with ``Compact``, each partition-bucket restores the latest snapshot -when its writer is initialized and reuses that state. Files need not be contiguous; -compaction thresholds do not apply. +``FileStoreWrite::CompactFiles(partition, bucket, file_names)`` asynchronously +rewrites selected files, accepting partition values, a bucket, and distinct data +file basenames. As with ``Compact``, each partition-bucket restores the latest +snapshot when its writer is initialized and reuses that state. Files need not be +contiguous; compaction thresholds do not apply. Supports fixed-bucket append tables with or without deletion vectors. Requires ``bucket-append-ordered = false``, ``write-only = false`` and a dedicated writer diff --git a/include/paimon/compact_request.h b/include/paimon/compact_request.h deleted file mode 100644 index a8dfa2c4a..000000000 --- a/include/paimon/compact_request.h +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#pragma once - -#include -#include -#include -#include - -#include "paimon/visibility.h" - -namespace paimon { - -/// Files to rewrite in one partition-bucket of an append-only table. -struct PAIMON_EXPORT CompactRequest { - /// Complete logical partition values; empty for an unpartitioned table. - std::map partition; - int32_t bucket = -1; - /// Nonempty, distinct data file names as recorded in the manifest, without directories. - /// Files need not be contiguous in append sequence order. - std::vector file_names; -}; - -} // namespace paimon diff --git a/include/paimon/file_store_write.h b/include/paimon/file_store_write.h index 48d0881ca..88e7d06c0 100644 --- a/include/paimon/file_store_write.h +++ b/include/paimon/file_store_write.h @@ -26,7 +26,6 @@ #include #include "paimon/commit_message.h" -#include "paimon/compact_request.h" #include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" @@ -94,9 +93,13 @@ class PAIMON_EXPORT FileStoreWrite { /// Supports fixed-bucket append tables with or without deletion vectors. /// Requires bucket-append-ordered=false, write-only=false and the full write schema. /// Use a dedicated writer. Real-time mode and data evolution are unsupported. + /// Accepts complete partition values and distinct data file basenames. + /// Files need not be contiguous. /// Collect results and background errors through PrepareCommit(), as with Compact(). /// Do not update deletion vectors or compact the same partition-bucket until commit or discard. - virtual Status CustomizeCompact(const CompactRequest& request); + /// TODO(xinyu.lxy): Consider extracting this API into a separate interface. + virtual Status CompactFiles(const std::map& partition, int32_t bucket, + const std::vector& file_names); /// Generate a list of commit messages with the latest generated data file meta /// information of the current snapshot. diff --git a/src/paimon/core/append/append_only_writer.cpp b/src/paimon/core/append/append_only_writer.cpp index 711a78b2f..6afda6af0 100644 --- a/src/paimon/core/append/append_only_writer.cpp +++ b/src/paimon/core/append/append_only_writer.cpp @@ -111,13 +111,13 @@ Status AppendOnlyWriter::Write(std::unique_ptr&& batch) { return writer_->Write(batch->GetData()); } -Status AppendOnlyWriter::CustomizeCompact(const std::vector& file_names) { +Status AppendOnlyWriter::CompactFiles(const std::vector& file_names) { auto manager = std::dynamic_pointer_cast(compact_manager_); if (!manager) { return Status::Invalid("custom compaction requires a bucketed append compact manager"); } PAIMON_RETURN_NOT_OK(Sync()); - return manager->CustomizeCompact(file_names); + return manager->CompactFiles(file_names); } Result AppendOnlyWriter::PrepareCommit(bool wait_compaction) { diff --git a/src/paimon/core/append/append_only_writer.h b/src/paimon/core/append/append_only_writer.h index a717ffd06..6f1b29fab 100644 --- a/src/paimon/core/append/append_only_writer.h +++ b/src/paimon/core/append/append_only_writer.h @@ -70,7 +70,7 @@ class AppendOnlyWriter : public BatchWriter { Status Compact(bool full_compaction) override { return Flush(/*wait_for_latest_compaction=*/true, full_compaction); } - Status CustomizeCompact(const std::vector& file_names); + Status CompactFiles(const std::vector& file_names); uint64_t GetMemoryUsage() const override { // used for spill, AppendOnlyWriter do not support spill, so return 0 to avoid triggering diff --git a/src/paimon/core/append/bucketed_append_compact_manager.cpp b/src/paimon/core/append/bucketed_append_compact_manager.cpp index 963e19142..9f95a59a5 100644 --- a/src/paimon/core/append/bucketed_append_compact_manager.cpp +++ b/src/paimon/core/append/bucketed_append_compact_manager.cpp @@ -69,7 +69,7 @@ Status BucketedAppendCompactManager::TriggerCompaction(bool full_compaction) { return Status::OK(); } -Status BucketedAppendCompactManager::CustomizeCompact(const std::vector& file_names) { +Status BucketedAppendCompactManager::CompactFiles(const std::vector& file_names) { if (task_future_.valid()) { return Status::Invalid("a compaction task is still running"); } diff --git a/src/paimon/core/append/bucketed_append_compact_manager.h b/src/paimon/core/append/bucketed_append_compact_manager.h index 3db0f2c24..6e52ecf2b 100644 --- a/src/paimon/core/append/bucketed_append_compact_manager.h +++ b/src/paimon/core/append/bucketed_append_compact_manager.h @@ -90,7 +90,7 @@ class BucketedAppendCompactManager : public CompactFutureManager { Status TriggerCompaction(bool full_compaction) override; - Status CustomizeCompact(const std::vector& file_names); + Status CompactFiles(const std::vector& file_names); bool ShouldWaitForLatestCompaction() const override { return false; diff --git a/src/paimon/core/append/bucketed_append_compact_manager_test.cpp b/src/paimon/core/append/bucketed_append_compact_manager_test.cpp index 104b029bc..76cd71b34 100644 --- a/src/paimon/core/append/bucketed_append_compact_manager_test.cpp +++ b/src/paimon/core/append/bucketed_append_compact_manager_test.cpp @@ -312,7 +312,7 @@ TEST_F(BucketedAppendCompactManagerTest, TestPick) { NewFile(2601, 2610), NewFile(2611, 2620), NewFile(2621, 2630)}); } -TEST_F(BucketedAppendCompactManagerTest, TestCustomizeCompactUsesAsyncResultLifecycle) { +TEST_F(BucketedAppendCompactManagerTest, TestCompactFilesUsesAsyncResultLifecycle) { auto cancellation_controller = std::make_shared(); auto files = GenerateDataFileMeta(); std::vector> selected = {files[0], files[2]}; @@ -341,15 +341,14 @@ TEST_F(BucketedAppendCompactManagerTest, TestCustomizeCompactUsesAsyncResultLife // Unblock the rewrite before an assertion failure joins the submission future. ScopeGuard unblock([&release]() { release.set_value(); }); cancellation_controller->Cancel(); - submission = - std::async(std::launch::async, [&]() { return manager.CustomizeCompact(file_names); }); + submission = std::async(std::launch::async, [&]() { return manager.CompactFiles(file_names); }); ASSERT_EQ(std::future_status::ready, submission.wait_for(std::chrono::seconds(5))); ASSERT_OK(submission.get()); ASSERT_FALSE(cancellation_controller->IsCancelled()); ASSERT_OK_AND_ASSIGN(auto pending, manager.GetCompactionResult(/*blocking=*/false)); ASSERT_FALSE(pending); ASSERT_TRUE(manager.CompactNotCompleted()); - ASSERT_NOK_WITH_MSG(manager.CustomizeCompact(file_names), "still running"); + ASSERT_NOK_WITH_MSG(manager.CompactFiles(file_names), "still running"); release.set_value(); unblock.Release(); ASSERT_OK_AND_ASSIGN(auto result, manager.GetCompactionResult(/*blocking=*/true)); @@ -359,11 +358,11 @@ TEST_F(BucketedAppendCompactManagerTest, TestCustomizeCompactUsesAsyncResultLife ExpectVectorsEqual(outputs, result.value()->After()); std::vector> active = {outputs[0], files[1], outputs[1]}; ExpectVectorsEqual(active, manager.AllFiles()); - ASSERT_NOK_WITH_MSG(manager.CustomizeCompact(file_names), "not active"); + ASSERT_NOK_WITH_MSG(manager.CompactFiles(file_names), "not active"); ExpectVectorsEqual(active, manager.AllFiles()); ASSERT_OK(manager.TriggerCompaction(/*full_compaction=*/false)); ASSERT_FALSE(manager.CompactNotCompleted()); - ASSERT_OK(manager.CustomizeCompact({outputs[1]->file_name, outputs[0]->file_name})); + ASSERT_OK(manager.CompactFiles({outputs[1]->file_name, outputs[0]->file_name})); ASSERT_OK_AND_ASSIGN(auto next_result, manager.GetCompactionResult(/*blocking=*/true)); ASSERT_TRUE(next_result); ExpectVectorsEqual(outputs, next_result.value()->Before()); diff --git a/src/paimon/core/operation/append_only_file_store_write.cpp b/src/paimon/core/operation/append_only_file_store_write.cpp index 295a9c177..4d12233d6 100644 --- a/src/paimon/core/operation/append_only_file_store_write.cpp +++ b/src/paimon/core/operation/append_only_file_store_write.cpp @@ -143,7 +143,9 @@ Status AppendOnlyFileStoreWrite::Compact(const std::map& partition, + int32_t bucket, + const std::vector& file_names) { if (realtime_context_ || options_.RealtimeEnabled() || !options_.GetRealtimeDeduplicateKeyFields().empty()) { return Status::NotImplemented("custom compaction does not support real-time mode"); @@ -164,28 +166,26 @@ Status AppendOnlyFileStoreWrite::CustomizeCompact(const CompactRequest& request) if (regular_write_) { return Status::Invalid("cannot mix custom compaction with Write or Compact"); } - if (request.bucket < 0 || request.bucket >= options_.GetBucket()) { + if (bucket < 0 || bucket >= options_.GetBucket()) { return Status::Invalid("custom compaction bucket is out of range"); } - if (request.file_names.empty()) { + if (file_names.empty()) { return Status::Invalid("custom compaction requires at least one input file"); } - if (request.partition.size() != table_schema_->PartitionKeys().size()) { + if (partition.size() != table_schema_->PartitionKeys().size()) { return Status::Invalid("custom compaction requires exactly the table partition keys"); } - for (const std::string& name : request.file_names) { + for (const std::string& name : file_names) { PAIMON_RETURN_NOT_OK(PathUtil::CheckSinglePathComponent("data file", name)); } - PAIMON_ASSIGN_OR_RAISE(BinaryRow partition, - file_store_path_factory_->ToBinaryRow(request.partition)); + PAIMON_ASSIGN_OR_RAISE(BinaryRow part, file_store_path_factory_->ToBinaryRow(partition)); custom_compaction_ = true; - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr writer, - GetWriter(partition, request.bucket)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr writer, GetWriter(part, bucket)); auto append_writer = std::dynamic_pointer_cast(writer); if (!append_writer) { return Status::Invalid("custom compaction requires an append-only writer"); } - return append_writer->CustomizeCompact(request.file_names); + return append_writer->CompactFiles(file_names); } Status AppendOnlyFileStoreWrite::RefreshCommittedSnapshot(int64_t snapshot_id) { diff --git a/src/paimon/core/operation/append_only_file_store_write.h b/src/paimon/core/operation/append_only_file_store_write.h index 66ea1cf1a..e10d82649 100644 --- a/src/paimon/core/operation/append_only_file_store_write.h +++ b/src/paimon/core/operation/append_only_file_store_write.h @@ -95,7 +95,8 @@ class AppendOnlyFileStoreWrite : public AbstractFileStoreWrite { Status Write(std::unique_ptr&& batch) override; Status Compact(const std::map& partition, int32_t bucket, bool full_compaction) override; - Status CustomizeCompact(const CompactRequest& request) override; + Status CompactFiles(const std::map& partition, int32_t bucket, + const std::vector& file_names) override; /// Rewrites the given files into new compacted files. /// diff --git a/src/paimon/core/operation/append_only_file_store_write_test.cpp b/src/paimon/core/operation/append_only_file_store_write_test.cpp index 879e32fe9..7f30b0e26 100644 --- a/src/paimon/core/operation/append_only_file_store_write_test.cpp +++ b/src/paimon/core/operation/append_only_file_store_write_test.cpp @@ -269,7 +269,7 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestWriteWithInvalidBatch) { } } -TEST_F(AppendOnlyFileStoreWriteTest, TestCustomizeCompactValidationAndRestoredSnapshot) { +TEST_F(AppendOnlyFileStoreWriteTest, TestCompactFilesValidationAndRestoredSnapshot) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); auto schema = @@ -301,64 +301,69 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestCustomizeCompactValidationAndRestoredSn builder.SetOptions(opts).WithWriteSchema(columns).Finish()); return FileStoreWrite::Create(std::move(context)); }; - CompactRequest request{{}, 0, {first_name}}; + std::map partition; + int32_t bucket = 0; + std::vector file_names = {first_name}; ASSERT_OK_AND_ASSIGN(auto ordered, new_writer(options)); - ASSERT_NOK_WITH_MSG(ordered->CustomizeCompact(request), "bucket-append-ordered=false"); + ASSERT_NOK_WITH_MSG(ordered->CompactFiles(partition, bucket, file_names), + "bucket-append-ordered=false"); ASSERT_OK(ordered->Close()); options[Options::BUCKET_APPEND_ORDERED] = "false"; auto write_only_options = options; write_only_options[Options::WRITE_ONLY] = "true"; ASSERT_OK_AND_ASSIGN(auto write_only, new_writer(write_only_options)); - ASSERT_NOK_WITH_MSG(write_only->CustomizeCompact(request), "write-only=false"); + ASSERT_NOK_WITH_MSG(write_only->CompactFiles(partition, bucket, file_names), + "write-only=false"); ASSERT_OK(write_only->Close()); ASSERT_OK_AND_ASSIGN(auto partial, new_writer(options, {"id"})); - ASSERT_NOK_WITH_MSG(partial->CustomizeCompact(request), "full write schema"); + ASSERT_NOK_WITH_MSG(partial->CompactFiles(partition, bucket, file_names), "full write schema"); ASSERT_OK(partial->Close()); auto realtime_options = options; realtime_options[Options::REALTIME_ENABLED] = "true"; ASSERT_OK_AND_ASSIGN(auto realtime, new_writer(realtime_options)); - ASSERT_NOK_WITH_MSG(realtime->CustomizeCompact(request), "real-time mode"); + ASSERT_NOK_WITH_MSG(realtime->CompactFiles(partition, bucket, file_names), "real-time mode"); ASSERT_OK(realtime->Close()); auto unaware_options = options; unaware_options[Options::BUCKET] = "-1"; ASSERT_OK_AND_ASSIGN(auto unaware, new_writer(unaware_options)); - ASSERT_NOK_WITH_MSG(unaware->CustomizeCompact(request), "fixed-bucket append"); + ASSERT_NOK_WITH_MSG(unaware->CompactFiles(partition, bucket, file_names), + "fixed-bucket append"); ASSERT_OK(unaware->Close()); ASSERT_OK_AND_ASSIGN(auto compactor, new_writer(options)); - for (int32_t bucket : {-1, 2}) { - request.bucket = bucket; - ASSERT_NOK_WITH_MSG(compactor->CustomizeCompact(request), "out of range"); + for (int32_t invalid_bucket : {-1, 2}) { + ASSERT_NOK_WITH_MSG(compactor->CompactFiles(partition, invalid_bucket, file_names), + "out of range"); } - request.bucket = 0; - request.partition = {{"unknown", "value"}}; - ASSERT_NOK_WITH_MSG(compactor->CustomizeCompact(request), "partition keys"); - request.partition.clear(); - request.file_names.clear(); - ASSERT_NOK_WITH_MSG(compactor->CustomizeCompact(request), "at least one"); - request.file_names = {first_name, first_name}; - ASSERT_NOK_WITH_MSG(compactor->CustomizeCompact(request), "distinct file names"); - request.file_names = {"../" + first_name}; - ASSERT_NOK(compactor->CustomizeCompact(request)); - request.file_names = {"missing.parquet"}; - ASSERT_NOK_WITH_MSG(compactor->CustomizeCompact(request), "not active"); - request.file_names = {first_name}; - request.bucket = 1; - ASSERT_NOK_WITH_MSG(compactor->CustomizeCompact(request), "not active"); - request.bucket = 0; - request.file_names = {second_name}; - ASSERT_NOK_WITH_MSG(compactor->CustomizeCompact(request), "not active"); + partition = {{"unknown", "value"}}; + ASSERT_NOK_WITH_MSG(compactor->CompactFiles(partition, bucket, file_names), "partition keys"); + partition.clear(); + file_names.clear(); + ASSERT_NOK_WITH_MSG(compactor->CompactFiles(partition, bucket, file_names), "at least one"); + file_names = {first_name, first_name}; + ASSERT_NOK_WITH_MSG(compactor->CompactFiles(partition, bucket, file_names), + "distinct file names"); + file_names = {"../" + first_name}; + ASSERT_NOK(compactor->CompactFiles(partition, bucket, file_names)); + file_names = {"missing.parquet"}; + ASSERT_NOK_WITH_MSG(compactor->CompactFiles(partition, bucket, file_names), "not active"); + file_names = {first_name}; + bucket = 1; + ASSERT_NOK_WITH_MSG(compactor->CompactFiles(partition, bucket, file_names), "not active"); + bucket = 0; + file_names = {second_name}; + ASSERT_NOK_WITH_MSG(compactor->CompactFiles(partition, bucket, file_names), "not active"); ASSERT_OK_AND_ASSIGN(auto restored_snapshot, compactor->GetBaseSnapshotId({}, 0)); ASSERT_TRUE(restored_snapshot); auto third = WriteAndPrepare(path, schema, options, R"([[4, "d"]])", 2); Commit(path, options, third); const std::string third_name = OnlyNewFile(third)->file_name; - request.file_names = {third_name}; - ASSERT_NOK_WITH_MSG(compactor->CustomizeCompact(request), "not active"); + file_names = {third_name}; + ASSERT_NOK_WITH_MSG(compactor->CompactFiles(partition, bucket, file_names), "not active"); ASSERT_OK_AND_ASSIGN(auto reused_snapshot, compactor->GetBaseSnapshotId({}, 0)); ASSERT_EQ(restored_snapshot, reused_snapshot); - request.file_names = {first_name}; - ASSERT_OK(compactor->CustomizeCompact(request)); - ASSERT_OK(compactor->CustomizeCompact({{}, 1, {second_name}})); + file_names = {first_name}; + ASSERT_OK(compactor->CompactFiles(partition, bucket, file_names)); + ASSERT_OK(compactor->CompactFiles({}, 1, {second_name})); ASSERT_OK_AND_ASSIGN(auto messages, compactor->PrepareCommit()); ASSERT_EQ(2, messages.size()); std::shared_ptr impl; @@ -393,7 +398,7 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestCustomizeCompactValidationAndRestoredSn ASSERT_FALSE(exists); ASSERT_OK_AND_ASSIGN(auto mixed, new_writer(options)); ASSERT_OK(mixed->Write(MakeBatch(schema, R"([[4, "d"]])"))); - ASSERT_NOK_WITH_MSG(mixed->CustomizeCompact(request), "cannot mix"); + ASSERT_NOK_WITH_MSG(mixed->CompactFiles(partition, bucket, file_names), "cannot mix"); ASSERT_OK(mixed->Close()); } @@ -482,7 +487,7 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestRealtimeWriteTracksExternalOffsetRange) ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); - ASSERT_NOK_WITH_MSG(file_store_write->CustomizeCompact({{}, 0, {"unused.parquet"}}), + ASSERT_NOK_WITH_MSG(file_store_write->CompactFiles({}, 0, {"unused.parquet"}), "real-time mode"); ASSERT_OK(file_store_write->Write(MakeBatch(realtime_schema, R"([ [10, 1, "a"], diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp index 50690b1c7..b1d93608a 100644 --- a/src/paimon/core/operation/file_store_write.cpp +++ b/src/paimon/core/operation/file_store_write.cpp @@ -20,7 +20,8 @@ namespace paimon { -Status FileStoreWrite::CustomizeCompact(const CompactRequest&) { +Status FileStoreWrite::CompactFiles(const std::map&, int32_t, + const std::vector&) { return Status::NotImplemented("custom compaction only supports fixed-bucket append tables"); } diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp index 117ec484b..089575b3e 100644 --- a/src/paimon/core/operation/key_value_file_store_write_test.cpp +++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp @@ -349,7 +349,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriteWithInvalidBatch) { ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); ASSERT_NOK_WITH_MSG(file_store_write->Write(nullptr), "batch is null pointer"); - ASSERT_NOK_WITH_MSG(file_store_write->CustomizeCompact({{}, 0, {"unused.parquet"}}), + ASSERT_NOK_WITH_MSG(file_store_write->CompactFiles({}, 0, {"unused.parquet"}), "only supports fixed-bucket append tables"); } { diff --git a/test/inte/append_compaction_inte_test.cpp b/test/inte/append_compaction_inte_test.cpp index 13c4753f9..a6fc5d011 100644 --- a/test/inte/append_compaction_inte_test.cpp +++ b/test/inte/append_compaction_inte_test.cpp @@ -207,7 +207,7 @@ std::vector GetTestValuesForAppendCompactionInteTest() { INSTANTIATE_TEST_SUITE_P(FileFormat, AppendCompactionInteTest, ::testing::ValuesIn(GetTestValuesForAppendCompactionInteTest())); -TEST_P(AppendCompactionInteTest, TestCustomizeCompactFilesWithGap) { +TEST_P(AppendCompactionInteTest, TestCompactFilesWithGap) { for (bool with_dv : {false, true}) { SCOPED_TRACE(with_dv); auto dir = UniqueTestDirectory::Create(); @@ -241,8 +241,8 @@ TEST_P(AppendCompactionInteTest, TestCustomizeCompactFilesWithGap) { ASSERT_EQ(3, files.size()); // Reverse the inputs and leave the middle file untouched. Neither order nor adjacency // of the supplied names determines eligibility. - CompactRequest request{{{"f1", "10"}}, 1, {files[2]->file_name, files[0]->file_name}}; - ASSERT_OK(compactor->write_->CustomizeCompact(request)); + ASSERT_OK(compactor->write_->CompactFiles({{"f1", "10"}}, 1, + {files[2]->file_name, files[0]->file_name})); ASSERT_OK_AND_ASSIGN(auto messages, compactor->write_->PrepareCommit(true, identifier)); ASSERT_EQ(1, messages.size()); auto message = messages[0]; @@ -306,7 +306,7 @@ TEST_P(AppendCompactionInteTest, TestCustomizeCompactFilesWithGap) { } } -TEST_P(AppendCompactionInteTest, TestCustomizeCompactSingleFullyDeletedFile) { +TEST_P(AppendCompactionInteTest, TestCompactFilesSingleFullyDeletedFile) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); auto schema = @@ -334,7 +334,7 @@ TEST_P(AppendCompactionInteTest, TestCustomizeCompactSingleFullyDeletedFile) { std::sort(files.begin(), files.end(), BucketedAppendCompactManager::FileComparator(true)); ASSERT_EQ(3, files.size()); ASSERT_EQ(1, files.back()->row_count); - ASSERT_OK(write->CustomizeCompact({{{"f1", "10"}}, 1, {files.back()->file_name}})); + ASSERT_OK(write->CompactFiles({{"f1", "10"}}, 1, {files.back()->file_name})); ASSERT_OK_AND_ASSIGN(auto messages, compactor->write_->PrepareCommit(true, identifier)); ASSERT_EQ(1, messages.size()); auto message = messages[0]; diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp index 8e7ef28ea..abf948797 100644 --- a/test/inte/realtime_write_inte_test.cpp +++ b/test/inte/realtime_write_inte_test.cpp @@ -1997,7 +1997,7 @@ TEST_P(RealtimeDeduplicateCompactionInteTest, TestCommitTransfersLatestDvToCompa write_builder.SetOptions(options_).WithStreamingMode(true).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr compactor, FileStoreWrite::Create(std::move(write_context))); - ASSERT_NOK_WITH_MSG(compactor->CustomizeCompact({{}, 0, {"unused.parquet"}}), "real-time mode"); + ASSERT_NOK_WITH_MSG(compactor->CompactFiles({}, 0, {"unused.parquet"}), "real-time mode"); ASSERT_OK_AND_ASSIGN(std::unique_ptr forbidden_batch, MakeBatch({{99, "not-written", "p0"}}, /*partitioned=*/false)); ASSERT_NOK_WITH_MSG(compactor->Write(std::move(forbidden_batch)), "only supports compaction");