diff --git a/docs/source/user_guide/append_only_table.rst b/docs/source/user_guide/append_only_table.rst index 4ed6061b5..07c73a142 100644 --- a/docs/source/user_guide/append_only_table.rst +++ b/docs/source/user_guide/append_only_table.rst @@ -24,6 +24,13 @@ If a table does not have a primary key defined, it is an append table. Ordinary append writes accept ``INSERT`` rows only and do not apply upserts or deletes by key. +Fixed-bucket append tables preserve append order by default +(``bucket-append-ordered = true``). Set it to ``false`` to allow unordered writes +and custom compaction of files that are not contiguous in append order. +In unordered mode, ``write-only = true`` skips restoring historical files and +sequence numbers; with ``write-only = false``, historical files are restored for +compaction even if the write context requests ignoring previous files. + The experimental real-time append deduplicate mode is an exception. With ``realtime.deduplicate-key-fields`` configured, it also accepts ``UPDATE_AFTER``, ``DELETE`` and ``UPDATE_BEFORE``, using deletion vectors to hide superseded diff --git a/docs/source/user_guide/compaction.rst b/docs/source/user_guide/compaction.rst index 92aa01c81..b7e488eec 100644 --- a/docs/source/user_guide/compaction.rst +++ b/docs/source/user_guide/compaction.rst @@ -102,6 +102,25 @@ size, so that a file already close to the target is not compacted twice. primary-key tables, which makes the derived threshold about 179 MiB and 89 MiB respectively. Adjust ``target-file-size`` to move the threshold. +Custom Compaction +~~~~~~~~~~~~~~~~~ +``FileStoreWrite::CompactFiles(partition, bucket, file_names)`` asynchronously +rewrites selected files, accepting partition values, a bucket, and distinct data +file basenames. As with ``Compact``, each partition-bucket restores the latest +snapshot when its writer is initialized and reuses that state. Files need not be +contiguous; compaction thresholds do not apply. + +Supports fixed-bucket append tables with or without deletion vectors. Requires +``bucket-append-ordered = false``, ``write-only = false`` and a dedicated writer +with the full schema. Real-time mode and data evolution are unsupported. + +Use ``PrepareCommit`` to collect results and background errors, following the +usual ``Compact`` waiting rules. Commit or abort the returned messages through +``FileStoreCommit``. + +Do not update deletion vectors or run another compaction for the same +partition-bucket until the custom compaction result is committed or discarded. + Dictionary Passthrough ~~~~~~~~~~~~~~~~~~~~~~ An append-only compaction rewrite copies rows into the new file without diff --git a/include/paimon/defs.h b/include/paimon/defs.h index db94955de..9b96d7220 100644 --- a/include/paimon/defs.h +++ b/include/paimon/defs.h @@ -720,8 +720,8 @@ struct PAIMON_EXPORT Options { /// "write-only" - If set to "true", compactions and snapshot expiration will be skipped. This /// option is used along with dedicated compact jobs. Default value is "false". static const char WRITE_ONLY[]; - /// "bucket-append-ordered" - Whether append writes in fixed bucket mode are ordered. This - /// option is used by commit conflict checks. Default value is "false". + /// "bucket-append-ordered" - Whether append writes in fixed bucket mode preserve append order. + /// Set to "false" to allow unordered writes and custom compaction. Default value is "true". static const char BUCKET_APPEND_ORDERED[]; /// "write.sequence-number-init-mode" - Specify how to initialize the next sequence number for /// primary key table writers. Values can be: "scan", "snapshot". Default value is "scan". diff --git a/include/paimon/file_store_write.h b/include/paimon/file_store_write.h index ae01dd080..88e7d06c0 100644 --- a/include/paimon/file_store_write.h +++ b/include/paimon/file_store_write.h @@ -89,6 +89,18 @@ class PAIMON_EXPORT FileStoreWrite { virtual Status Compact(const std::map& partition, int32_t bucket, bool full_compaction) = 0; + /// Asynchronously compacts requested files from the writer's restored state, as with Compact(). + /// Supports fixed-bucket append tables with or without deletion vectors. + /// Requires bucket-append-ordered=false, write-only=false and the full write schema. + /// Use a dedicated writer. Real-time mode and data evolution are unsupported. + /// Accepts complete partition values and distinct data file basenames. + /// Files need not be contiguous. + /// Collect results and background errors through PrepareCommit(), as with Compact(). + /// Do not update deletion vectors or compact the same partition-bucket until commit or discard. + /// TODO(xinyu.lxy): Consider extracting this API into a separate interface. + virtual Status CompactFiles(const std::map& partition, int32_t bucket, + const std::vector& file_names); + /// Generate a list of commit messages with the latest generated data file meta /// information of the current snapshot. /// diff --git a/src/paimon/core/append/append_only_writer.cpp b/src/paimon/core/append/append_only_writer.cpp index 9b3f32c59..6afda6af0 100644 --- a/src/paimon/core/append/append_only_writer.cpp +++ b/src/paimon/core/append/append_only_writer.cpp @@ -37,6 +37,7 @@ #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/long_counter.h" #include "paimon/common/utils/scope_guard.h" +#include "paimon/core/append/bucketed_append_compact_manager.h" #include "paimon/core/io/append_data_file_writer_factory.h" #include "paimon/core/io/blob_data_file_writer_factory.h" #include "paimon/core/io/compact_increment.h" @@ -110,6 +111,15 @@ Status AppendOnlyWriter::Write(std::unique_ptr&& batch) { return writer_->Write(batch->GetData()); } +Status AppendOnlyWriter::CompactFiles(const std::vector& file_names) { + auto manager = std::dynamic_pointer_cast(compact_manager_); + if (!manager) { + return Status::Invalid("custom compaction requires a bucketed append compact manager"); + } + PAIMON_RETURN_NOT_OK(Sync()); + return manager->CompactFiles(file_names); +} + Result AppendOnlyWriter::PrepareCommit(bool wait_compaction) { PAIMON_RETURN_NOT_OK( Flush(/*wait_for_latest_compaction=*/false, /*forced_full_compaction=*/false)); diff --git a/src/paimon/core/append/append_only_writer.h b/src/paimon/core/append/append_only_writer.h index eb4c2de67..6f1b29fab 100644 --- a/src/paimon/core/append/append_only_writer.h +++ b/src/paimon/core/append/append_only_writer.h @@ -70,6 +70,7 @@ class AppendOnlyWriter : public BatchWriter { Status Compact(bool full_compaction) override { return Flush(/*wait_for_latest_compaction=*/true, full_compaction); } + Status CompactFiles(const std::vector& file_names); uint64_t GetMemoryUsage() const override { // used for spill, AppendOnlyWriter do not support spill, so return 0 to avoid triggering diff --git a/src/paimon/core/append/bucketed_append_compact_manager.cpp b/src/paimon/core/append/bucketed_append_compact_manager.cpp index e1fd343cc..9f95a59a5 100644 --- a/src/paimon/core/append/bucketed_append_compact_manager.cpp +++ b/src/paimon/core/append/bucketed_append_compact_manager.cpp @@ -19,6 +19,7 @@ #include "paimon/core/append/bucketed_append_compact_manager.h" #include +#include #include "paimon/common/executor/future.h" @@ -57,6 +58,9 @@ void BucketedAppendCompactManager::RequestCancelCompaction() { } Status BucketedAppendCompactManager::TriggerCompaction(bool full_compaction) { + if (custom_compaction_) { + return Status::OK(); + } if (full_compaction) { PAIMON_RETURN_NOT_OK(TriggerFullCompaction()); } else { @@ -65,6 +69,42 @@ Status BucketedAppendCompactManager::TriggerCompaction(bool full_compaction) { return Status::OK(); } +Status BucketedAppendCompactManager::CompactFiles(const std::vector& file_names) { + if (task_future_.valid()) { + return Status::Invalid("a compaction task is still running"); + } + custom_compaction_ = true; + if (file_names.empty()) { + return Status::Invalid("custom compaction requires at least one input file"); + } + std::unordered_set names(file_names.begin(), file_names.end()); + if (names.size() != file_names.size()) { + return Status::Invalid("custom compaction requires distinct file names"); + } + auto candidates = to_compact_; + std::vector> selected; + std::vector> retained; + while (!candidates.empty()) { + std::shared_ptr file = candidates.top(); + candidates.pop(); + if (names.erase(file->file_name) > 0) { + selected.push_back(file); + } else { + retained.push_back(file); + } + } + if (!names.empty()) { + return Status::Invalid("custom compaction input file is not active in this writer: ", + *names.begin()); + } + to_compact_.swap(candidates); + for (const auto& file : retained) { + to_compact_.push(file); + } + SubmitCompaction(selected); + return Status::OK(); +} + Status BucketedAppendCompactManager::TriggerFullCompaction() { if (task_future_.valid()) { return Status::Invalid( @@ -100,17 +140,21 @@ void BucketedAppendCompactManager::TriggerCompactionWithBestEffort() { } std::optional>> picked = PickCompactBefore(); if (picked) { - cancellation_controller_->Reset(); - compacting_ = picked.value(); - auto compact_task = std::make_shared( - reporter_, dv_maintainer_, compacting_.value(), rewriter_, materialize_dv_on_commit_); - task_future_ = - Via(executor_.get(), [compact_task]() -> Result> { - return compact_task->Execute(); - }); + SubmitCompaction(picked.value()); } } +void BucketedAppendCompactManager::SubmitCompaction( + const std::vector>& files) { + cancellation_controller_->Reset(); + compacting_ = files; + auto compact_task = std::make_shared(reporter_, dv_maintainer_, files, + rewriter_, materialize_dv_on_commit_); + task_future_ = Via(executor_.get(), [compact_task]() -> Result> { + return compact_task->Execute(); + }); +} + std::optional>> BucketedAppendCompactManager::PickCompactBefore() { if (to_compact_.empty()) { @@ -162,7 +206,11 @@ BucketedAppendCompactManager::GetCompactionResult(bool blocking) { InnerGetCompactionResult(blocking)); if (result) { std::shared_ptr compact_result = result.value(); - if (!compact_result->After().empty()) { + if (custom_compaction_) { + for (const auto& file : compact_result->After()) { + to_compact_.push(file); + } + } else if (!compact_result->After().empty()) { // if the last compacted file is still small, // add it back to the head std::shared_ptr last_file = compact_result->After().back(); diff --git a/src/paimon/core/append/bucketed_append_compact_manager.h b/src/paimon/core/append/bucketed_append_compact_manager.h index 8ad375181..6e52ecf2b 100644 --- a/src/paimon/core/append/bucketed_append_compact_manager.h +++ b/src/paimon/core/append/bucketed_append_compact_manager.h @@ -23,6 +23,7 @@ #include #include #include +#include #include #include "paimon/common/executor/future.h" @@ -89,6 +90,8 @@ class BucketedAppendCompactManager : public CompactFutureManager { Status TriggerCompaction(bool full_compaction) override; + Status CompactFiles(const std::vector& file_names); + bool ShouldWaitForLatestCompaction() const override { return false; } @@ -166,7 +169,7 @@ class BucketedAppendCompactManager : public CompactFutureManager { bool materialize_dv_on_commit_; }; - /// A `CompactTask` impl for append-only table auto-compaction. + /// A `CompactTask` impl for selected append-only files. /// /// This task accepts an already-picked candidate to perform one-time rewrite. And for the /// rest of input files, it is the duty of `AppendOnlyWriter` to invoke the next time @@ -202,6 +205,7 @@ class BucketedAppendCompactManager : public CompactFutureManager { std::optional>> PickCompactBefore(); Status TriggerFullCompaction(); void TriggerCompactionWithBestEffort(); + void SubmitCompaction(const std::vector>& files); std::shared_ptr executor_; std::shared_ptr dv_maintainer_; @@ -217,6 +221,7 @@ class BucketedAppendCompactManager : public CompactFutureManager { // If true, compaction leaves DV file generation to commit using the latest snapshot; // otherwise, compaction generates its DV files before commit. bool materialize_dv_on_commit_; + bool custom_compaction_ = false; }; } // namespace paimon diff --git a/src/paimon/core/append/bucketed_append_compact_manager_test.cpp b/src/paimon/core/append/bucketed_append_compact_manager_test.cpp index 91fa25650..76cd71b34 100644 --- a/src/paimon/core/append/bucketed_append_compact_manager_test.cpp +++ b/src/paimon/core/append/bucketed_append_compact_manager_test.cpp @@ -27,6 +27,7 @@ #include #include "gtest/gtest.h" +#include "paimon/common/utils/scope_guard.h" #include "paimon/core/deletionvectors/bitmap_deletion_vector.h" #include "paimon/core/deletionvectors/bucketed_dv_maintainer.h" #include "paimon/core/deletionvectors/deletion_vectors_index_file.h" @@ -311,6 +312,63 @@ TEST_F(BucketedAppendCompactManagerTest, TestPick) { NewFile(2601, 2610), NewFile(2611, 2620), NewFile(2621, 2630)}); } +TEST_F(BucketedAppendCompactManagerTest, TestCompactFilesUsesAsyncResultLifecycle) { + auto cancellation_controller = std::make_shared(); + auto files = GenerateDataFileMeta(); + std::vector> selected = {files[0], files[2]}; + std::vector file_names = {files[2]->file_name, files[0]->file_name}; + std::vector> outputs = { + std::make_shared(*files[0]), std::make_shared(*files[2])}; + outputs[0]->file_name = "compacted1"; + outputs[0]->file_size = 2048; + outputs[1]->file_name = "compacted3"; + std::vector> rewritten; + std::promise release; + auto released = release.get_future().share(); + auto rewriter = [&rewritten, released, + outputs](const std::vector>& inputs) + -> Result>> { + released.wait(); + rewritten = inputs; + return outputs; + }; + BucketedAppendCompactManager manager( + executor_, files, /*dv_maintainer=*/nullptr, /*min_file_num=*/100, + /*target_file_size=*/1024, /*compaction_file_size=*/700, + /*force_rewrite_all_files=*/false, rewriter, /*reporter=*/nullptr, cancellation_controller, + /*materialize_dv_on_commit=*/false); + std::future submission; + // Unblock the rewrite before an assertion failure joins the submission future. + ScopeGuard unblock([&release]() { release.set_value(); }); + cancellation_controller->Cancel(); + submission = std::async(std::launch::async, [&]() { return manager.CompactFiles(file_names); }); + ASSERT_EQ(std::future_status::ready, submission.wait_for(std::chrono::seconds(5))); + ASSERT_OK(submission.get()); + ASSERT_FALSE(cancellation_controller->IsCancelled()); + ASSERT_OK_AND_ASSIGN(auto pending, manager.GetCompactionResult(/*blocking=*/false)); + ASSERT_FALSE(pending); + ASSERT_TRUE(manager.CompactNotCompleted()); + ASSERT_NOK_WITH_MSG(manager.CompactFiles(file_names), "still running"); + release.set_value(); + unblock.Release(); + ASSERT_OK_AND_ASSIGN(auto result, manager.GetCompactionResult(/*blocking=*/true)); + ASSERT_TRUE(result); + ExpectVectorsEqual(selected, rewritten); + ExpectVectorsEqual(selected, result.value()->Before()); + ExpectVectorsEqual(outputs, result.value()->After()); + std::vector> active = {outputs[0], files[1], outputs[1]}; + ExpectVectorsEqual(active, manager.AllFiles()); + ASSERT_NOK_WITH_MSG(manager.CompactFiles(file_names), "not active"); + ExpectVectorsEqual(active, manager.AllFiles()); + ASSERT_OK(manager.TriggerCompaction(/*full_compaction=*/false)); + ASSERT_FALSE(manager.CompactNotCompleted()); + ASSERT_OK(manager.CompactFiles({outputs[1]->file_name, outputs[0]->file_name})); + ASSERT_OK_AND_ASSIGN(auto next_result, manager.GetCompactionResult(/*blocking=*/true)); + ASSERT_TRUE(next_result); + ExpectVectorsEqual(outputs, next_result.value()->Before()); + ExpectVectorsEqual(active, manager.AllFiles()); +} + TEST_F(BucketedAppendCompactManagerTest, TestCancelCompactionPropagatesToRewriteLoop) { auto cancellation_controller = std::make_shared(); auto exit_signal = std::make_shared>(); diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp index 26f1cb7ee..3a87f6616 100644 --- a/src/paimon/core/core_options.cpp +++ b/src/paimon/core/core_options.cpp @@ -463,7 +463,7 @@ struct CoreOptions::Impl { bool manifest_delete_file_drop_stats = false; bool write_buffer_spillable = true; bool write_only = false; - bool bucket_append_ordered = false; + bool bucket_append_ordered = true; bool deletion_vectors_enabled = false; bool pk_clustering_override = false; bool deletion_vectors_bitmap64 = false; diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp index 2c0b53989..c3b78a696 100644 --- a/src/paimon/core/core_options_test.cpp +++ b/src/paimon/core/core_options_test.cpp @@ -124,7 +124,7 @@ TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_EQ(SortEngine::LOSER_TREE, core_options.GetSortEngine()); ASSERT_FALSE(core_options.IgnoreDelete()); ASSERT_FALSE(core_options.WriteOnly()); - ASSERT_FALSE(core_options.BucketAppendOrdered()); + ASSERT_TRUE(core_options.BucketAppendOrdered()); ASSERT_EQ(CoreOptions::SequenceNumberInitMode::SCAN, core_options.WriteSequenceNumberInitMode()); ASSERT_EQ(5, core_options.GetCompactionMinFileNum()); @@ -322,7 +322,7 @@ TEST(CoreOptionsTest, TestFromMap) { {Options::GLOBAL_INDEX_EXTERNAL_PATH, "FILE:///tmp/global_index/"}, {Options::SCAN_TAG_NAME, "test-tag"}, {Options::WRITE_ONLY, "true"}, - {Options::BUCKET_APPEND_ORDERED, "true"}, + {Options::BUCKET_APPEND_ORDERED, "false"}, {Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, "snapshot"}, {Options::COMPACTION_MIN_FILE_NUM, "10"}, {Options::COMPACTION_FORCE_REWRITE_ALL_FILES, "true"}, @@ -504,7 +504,7 @@ TEST(CoreOptionsTest, TestFromMap) { ASSERT_EQ(375809637, core_options.GetCompactionFileSize(/*has_primary_key=*/true)); ASSERT_EQ(375809637, core_options.GetCompactionFileSize(/*has_primary_key=*/false)); ASSERT_TRUE(core_options.WriteOnly()); - ASSERT_TRUE(core_options.BucketAppendOrdered()); + ASSERT_FALSE(core_options.BucketAppendOrdered()); ASSERT_EQ(CoreOptions::SequenceNumberInitMode::SNAPSHOT, core_options.WriteSequenceNumberInitMode()); ASSERT_EQ(10, core_options.GetCompactionMinFileNum()); diff --git a/src/paimon/core/operation/abstract_file_store_write.h b/src/paimon/core/operation/abstract_file_store_write.h index 8ed3d5266..008d83ce1 100644 --- a/src/paimon/core/operation/abstract_file_store_write.h +++ b/src/paimon/core/operation/abstract_file_store_write.h @@ -152,6 +152,8 @@ class AbstractFileStoreWrite : public FileStoreWrite { const RealtimeOffsetMap& committed_offsets, const RealtimeDataFileMap& committed_data_files); + Result> GetWriter(const BinaryRow& partition, int32_t bucket); + Result> ScanExistingFileMetas(const BinaryRow& partition, int32_t bucket) const; int32_t GetDefaultBucketNum() const; @@ -184,7 +186,6 @@ class AbstractFileStoreWrite : public FileStoreWrite { void InvalidateRealtimeContext(); Status PoisonRealtimeWrite(const Status& cause); - Result> GetWriter(const BinaryRow& partition, int32_t bucket); Result> PrepareRealtimeCommit(); private: diff --git a/src/paimon/core/operation/append_only_file_store_write.cpp b/src/paimon/core/operation/append_only_file_store_write.cpp index 5887ddd0b..4d12233d6 100644 --- a/src/paimon/core/operation/append_only_file_store_write.cpp +++ b/src/paimon/core/operation/append_only_file_store_write.cpp @@ -33,6 +33,7 @@ #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/options_utils.h" +#include "paimon/common/utils/path_util.h" #include "paimon/core/append/append_only_writer.h" #include "paimon/core/append/bucketed_append_compact_manager.h" #include "paimon/core/compact/noop_compact_manager.h" @@ -125,6 +126,68 @@ AppendOnlyFileStoreWrite::AppendOnlyFileStoreWrite( AppendOnlyFileStoreWrite::~AppendOnlyFileStoreWrite() = default; +Status AppendOnlyFileStoreWrite::Write(std::unique_ptr&& batch) { + if (custom_compaction_) { + return Status::Invalid("cannot mix custom compaction with Write or Compact"); + } + regular_write_ = true; + return AbstractFileStoreWrite::Write(std::move(batch)); +} + +Status AppendOnlyFileStoreWrite::Compact(const std::map& partition, + int32_t bucket, bool full_compaction) { + if (custom_compaction_) { + return Status::Invalid("cannot mix custom compaction with Write or Compact"); + } + regular_write_ = true; + return AbstractFileStoreWrite::Compact(partition, bucket, full_compaction); +} + +Status AppendOnlyFileStoreWrite::CompactFiles(const std::map& partition, + int32_t bucket, + const std::vector& file_names) { + if (realtime_context_ || options_.RealtimeEnabled() || + !options_.GetRealtimeDeduplicateKeyFields().empty()) { + return Status::NotImplemented("custom compaction does not support real-time mode"); + } + if (options_.GetBucket() <= 0 || options_.DataEvolutionEnabled()) { + return Status::NotImplemented( + "custom compaction only supports fixed-bucket append tables without data evolution"); + } + if (options_.BucketAppendOrdered()) { + return Status::Invalid("custom compaction requires bucket-append-ordered=false"); + } + if (options_.WriteOnly()) { + return Status::Invalid("custom compaction requires write-only=false"); + } + if (write_cols_) { + return Status::Invalid("custom compaction requires the full write schema"); + } + if (regular_write_) { + return Status::Invalid("cannot mix custom compaction with Write or Compact"); + } + if (bucket < 0 || bucket >= options_.GetBucket()) { + return Status::Invalid("custom compaction bucket is out of range"); + } + if (file_names.empty()) { + return Status::Invalid("custom compaction requires at least one input file"); + } + if (partition.size() != table_schema_->PartitionKeys().size()) { + return Status::Invalid("custom compaction requires exactly the table partition keys"); + } + for (const std::string& name : file_names) { + PAIMON_RETURN_NOT_OK(PathUtil::CheckSinglePathComponent("data file", name)); + } + PAIMON_ASSIGN_OR_RAISE(BinaryRow part, file_store_path_factory_->ToBinaryRow(partition)); + custom_compaction_ = true; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr writer, GetWriter(part, bucket)); + auto append_writer = std::dynamic_pointer_cast(writer); + if (!append_writer) { + return Status::Invalid("custom compaction requires an append-only writer"); + } + return append_writer->CompactFiles(file_names); +} + Status AppendOnlyFileStoreWrite::RefreshCommittedSnapshot(int64_t snapshot_id) { if (!realtime_context_) { return Status::Invalid("refresh committed snapshot requires a real-time writer"); diff --git a/src/paimon/core/operation/append_only_file_store_write.h b/src/paimon/core/operation/append_only_file_store_write.h index 9455594b8..e10d82649 100644 --- a/src/paimon/core/operation/append_only_file_store_write.h +++ b/src/paimon/core/operation/append_only_file_store_write.h @@ -18,7 +18,9 @@ #pragma once +#include #include +#include #include #include #include @@ -90,6 +92,12 @@ class AppendOnlyFileStoreWrite : public AbstractFileStoreWrite { Status RefreshCommittedSnapshot(int64_t snapshot_id) override; + Status Write(std::unique_ptr&& batch) override; + Status Compact(const std::map& partition, int32_t bucket, + bool full_compaction) override; + Status CompactFiles(const std::map& partition, int32_t bucket, + const std::vector& file_names) override; + /// Rewrites the given files into new compacted files. /// /// @param partition The partition of the files. @@ -161,6 +169,8 @@ class AppendOnlyFileStoreWrite : public AbstractFileStoreWrite { const std::shared_ptr& plan_factory) const; std::optional> write_cols_; + std::atomic custom_compaction_{false}; + std::atomic regular_write_{false}; std::shared_ptr realtime_context_; std::unique_ptr logger_; }; diff --git a/src/paimon/core/operation/append_only_file_store_write_test.cpp b/src/paimon/core/operation/append_only_file_store_write_test.cpp index bfeaefced..7f30b0e26 100644 --- a/src/paimon/core/operation/append_only_file_store_write_test.cpp +++ b/src/paimon/core/operation/append_only_file_store_write_test.cpp @@ -19,6 +19,7 @@ #include "paimon/core/operation/append_only_file_store_write.h" #include +#include #include #include #include @@ -268,6 +269,185 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestWriteWithInvalidBatch) { } } +TEST_F(AppendOnlyFileStoreWriteTest, TestCompactFilesValidationAndRestoredSnapshot) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto schema = + arrow::schema({arrow::field("id", arrow::int32()), arrow::field("name", arrow::utf8())}); + std::map options = {{Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "id"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::COMPACTION_MIN_FILE_NUM, "100"}}; + CreateTable(dir->Str(), schema, options); + const std::string path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + auto first = WriteAndPrepare(path, schema, options, R"([[1, "a"], [2, "b"]])", 0); + Commit(path, options, first); + WriteContextBuilder second_builder(path, commit_user_); + ASSERT_OK_AND_ASSIGN(auto second_context, second_builder.SetOptions(options).Finish()); + ASSERT_OK_AND_ASSIGN(auto second_writer, FileStoreWrite::Create(std::move(second_context))); + auto second_batch = MakeBatch(schema, R"([[3, "c"]])"); + second_batch->SetBucket(1); + ASSERT_OK(second_writer->Write(std::move(second_batch))); + ASSERT_OK_AND_ASSIGN(auto second, second_writer->PrepareCommit(false, 1)); + ASSERT_OK(second_writer->Close()); + Commit(path, options, second); + const std::string first_name = OnlyNewFile(first)->file_name; + const std::string second_name = OnlyNewFile(second)->file_name; + auto new_writer = [&](const std::map& opts, + const std::vector& columns = {}) + -> Result> { + WriteContextBuilder builder(path, "compactor"); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, + builder.SetOptions(opts).WithWriteSchema(columns).Finish()); + return FileStoreWrite::Create(std::move(context)); + }; + std::map partition; + int32_t bucket = 0; + std::vector file_names = {first_name}; + ASSERT_OK_AND_ASSIGN(auto ordered, new_writer(options)); + ASSERT_NOK_WITH_MSG(ordered->CompactFiles(partition, bucket, file_names), + "bucket-append-ordered=false"); + ASSERT_OK(ordered->Close()); + options[Options::BUCKET_APPEND_ORDERED] = "false"; + auto write_only_options = options; + write_only_options[Options::WRITE_ONLY] = "true"; + ASSERT_OK_AND_ASSIGN(auto write_only, new_writer(write_only_options)); + ASSERT_NOK_WITH_MSG(write_only->CompactFiles(partition, bucket, file_names), + "write-only=false"); + ASSERT_OK(write_only->Close()); + ASSERT_OK_AND_ASSIGN(auto partial, new_writer(options, {"id"})); + ASSERT_NOK_WITH_MSG(partial->CompactFiles(partition, bucket, file_names), "full write schema"); + ASSERT_OK(partial->Close()); + auto realtime_options = options; + realtime_options[Options::REALTIME_ENABLED] = "true"; + ASSERT_OK_AND_ASSIGN(auto realtime, new_writer(realtime_options)); + ASSERT_NOK_WITH_MSG(realtime->CompactFiles(partition, bucket, file_names), "real-time mode"); + ASSERT_OK(realtime->Close()); + auto unaware_options = options; + unaware_options[Options::BUCKET] = "-1"; + ASSERT_OK_AND_ASSIGN(auto unaware, new_writer(unaware_options)); + ASSERT_NOK_WITH_MSG(unaware->CompactFiles(partition, bucket, file_names), + "fixed-bucket append"); + ASSERT_OK(unaware->Close()); + ASSERT_OK_AND_ASSIGN(auto compactor, new_writer(options)); + for (int32_t invalid_bucket : {-1, 2}) { + ASSERT_NOK_WITH_MSG(compactor->CompactFiles(partition, invalid_bucket, file_names), + "out of range"); + } + partition = {{"unknown", "value"}}; + ASSERT_NOK_WITH_MSG(compactor->CompactFiles(partition, bucket, file_names), "partition keys"); + partition.clear(); + file_names.clear(); + ASSERT_NOK_WITH_MSG(compactor->CompactFiles(partition, bucket, file_names), "at least one"); + file_names = {first_name, first_name}; + ASSERT_NOK_WITH_MSG(compactor->CompactFiles(partition, bucket, file_names), + "distinct file names"); + file_names = {"../" + first_name}; + ASSERT_NOK(compactor->CompactFiles(partition, bucket, file_names)); + file_names = {"missing.parquet"}; + ASSERT_NOK_WITH_MSG(compactor->CompactFiles(partition, bucket, file_names), "not active"); + file_names = {first_name}; + bucket = 1; + ASSERT_NOK_WITH_MSG(compactor->CompactFiles(partition, bucket, file_names), "not active"); + bucket = 0; + file_names = {second_name}; + ASSERT_NOK_WITH_MSG(compactor->CompactFiles(partition, bucket, file_names), "not active"); + ASSERT_OK_AND_ASSIGN(auto restored_snapshot, compactor->GetBaseSnapshotId({}, 0)); + ASSERT_TRUE(restored_snapshot); + auto third = WriteAndPrepare(path, schema, options, R"([[4, "d"]])", 2); + Commit(path, options, third); + const std::string third_name = OnlyNewFile(third)->file_name; + file_names = {third_name}; + ASSERT_NOK_WITH_MSG(compactor->CompactFiles(partition, bucket, file_names), "not active"); + ASSERT_OK_AND_ASSIGN(auto reused_snapshot, compactor->GetBaseSnapshotId({}, 0)); + ASSERT_EQ(restored_snapshot, reused_snapshot); + file_names = {first_name}; + ASSERT_OK(compactor->CompactFiles(partition, bucket, file_names)); + ASSERT_OK(compactor->CompactFiles({}, 1, {second_name})); + ASSERT_OK_AND_ASSIGN(auto messages, compactor->PrepareCommit()); + ASSERT_EQ(2, messages.size()); + std::shared_ptr impl; + for (const auto& message : messages) { + auto candidate = std::dynamic_pointer_cast(message); + ASSERT_TRUE(candidate); + if (candidate->Bucket() == 0) { + impl = candidate; + } + } + ASSERT_TRUE(impl); + ASSERT_EQ(1, impl->GetCompactIncrement().CompactBefore().size()); + ASSERT_EQ(first_name, impl->GetCompactIncrement().CompactBefore()[0]->file_name); + ASSERT_EQ(1, impl->GetCompactIncrement().CompactAfter().size()); + ASSERT_EQ(2, impl->GetCompactIncrement().CompactAfter()[0]->row_count); + ASSERT_NOK_WITH_MSG(compactor->PrepareCommit(), "one-time committing"); + ASSERT_NOK_WITH_MSG(compactor->Write(MakeBatch(schema, R"([[4, "d"]])")), "cannot mix"); + ASSERT_NOK_WITH_MSG(compactor->Compact({}, 0, true), "cannot mix"); + ASSERT_OK(compactor->Close()); + ASSERT_EQ( + 2, + ReadDataFileArray(path, impl->GetCompactIncrement().CompactAfter()[0], options)->length()); + CommitContextBuilder commit_builder(path, "compactor"); + ASSERT_OK_AND_ASSIGN(auto commit_context, commit_builder.SetOptions(options).Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK(commit->Abort(messages)); + auto fs = std::make_shared(); + ASSERT_OK_AND_ASSIGN( + bool exists, + fs->Exists(PathUtil::JoinPath( + path, "bucket-0/" + impl->GetCompactIncrement().CompactAfter()[0]->file_name))); + ASSERT_FALSE(exists); + ASSERT_OK_AND_ASSIGN(auto mixed, new_writer(options)); + ASSERT_OK(mixed->Write(MakeBatch(schema, R"([[4, "d"]])"))); + ASSERT_NOK_WITH_MSG(mixed->CompactFiles(partition, bucket, file_names), "cannot mix"); + ASSERT_OK(mixed->Close()); +} + +TEST_F(AppendOnlyFileStoreWriteTest, TestUnorderedWriterRestore) { + struct TestCase { + bool ordered; + bool write_only; + bool ignore_previous; + int64_t expected_seq; + }; + const std::vector test_cases = { + {/*ordered=*/false, /*write_only=*/false, /*ignore_previous=*/false, /*expected_seq=*/2}, + {/*ordered=*/false, /*write_only=*/false, /*ignore_previous=*/true, /*expected_seq=*/2}, + {/*ordered=*/false, /*write_only=*/true, /*ignore_previous=*/false, /*expected_seq=*/0}, + {/*ordered=*/false, /*write_only=*/true, /*ignore_previous=*/true, /*expected_seq=*/0}, + {/*ordered=*/true, /*write_only=*/false, /*ignore_previous=*/false, /*expected_seq=*/2}, + {/*ordered=*/true, /*write_only=*/false, /*ignore_previous=*/true, /*expected_seq=*/0}, + {/*ordered=*/true, /*write_only=*/true, /*ignore_previous=*/false, /*expected_seq=*/2}, + {/*ordered=*/true, /*write_only=*/true, /*ignore_previous=*/true, /*expected_seq=*/0}, + }; + auto schema = arrow::schema({arrow::field("id", arrow::int32())}); + for (const auto& test_case : test_cases) { + SCOPED_TRACE(::testing::Message() + << "ordered=" << test_case.ordered << ", write_only=" << test_case.write_only + << ", ignore_previous=" << test_case.ignore_previous + << ", expected_seq=" << test_case.expected_seq); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::map options = { + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::BUCKET_APPEND_ORDERED, test_case.ordered ? "true" : "false"}, + {Options::WRITE_ONLY, test_case.write_only ? "true" : "false"}}; + CreateTable(dir->Str(), schema, options); + const std::string path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + Commit(path, options, WriteAndPrepare(path, schema, options, "[[1], [2]]", 0)); + WriteContextBuilder builder(path, commit_user_); + ASSERT_OK_AND_ASSIGN(auto context, builder.SetOptions(options) + .WithIgnorePreviousFiles(test_case.ignore_previous) + .Finish()); + ASSERT_OK_AND_ASSIGN(auto writer, FileStoreWrite::Create(std::move(context))); + ASSERT_OK(writer->Write(MakeBatch(schema, "[[3]]"))); + ASSERT_OK_AND_ASSIGN(auto messages, writer->PrepareCommit()); + ASSERT_EQ(test_case.expected_seq, OnlyNewFile(messages)->min_sequence_number); + ASSERT_OK(writer->Close()); + } +} + TEST_F(AppendOnlyFileStoreWriteTest, TestRealtimeWriteTracksExternalOffsetRange) { std::map options = { {"file.format", "parquet"}, @@ -307,6 +487,8 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestRealtimeWriteTracksExternalOffsetRange) ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_NOK_WITH_MSG(file_store_write->CompactFiles({}, 0, {"unused.parquet"}), + "real-time mode"); ASSERT_OK(file_store_write->Write(MakeBatch(realtime_schema, R"([ [10, 1, "a"], [20, 2, "b"] diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp index c1b889473..b1d93608a 100644 --- a/src/paimon/core/operation/file_store_write.cpp +++ b/src/paimon/core/operation/file_store_write.cpp @@ -20,6 +20,11 @@ namespace paimon { +Status FileStoreWrite::CompactFiles(const std::map&, int32_t, + const std::vector&) { + return Status::NotImplemented("custom compaction only supports fixed-bucket append tables"); +} + Result> FileStoreWrite::PrepareCommitWithProgress(int64_t) { return Status::Invalid("prepare commit with progress requires a real-time writer"); } diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp index b76203a9a..089575b3e 100644 --- a/src/paimon/core/operation/key_value_file_store_write_test.cpp +++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp @@ -349,6 +349,8 @@ TEST_F(KeyValueFileStoreWriteTest, TestWriteWithInvalidBatch) { ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); ASSERT_NOK_WITH_MSG(file_store_write->Write(nullptr), "batch is null pointer"); + ASSERT_NOK_WITH_MSG(file_store_write->CompactFiles({}, 0, {"unused.parquet"}), + "only supports fixed-bucket append tables"); } { arrow::Schema typed_schema(fields); diff --git a/src/paimon/core/table/append_only_file_store_table.cpp b/src/paimon/core/table/append_only_file_store_table.cpp index 7be44d2ed..b8e3906f6 100644 --- a/src/paimon/core/table/append_only_file_store_table.cpp +++ b/src/paimon/core/table/append_only_file_store_table.cpp @@ -76,8 +76,15 @@ Result> AppendOnlyFileStoreTable::CreateWrite( } else if (options.GetBucket() <= 0) { return Status::Invalid( fmt::format("not support bucket {} in append table", options.GetBucket())); + } else if (!options.BucketAppendOrdered()) { + // Unordered write-only writers do not need historical sequence numbers. Compactors must + // restore existing files even if the write context requested ignoring them. + ignore_previous_files = options.WriteOnly(); } if (ctx.GetRealtimeContext()) { + if (!options.BucketAppendOrdered()) { + return Status::Invalid("real-time append write requires bucket-append-ordered=true"); + } if (!ctx.GetWriteSchema().empty()) { return Status::Invalid("real-time append write does not support a custom schema"); } diff --git a/test/inte/append_compaction_inte_test.cpp b/test/inte/append_compaction_inte_test.cpp index ae5dcf086..a6fc5d011 100644 --- a/test/inte/append_compaction_inte_test.cpp +++ b/test/inte/append_compaction_inte_test.cpp @@ -16,6 +16,7 @@ * limitations under the License. */ +#include #include #include #include @@ -206,6 +207,172 @@ std::vector GetTestValuesForAppendCompactionInteTest() { INSTANTIATE_TEST_SUITE_P(FileFormat, AppendCompactionInteTest, ::testing::ValuesIn(GetTestValuesForAppendCompactionInteTest())); +TEST_P(AppendCompactionInteTest, TestCompactFilesWithGap) { + for (bool with_dv : {false, true}) { + SCOPED_TRACE(with_dv); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto schema = arrow::schema( + {arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::int32()), + arrow::field("f2", arrow::int32()), arrow::field("f3", arrow::float64())}); + std::map options = { + {Options::FILE_FORMAT, GetParam()}, + {Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "f2"}, + {Options::BUCKET_APPEND_ORDERED, "false"}, + {Options::COMPACTION_MIN_FILE_NUM, "100"}, + {Options::DELETION_VECTORS_ENABLED, with_dv ? "true" : "false"}}; + ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(dir->Str(), schema, {"f1"}, {}, + options, /*is_streaming_mode=*/true)); + ASSERT_OK_AND_ASSIGN(auto table_schema, helper->LatestSchema()); + ASSERT_TRUE(table_schema); + auto gen = std::make_shared(*table_schema, pool_); + int64_t identifier = 0; + PrepareSimpleAppendData(gen, with_dv, helper.get(), &identifier); + ASSERT_OK(helper->write_->Close()); + ASSERT_OK_AND_ASSIGN(auto compactor, + TestHelper::Create(helper->table_path_, options, true)); + auto* write = dynamic_cast(compactor->write_.get()); + ASSERT_NE(nullptr, write); + auto partition = BinaryRowGenerator::GenerateRow({10}, pool_.get()); + ASSERT_OK_AND_ASSIGN(auto restored, write->ScanExistingFileMetas(partition, 1)); + auto files = restored->DataFiles(); + std::sort(files.begin(), files.end(), BucketedAppendCompactManager::FileComparator(true)); + ASSERT_EQ(3, files.size()); + // Reverse the inputs and leave the middle file untouched. Neither order nor adjacency + // of the supplied names determines eligibility. + ASSERT_OK(compactor->write_->CompactFiles({{"f1", "10"}}, 1, + {files[2]->file_name, files[0]->file_name})); + ASSERT_OK_AND_ASSIGN(auto messages, compactor->write_->PrepareCommit(true, identifier)); + ASSERT_EQ(1, messages.size()); + auto message = messages[0]; + auto impl = std::dynamic_pointer_cast(message); + ASSERT_TRUE(impl); + ASSERT_TRUE(impl->GetNewFilesIncrement().IsEmpty()); + const auto& increment = impl->GetCompactIncrement(); + ASSERT_EQ(2, increment.CompactBefore().size()); + ASSERT_EQ(files[0]->file_name, increment.CompactBefore()[0]->file_name); + ASSERT_EQ(files[2]->file_name, increment.CompactBefore()[1]->file_name); + ASSERT_EQ(1, increment.CompactAfter().size()); + ASSERT_EQ(with_dv ? 2 : 4, increment.CompactAfter()[0]->row_count); + ASSERT_EQ(with_dv ? 1 : 0, increment.NewIndexFiles().size()); + ASSERT_OK_AND_ASSIGN(auto pending, compactor->write_->PrepareCommit(true, identifier)); + for (const auto& pending_message : pending) { + auto pending_impl = std::dynamic_pointer_cast(pending_message); + ASSERT_TRUE(pending_impl); + ASSERT_TRUE(pending_impl->IsEmpty()); + } + // Closing the executor must leave the returned files available for publication. + ASSERT_OK(compactor->write_->Close()); + ASSERT_OK_AND_ASSIGN(std::string bytes, CommitMessage::Serialize(message, pool_)); + ASSERT_OK_AND_ASSIGN( + message, CommitMessage::Deserialize(CommitMessage::CurrentVersion(), bytes.data(), + bytes.size(), pool_)); + ASSERT_OK(compactor->commit_->Commit({message}, identifier++)); + ASSERT_OK_AND_ASSIGN(auto active, write->ScanExistingFileMetas(partition, 1)); + ASSERT_EQ(2, active->DataFiles().size()); + if (with_dv) { + ASSERT_OK_AND_ASSIGN(auto dvs, write->dv_maintainer_factory_->Create( + partition, 1, active->DeleteVectorsIndex())); + ASSERT_TRUE(dvs->DeletionVectorOf(files[1]->file_name)); + ASSERT_FALSE(dvs->DeletionVectorOf(files[0]->file_name)); + ASSERT_FALSE(dvs->DeletionVectorOf(files[2]->file_name)); + } + ASSERT_OK_AND_ASSIGN(auto splits, helper->NewScan(StartupMode::LatestFull(), std::nullopt)); + auto fields = schema->fields(); + fields.insert(fields.begin(), arrow::field("_VALUE_KIND", arrow::int8())); + const std::string expected = with_dv ? R"([ + [0, "Emily", 10, 0, 13.1], [0, "Tony", 10, 0, 14.1], + [0, "Bob", 10, 0, 12.1], [0, "Alex", 10, 0, 16.1] + ])" + : R"([ + [0, "Bob", 10, 0, 12.1], [0, "Emily", 10, 0, 13.1], + [0, "Tony", 10, 0, 14.1], [0, "David", 10, 0, 17.1], + [0, "Emily", 10, 0, 15.1], [0, "Bob", 10, 0, 12.1], + [0, "Alex", 10, 0, 16.1] + ])"; + bool checked = false; + for (const auto& split : splits) { + auto data_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(data_split); + if (data_split->Partition() == partition && data_split->Bucket() == 1) { + ASSERT_OK_AND_ASSIGN(bool equal, helper->ReadAndCheckResult(arrow::struct_(fields), + {split}, expected)); + ASSERT_TRUE(equal); + checked = true; + } + } + ASSERT_TRUE(checked); + } +} + +TEST_P(AppendCompactionInteTest, TestCompactFilesSingleFullyDeletedFile) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto schema = + arrow::schema({arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::int32()), + arrow::field("f2", arrow::int32()), arrow::field("f3", arrow::float64())}); + std::map options = {{Options::FILE_FORMAT, GetParam()}, + {Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "f2"}, + {Options::BUCKET_APPEND_ORDERED, "false"}, + {Options::DELETION_VECTORS_ENABLED, "true"}}; + ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(dir->Str(), schema, {"f1"}, {}, options, + /*is_streaming_mode=*/true)); + ASSERT_OK_AND_ASSIGN(auto table_schema, helper->LatestSchema()); + ASSERT_TRUE(table_schema); + auto gen = std::make_shared(*table_schema, pool_); + int64_t identifier = 0; + PrepareSimpleAppendData(gen, true, helper.get(), &identifier); + ASSERT_OK(helper->write_->Close()); + ASSERT_OK_AND_ASSIGN(auto compactor, TestHelper::Create(helper->table_path_, options, true)); + auto* write = dynamic_cast(compactor->write_.get()); + ASSERT_NE(nullptr, write); + auto partition = BinaryRowGenerator::GenerateRow({10}, pool_.get()); + ASSERT_OK_AND_ASSIGN(auto restored, write->ScanExistingFileMetas(partition, 1)); + auto files = restored->DataFiles(); + std::sort(files.begin(), files.end(), BucketedAppendCompactManager::FileComparator(true)); + ASSERT_EQ(3, files.size()); + ASSERT_EQ(1, files.back()->row_count); + ASSERT_OK(write->CompactFiles({{"f1", "10"}}, 1, {files.back()->file_name})); + ASSERT_OK_AND_ASSIGN(auto messages, compactor->write_->PrepareCommit(true, identifier)); + ASSERT_EQ(1, messages.size()); + auto message = messages[0]; + auto impl = std::dynamic_pointer_cast(message); + ASSERT_TRUE(impl); + ASSERT_EQ(1, impl->GetCompactIncrement().CompactBefore().size()); + ASSERT_TRUE(impl->GetCompactIncrement().CompactAfter().empty()); + ASSERT_OK(compactor->commit_->Commit({message}, identifier++)); + ASSERT_OK_AND_ASSIGN(auto active, write->ScanExistingFileMetas(partition, 1)); + ASSERT_EQ(2, active->DataFiles().size()); + ASSERT_OK_AND_ASSIGN(auto dvs, write->dv_maintainer_factory_->Create( + partition, 1, active->DeleteVectorsIndex())); + ASSERT_TRUE(dvs->DeletionVectorOf(files[0]->file_name)); + ASSERT_TRUE(dvs->DeletionVectorOf(files[1]->file_name)); + ASSERT_FALSE(dvs->DeletionVectorOf(files[2]->file_name)); + ASSERT_OK(compactor->write_->Close()); + + ASSERT_OK_AND_ASSIGN(auto splits, helper->NewScan(StartupMode::LatestFull(), std::nullopt)); + std::vector> target_splits; + for (const auto& split : splits) { + auto data_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(data_split); + if (data_split->Partition() == partition && data_split->Bucket() == 1) { + target_splits.push_back(split); + } + } + ASSERT_FALSE(target_splits.empty()); + auto fields = schema->fields(); + fields.insert(fields.begin(), arrow::field("_VALUE_KIND", arrow::int8())); + ASSERT_OK_AND_ASSIGN(bool equal, + helper->ReadAndCheckResult(arrow::struct_(fields), target_splits, + R"([ + [0, "Emily", 10, 0, 13.1], [0, "Tony", 10, 0, 14.1], + [0, "Bob", 10, 0, 12.1], [0, "Alex", 10, 0, 16.1] + ])")); + ASSERT_TRUE(equal); +} + TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteFullCompaction) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp index 2b1f74dbd..abf948797 100644 --- a/test/inte/realtime_write_inte_test.cpp +++ b/test/inte/realtime_write_inte_test.cpp @@ -1997,6 +1997,7 @@ TEST_P(RealtimeDeduplicateCompactionInteTest, TestCommitTransfersLatestDvToCompa write_builder.SetOptions(options_).WithStreamingMode(true).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr compactor, FileStoreWrite::Create(std::move(write_context))); + ASSERT_NOK_WITH_MSG(compactor->CompactFiles({}, 0, {"unused.parquet"}), "real-time mode"); ASSERT_OK_AND_ASSIGN(std::unique_ptr forbidden_batch, MakeBatch({{99, "not-written", "p0"}}, /*partitioned=*/false)); ASSERT_NOK_WITH_MSG(compactor->Write(std::move(forbidden_batch)), "only supports compaction");