From d4d7950c8ca983415c2d015d042460241b8c263c Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Fri, 13 Nov 2026 19:26:26 +0800 Subject: [PATCH 1/2] feat: add APIs for append-table updates with deletion vectors --- docs/source/api/file_index.rst | 9 + docs/source/api/write.rst | 3 + docs/source/user_guide/write.rst | 28 + include/paimon/api.h | 2 + include/paimon/deletion_vector_write.h | 58 ++ include/paimon/file_index/file_index_read.h | 54 ++ include/paimon/table/source/data_split.h | 2 + .../paimon/table/source/file_row_selection.h | 40 + src/paimon/CMakeLists.txt | 4 + .../deletionvectors/bucketed_dv_maintainer.h | 30 + .../bucketed_dv_maintainer_test.cpp | 61 ++ .../core/operation/append_dv_context.cpp | 81 ++ src/paimon/core/operation/append_dv_context.h | 55 ++ src/paimon/core/operation/append_dv_test.cpp | 732 ++++++++++++++++++ .../core/operation/deletion_vector_write.cpp | 256 ++++++ src/paimon/core/operation/file_index_read.cpp | 248 ++++++ .../operation/precise_file_index_read.cpp | 84 +- .../core/operation/precise_file_index_read.h | 37 +- .../precise_file_index_read_test.cpp | 47 ++ .../core/table/source/data_split_impl.cpp | 9 +- .../core/table/source/data_split_test.cpp | 16 + 21 files changed, 1816 insertions(+), 40 deletions(-) create mode 100644 include/paimon/deletion_vector_write.h create mode 100644 include/paimon/file_index/file_index_read.h create mode 100644 include/paimon/table/source/file_row_selection.h create mode 100644 src/paimon/core/operation/append_dv_context.cpp create mode 100644 src/paimon/core/operation/append_dv_context.h create mode 100644 src/paimon/core/operation/append_dv_test.cpp create mode 100644 src/paimon/core/operation/deletion_vector_write.cpp create mode 100644 src/paimon/core/operation/file_index_read.cpp diff --git a/docs/source/api/file_index.rst b/docs/source/api/file_index.rst index c9fedf377..d8de7bbae 100644 --- a/docs/source/api/file_index.rst +++ b/docs/source/api/file_index.rst @@ -24,6 +24,15 @@ File Index Interface ========= +.. doxygenclass:: paimon::FileIndexRead + :members: + +.. doxygenstruct:: paimon::DataSplit::SimpleDataFileMeta + :members: + +.. doxygenstruct:: paimon::FileRowSelection + :members: + .. doxygenstruct:: paimon::FullTextMatch :members: diff --git a/docs/source/api/write.rst b/docs/source/api/write.rst index 7abeb5b1e..1a5c23b47 100644 --- a/docs/source/api/write.rst +++ b/docs/source/api/write.rst @@ -24,6 +24,9 @@ Write Interface ========= +.. doxygenclass:: paimon::DeletionVectorWrite + :members: + .. doxygenclass:: paimon::FileStoreWrite :members: :undoc-members: diff --git a/docs/source/user_guide/write.rst b/docs/source/user_guide/write.rst index 0acce59a4..b1b1a39c4 100644 --- a/docs/source/user_guide/write.rst +++ b/docs/source/user_guide/write.rst @@ -38,6 +38,34 @@ Parquet, and Avro, thereby improving write throughput. the files it ignored. See :doc:`compaction` for the ``full-compaction`` and ``lookup`` producers. +Updates with Append and Deletion Vectors +--------------------------------------- + +For fixed-bucket or unaware-bucket managed append tables, enable +``deletion-vectors.enabled``. + +1. Plan batch splits at a base snapshot without limits or pruning by individual + keys. Create ``FileIndexRead`` per split and reuse ``Query(predicate)`` to return + files and physical positions excluding historical DV, or an empty result. + Metadata statistics may exclude whole files. The precise read path uses indexes + and data filtering to find exact positions. Index readers and historical DV are reused. +2. Compute replacement rows externally and append through ``FileStoreWrite`` + with ``write-only=true``. +3. Call ``DeletionVectorWrite::PrepareCommit()`` with the table path, branch, + options, ``base_snapshot_id``, all deletion positions, file system, memory pool, + and executor. The entire batch uses one snapshot; passing ``std::nullopt`` + selects the latest snapshot for this call. +4. Combine the returned DV messages with ``FileStoreWrite::PrepareCommit()`` messages in one + ``FileStoreCommit::Commit()`` call. Deletes alone need only the DV messages. + +Different buckets may be updated concurrently. For the same partition and bucket, +wait for the previous update to commit before selecting the next base snapshot. +Do not run compaction or overwrite on that bucket during an update. + +Both APIs load table metadata from disk and support bitmap32 only, including +historical DV. Primary-key, Data Evolution, real-time, query-authorized, and +format tables are unsupported. + External Data Paths ------------------- diff --git a/include/paimon/api.h b/include/paimon/api.h index b0664cea1..7a5eb5414 100644 --- a/include/paimon/api.h +++ b/include/paimon/api.h @@ -22,7 +22,9 @@ #include "paimon/commit_context.h" // IWYU pragma: export #include "paimon/defs.h" // IWYU pragma: export +#include "paimon/deletion_vector_write.h" // IWYU pragma: export #include "paimon/factories/factory.h" // IWYU pragma: export +#include "paimon/file_index/file_index_read.h" // IWYU pragma: export #include "paimon/file_store_commit.h" // IWYU pragma: export #include "paimon/file_store_write.h" // IWYU pragma: export #include "paimon/fs/file_system_factory.h" // IWYU pragma: export diff --git a/include/paimon/deletion_vector_write.h b/include/paimon/deletion_vector_write.h new file mode 100644 index 000000000..29b6b24cb --- /dev/null +++ b/include/paimon/deletion_vector_write.h @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "paimon/commit_message.h" +#include "paimon/result.h" +#include "paimon/table/source/file_row_selection.h" +#include "paimon/visibility.h" + +namespace paimon { +class Executor; +class FileSystem; +class MemoryPool; + +/// Merges append-table DV updates. Different buckets may run concurrently. +/// Serialize updates, compaction and overwrite within each partition/bucket, +/// from snapshot selection through commit. +class PAIMON_EXPORT DeletionVectorWrite { + public: + DeletionVectorWrite() = delete; + ~DeletionVectorWrite() = delete; + + /// Uses one snapshot for the batch; std::nullopt selects latest. + /// Requires a managed fixed/unaware append table with bitmap32 DV. + /// Commit returned messages with append messages; callers handle recovery and orphan cleanup. + static Result>> PrepareCommit( + const std::string& table_path, const std::string& branch, + const std::map& options, + const std::optional& base_snapshot_id, + const std::vector& deletions, + const std::shared_ptr& file_system, const std::shared_ptr& pool, + const std::shared_ptr& executor); +}; + +} // namespace paimon diff --git a/include/paimon/file_index/file_index_read.h b/include/paimon/file_index/file_index_read.h new file mode 100644 index 000000000..4e69b98f6 --- /dev/null +++ b/include/paimon/file_index/file_index_read.h @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/result.h" +#include "paimon/table/source/file_row_selection.h" +#include "paimon/visibility.h" + +namespace paimon { +class Predicate; +class FileSystem; +class MemoryPool; +class Split; + +/// Queries live physical positions in a managed append-table batch split; bitmap32 only. +/// Uses indexes and data filtering to return exact positions. +/// Caches index readers and historical DV. Not thread-safe. +class PAIMON_EXPORT FileIndexRead { + public: + /// Binds a split; options override table options. + static Result> Create( + const std::string& table_path, const std::string& branch, + const std::map& options, const std::shared_ptr& split, + const std::shared_ptr& file_system, const std::shared_ptr& pool); + virtual ~FileIndexRead() = default; + + /// Returns matching files and positions; empty for an empty split or no matching rows. + /// Predicate must be non-null. + virtual Result> Query( + const std::shared_ptr& predicate) = 0; +}; + +} // namespace paimon diff --git a/include/paimon/table/source/data_split.h b/include/paimon/table/source/data_split.h index 4de0d1cf7..69ab8167d 100644 --- a/include/paimon/table/source/data_split.h +++ b/include/paimon/table/source/data_split.h @@ -70,6 +70,7 @@ class PAIMON_EXPORT DataSplit : public Split { int64_t schema_id; int32_t level; Timestamp creation_time; + /// Snapshot delete count: DV cardinality or stored metadata; nullopt if unknown. std::optional delete_row_count; bool operator==(const SimpleDataFileMeta& other) const; @@ -81,6 +82,7 @@ class PAIMON_EXPORT DataSplit : public Split { virtual int32_t Bucket() const = 0; /// Get the list of metadata for all data files in this split. + /// Uses snapshot DV metadata without reading DV files or changing stored metadata. /// @note This method will be removed in future versions and is only used for append tables. virtual std::vector GetFileList() const = 0; }; diff --git a/include/paimon/table/source/file_row_selection.h b/include/paimon/table/source/file_row_selection.h new file mode 100644 index 000000000..974c0a052 --- /dev/null +++ b/include/paimon/table/source/file_row_selection.h @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/table/source/data_split.h" +#include "paimon/utils/roaring_bitmap32.h" +#include "paimon/visibility.h" + +namespace paimon { + +/// Zero-based physical file positions, never renumbered after filtering. +struct PAIMON_EXPORT FileRowSelection { + DataSplit::SimpleDataFileMeta file; + /// String partition values; NULL uses partition.default-name. + std::map partition; + int32_t bucket = 0; + RoaringBitmap32 positions; +}; + +} // namespace paimon diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 30f6250e0..6d577c37e 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -419,6 +419,9 @@ set(PAIMON_CORE_SRCS core/operation/orphan_files_cleaner_impl.cpp core/operation/raw_file_split_read.cpp core/operation/precise_file_index_read.cpp + core/operation/append_dv_context.cpp + core/operation/file_index_read.cpp + core/operation/deletion_vector_write.cpp core/operation/read_context.cpp core/operation/scan_context.cpp core/manifest/snapshot_live_manifest_entries.cpp @@ -1015,6 +1018,7 @@ if(PAIMON_BUILD_TESTS) core/operation/commit/sequence_snapshot_properties_test.cpp core/operation/key_value_file_store_write_test.cpp core/operation/precise_file_index_read_test.cpp + core/operation/append_dv_test.cpp core/operation/internal_read_context_test.cpp core/operation/abstract_split_read_test.cpp core/operation/append_only_file_store_write_test.cpp diff --git a/src/paimon/core/deletionvectors/bucketed_dv_maintainer.h b/src/paimon/core/deletionvectors/bucketed_dv_maintainer.h index 39cda8bb5..c4dac174f 100644 --- a/src/paimon/core/deletionvectors/bucketed_dv_maintainer.h +++ b/src/paimon/core/deletionvectors/bucketed_dv_maintainer.h @@ -22,6 +22,7 @@ #include #include #include +#include #include "paimon/core/deletionvectors/bitmap64_deletion_vector.h" #include "paimon/core/deletionvectors/bitmap_deletion_vector.h" @@ -29,6 +30,7 @@ #include "paimon/core/deletionvectors/deletion_vectors_index_file.h" #include "paimon/core/index/index_file_handler.h" #include "paimon/core/index/index_file_meta.h" +#include "paimon/utils/roaring_bitmap32.h" namespace paimon { @@ -62,6 +64,34 @@ class BucketedDvMaintainer { return Status::OK(); } + Status NotifyNewDeletion(const std::string& file_name, const RoaringBitmap32& positions) { + if (positions.IsEmpty()) { + return Status::OK(); + } + auto bitmap = std::make_shared(positions); + auto iter = deletion_vectors_.find(file_name); + std::shared_ptr incoming = bitmap; + bool use_bitmap64 = + iter == deletion_vectors_.end() + ? bitmap64_ + : dynamic_cast(iter->second.get()) != nullptr; + if (use_bitmap64) { + incoming = Bitmap64DeletionVector::FromBitmapDeletionVector(*bitmap); + } + if (iter == deletion_vectors_.end()) { + deletion_vectors_.emplace(file_name, std::move(incoming)); + modified_ = true; + return Status::OK(); + } + PAIMON_ASSIGN_OR_RAISE(int64_t before, iter->second->GetCardinality()); + PAIMON_RETURN_NOT_OK(iter->second->Merge(incoming)); + PAIMON_ASSIGN_OR_RAISE(int64_t after, iter->second->GetCardinality()); + if (after != before) { + modified_ = true; + } + return Status::OK(); + } + std::optional> DeletionVectorOf( const std::string& file_name) const { if (auto it = deletion_vectors_.find(file_name); it != deletion_vectors_.end()) { diff --git a/src/paimon/core/deletionvectors/bucketed_dv_maintainer_test.cpp b/src/paimon/core/deletionvectors/bucketed_dv_maintainer_test.cpp index 9a0e80b56..8e3be3deb 100644 --- a/src/paimon/core/deletionvectors/bucketed_dv_maintainer_test.cpp +++ b/src/paimon/core/deletionvectors/bucketed_dv_maintainer_test.cpp @@ -137,6 +137,67 @@ TEST(BucketedDvMaintainerTest, TestNotifyNewDeletionOnExistingVectorOnlyMarksWhe ASSERT_TRUE(modified_write.has_value()); } +TEST(BucketedDvMaintainerTest, TestNotifyNewDeletionBitmap) { + for (bool restored_bitmap64 : {false, true}) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto index = CreateDvIndexFile(dir->Str(), !restored_bitmap64); + auto bitmap = std::make_shared(RoaringBitmap32::From({0, 1, 2})); + std::shared_ptr existing = bitmap; + if (restored_bitmap64) { + existing = Bitmap64DeletionVector::FromBitmapDeletionVector(*bitmap); + } + BucketedDvMaintainer maintainer(index, {{"existing", existing}}); + + ASSERT_OK(maintainer.NotifyNewDeletion("empty", RoaringBitmap32())); + ASSERT_FALSE(maintainer.DeletionVectorOf("empty").has_value()); + ASSERT_OK(maintainer.NotifyNewDeletion("existing", RoaringBitmap32::From({1, 2}))); + ASSERT_OK_AND_ASSIGN(std::optional> unchanged, + maintainer.WriteDeletionVectorsIndex()); + ASSERT_FALSE(unchanged.has_value()); + + auto positions = RoaringBitmap32::From({2, 7, 65537}); + ASSERT_OK(maintainer.NotifyNewDeletion("existing", positions)); + ASSERT_OK(maintainer.NotifyNewDeletion("new", positions)); + ASSERT_EQ(positions, RoaringBitmap32::From({2, 7, 65537})); + ASSERT_OK_AND_ASSIGN(int64_t cardinality, existing->GetCardinality()); + ASSERT_EQ(cardinality, 5); + ASSERT_OK_AND_ASSIGN(std::optional> written, + maintainer.WriteDeletionVectorsIndex()); + ASSERT_TRUE(written.has_value()); + std::map> restored; + ASSERT_OK_AND_ASSIGN(restored, index->ReadAllDeletionVectors(*written)); + ASSERT_EQ(restored.size(), 2); + ASSERT_EQ(dynamic_cast(restored.at("existing").get()) != nullptr, + restored_bitmap64); + ASSERT_EQ(dynamic_cast(restored.at("new").get()) != nullptr, + !restored_bitmap64); + for (int32_t position : {0, 1, 2, 7, 65537}) { + ASSERT_OK_AND_ASSIGN(bool deleted, restored.at("existing")->IsDeleted(position)); + ASSERT_TRUE(deleted); + } + ASSERT_OK_AND_ASSIGN(cardinality, restored.at("new")->GetCardinality()); + ASSERT_EQ(cardinality, 3); + for (int32_t position : {2, 7, 65537}) { + ASSERT_OK_AND_ASSIGN(bool deleted, restored.at("new")->IsDeleted(position)); + ASSERT_TRUE(deleted); + } + + ASSERT_OK(maintainer.NotifyNewDeletion("existing", positions)); + ASSERT_OK(maintainer.NotifyNewDeletion("new", positions)); + ASSERT_OK(maintainer.NotifyNewDeletion("existing", RoaringBitmap32())); + ASSERT_OK_AND_ASSIGN(unchanged, maintainer.WriteDeletionVectorsIndex()); + ASSERT_FALSE(unchanged.has_value()); + positions.Add(9); + ASSERT_OK_AND_ASSIGN(bool deleted, existing->IsDeleted(9)); + ASSERT_FALSE(deleted); + auto created = maintainer.DeletionVectorOf("new"); + ASSERT_TRUE(created.has_value()); + ASSERT_OK_AND_ASSIGN(deleted, (*created)->IsDeleted(9)); + ASSERT_FALSE(deleted); + } +} + TEST(BucketedDvMaintainerTest, MixedFormatsAfterConfigurationChanges) { for (bool initial_bitmap64 : {false, true}) { auto dir = UniqueTestDirectory::Create(); diff --git a/src/paimon/core/operation/append_dv_context.cpp b/src/paimon/core/operation/append_dv_context.cpp new file mode 100644 index 000000000..d9aba04df --- /dev/null +++ b/src/paimon/core/operation/append_dv_context.cpp @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/operation/append_dv_context.h" + +#include + +#include "paimon/common/types/data_field.h" +#include "paimon/core/options/table_type.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/schema_validation.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/defs.h" +#include "paimon/format/file_format.h" + +namespace paimon { + +Result AppendDvContext::Create(const std::string& table_path, + const std::string& branch, + const std::map& options, + const std::shared_ptr& file_system, + const std::shared_ptr& pool) { + if (table_path.empty() || !file_system || !pool) { + return Status::Invalid("Append DV requires a path, file system and memory pool"); + } + PAIMON_ASSIGN_OR_RAISE( + std::string resolved_branch, + BranchManager::ResolveBranch(std::nullopt, options, branch, "append DV")); + AppendDvContext context; + context.schema_manager = + std::make_shared(file_system, table_path, resolved_branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + context.schema_manager->Latest()); + if (!latest) { + return Status::Invalid("Append DV table schema not found"); + } + context.schema = *latest; + auto merged_options = context.schema->Options(); + for (const auto& [key, value] : options) { + merged_options[key] = value; + } + merged_options[Options::BRANCH] = resolved_branch; + PAIMON_ASSIGN_OR_RAISE(context.options, CoreOptions::FromMap(merged_options, file_system, {})); + PAIMON_ASSIGN_OR_RAISE(TableType type, TableTypeDefine::FromOptions(context.schema->Options())); + if (type != TableType::TABLE || !context.schema->PrimaryKeys().empty() || + context.options.DataEvolutionEnabled() || context.options.RealtimeEnabled() || + (context.options.GetBucket() != -1 && context.options.GetBucket() <= 0)) { + return Status::NotImplemented("Append DV requires an ordinary managed append table"); + } + PAIMON_RETURN_NOT_OK( + SchemaValidation::RejectQueryAuth(*context.schema, merged_options, "append DV")); + if (context.options.DeletionVectorsBitmap64()) { + return Status::NotImplemented("Append DV does not support bitmap64 deletion vectors"); + } + context.arrow_schema = DataField::ConvertDataFieldsToArrowSchema(context.schema->Fields()); + PAIMON_ASSIGN_OR_RAISE( + context.path_factory, + FileStore(table_path, context.options) + .NewPathFactory(context.arrow_schema, context.schema->PartitionKeys(), + context.options.GetFileFormat()->Identifier(), pool)); + return context; +} + +} // namespace paimon diff --git a/src/paimon/core/operation/append_dv_context.h b/src/paimon/core/operation/append_dv_context.h new file mode 100644 index 000000000..911060bfc --- /dev/null +++ b/src/paimon/core/operation/append_dv_context.h @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/core/core_options.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { +class FileSystem; +class FileStorePathFactory; +class MemoryPool; +class SchemaManager; +class TableSchema; + +// TODO(xinyu.lxy): Switch to Table APIs once Table is implemented. +struct AppendDvContext { + static Result Create(const std::string& table_path, const std::string& branch, + const std::map& options, + const std::shared_ptr& file_system, + const std::shared_ptr& pool); + + CoreOptions options; + std::shared_ptr schema; + std::shared_ptr arrow_schema; + std::shared_ptr schema_manager; + std::shared_ptr path_factory; + + private: + AppendDvContext() = default; +}; + +} // namespace paimon diff --git a/src/paimon/core/operation/append_dv_test.cpp b/src/paimon/core/operation/append_dv_test.cpp new file mode 100644 index 000000000..999783232 --- /dev/null +++ b/src/paimon/core/operation/append_dv_test.cpp @@ -0,0 +1,732 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include "arrow/api.h" +#include "gtest/gtest.h" +#include "paimon/api.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/deletionvectors/bitmap64_deletion_vector.h" +#include "paimon/core/deletionvectors/deletion_vectors_index_file.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/io/data_file_path_factory.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/table/sink/commit_message_impl.h" +#include "paimon/core/table/source/data_split_impl.h" +#include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/deletion_vector_write.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/predicate/predicate_builder.h" +#include "paimon/testing/utils/test_helper.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +class CountingDvReadFileSystem : public LocalFileSystem { + public: + Result> Open(const std::string& path) const override { + ++open_count; + ++file_open_counts[path]; + if (StringUtils::EndsWith(path, DataFilePathFactory::INDEX_PATH_SUFFIX)) { + ++index_open_count; + } + return LocalFileSystem::Open(path); + } + + mutable int64_t open_count = 0; + mutable int64_t index_open_count = 0; + mutable std::map file_open_counts; +}; + +class AppendDvTest : public testing::TestWithParam> { + protected: + void SetUp() override { + directory_ = UniqueTestDirectory::Create(); + ASSERT_TRUE(directory_); + schema_ = arrow::schema({arrow::field("id", arrow::int32()), + arrow::field("value", arrow::int32()), + arrow::field("pt", arrow::utf8())}); + options_ = {{"bucket", std::to_string(std::get<0>(GetParam()))}, + {"file.format", "orc"}, + {"write-only", "true"}, + {"deletion-vectors.enabled", "true"}, + {"deletion-vectors.bitmap64", "false"}, + {"file-index.bitmap.columns", "id"}, + {"file-index.bloom-filter.columns", "value"}, + {"file-index.in-manifest-threshold", std::to_string(std::get<1>(GetParam()))}}; + if (std::get<0>(GetParam()) > 0) { + options_[Options::BUCKET_KEY] = "id"; + } + ASSERT_OK_AND_ASSIGN(helper_, + TestHelper::Create(directory_->Str(), schema_, {"pt"}, {}, options_, + /*is_streaming_mode=*/true)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr first, + TestHelper::MakeRecordBatch(arrow::struct_(schema_->fields()), + R"([[1,10,"a"],[2,20,"a"],[3,30,"a"]])", + {{"pt", "a"}}, 0, {})); + ASSERT_OK(helper_->WriteAndCommit(std::move(first), next_commit_++, std::nullopt)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr second, + TestHelper::MakeRecordBatch(arrow::struct_(schema_->fields()), + R"([[4,40,"a"],[5,50,"a"]])", {{"pt", "a"}}, 0, {})); + ASSERT_OK(helper_->WriteAndCommit(std::move(second), next_commit_++, std::nullopt)); + } + + std::shared_ptr Id(int32_t id) const { + return PredicateBuilder::Equal(0, "id", FieldType::INT, Literal(id)); + } + + Result> PlanSnapshot(const std::optional& snapshot_id) { + ScanContextBuilder scan_builder(helper_->TablePath()); + scan_builder.SetOptions(options_).WithStreamingMode(false); + if (snapshot_id) { + scan_builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(*snapshot_id)); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, + TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scan->CreatePlan()); + if (!plan->SnapshotId()) { + return Status::Invalid("Expected a snapshot in the test table"); + } + last_query_snapshot_id_ = *plan->SnapshotId(); + return plan; + } + + Result> NewIndexRead(const std::shared_ptr& split) { + return FileIndexRead::Create(helper_->TablePath(), "main", options_, split, + std::make_shared(), GetDefaultPool()); + } + + Result> Query( + const std::shared_ptr& predicate, + const std::optional& snapshot_id = std::nullopt) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, PlanSnapshot(snapshot_id)); + std::vector result; + for (const auto& split : plan->Splits()) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, NewIndexRead(split)); + PAIMON_ASSIGN_OR_RAISE(std::vector matches, read->Query(predicate)); + result.insert(result.end(), std::make_move_iterator(matches.begin()), + std::make_move_iterator(matches.end())); + } + return result; + } + + Result>> PrepareDeletions( + const std::vector& deletions, + const std::optional& snapshot_id = std::nullopt) { + return DeletionVectorWrite::PrepareCommit( + helper_->TablePath(), "main", options_, snapshot_id, deletions, + std::make_shared(), GetDefaultPool(), CreateDefaultExecutor()); + } + + Status Commit(const std::vector>& messages) { + CommitContextBuilder builder(helper_->TablePath(), "external_update"); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, + builder.SetOptions(options_).Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, + FileStoreCommit::Create(std::move(context))); + return commit->Commit(messages, next_commit_++); + } + + std::unique_ptr directory_; + std::unique_ptr helper_; + std::shared_ptr schema_; + std::map options_; + int64_t next_commit_ = 1; + int64_t last_query_snapshot_id_ = -1; +}; + +TEST_P(AppendDvTest, ExactPositionsWithIndexAndScan) { + ASSERT_OK_AND_ASSIGN(std::vector matches, Query(Id(2))); + ASSERT_EQ(matches.size(), 1); + ASSERT_EQ(matches[0].file.row_count, 3); + ASSERT_EQ(matches[0].bucket, 0); + ASSERT_EQ(matches[0].partition.at("pt"), "a"); + ASSERT_EQ(matches[0].positions.Cardinality(), 1); + ASSERT_EQ(*matches[0].positions.Begin(), 1); + + ASSERT_OK_AND_ASSIGN(std::vector empty, Query(Id(99))); + ASSERT_TRUE(empty.empty()); + auto inexact = PredicateBuilder::Equal(1, "value", FieldType::INT, Literal(10)); + ASSERT_OK_AND_ASSIGN(matches, Query(inexact)); + ASSERT_EQ(matches.size(), 1); + ASSERT_EQ(matches[0].positions, RoaringBitmap32::From({0})); + ASSERT_OK_AND_ASSIGN(matches, + Query(PredicateBuilder::Equal(2, "pt", FieldType::STRING, + Literal(FieldType::STRING, "a", 1)))); + ASSERT_EQ(matches.size(), 2); + ASSERT_EQ(matches[0].positions.Cardinality() + matches[1].positions.Cardinality(), 5); + ASSERT_OK_AND_ASSIGN(matches, + Query(PredicateBuilder::GreaterThan(0, "id", FieldType::INT, Literal(0)))); + ASSERT_EQ(matches.size(), 2); + ASSERT_EQ(matches[0].positions.Cardinality() + matches[1].positions.Cardinality(), 5); + ASSERT_OK_AND_ASSIGN(std::shared_ptr conjunction, + PredicateBuilder::And({Id(1), Id(3), inexact})); + ASSERT_OK_AND_ASSIGN(empty, Query(conjunction)); + ASSERT_TRUE(empty.empty()); + ASSERT_OK_AND_ASSIGN(conjunction, PredicateBuilder::And({Id(1), inexact})); + ASSERT_OK_AND_ASSIGN(matches, Query(conjunction)); + ASSERT_EQ(matches.size(), 1); + ASSERT_EQ(matches[0].positions, RoaringBitmap32::From({0})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr disjunction, + PredicateBuilder::Or({Id(1), inexact})); + ASSERT_OK_AND_ASSIGN(matches, Query(disjunction)); + ASSERT_EQ(matches.size(), 1); + ASSERT_EQ(matches[0].positions, RoaringBitmap32::From({0})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr exact_or, PredicateBuilder::Or({Id(2), Id(4)})); + ASSERT_OK_AND_ASSIGN(std::vector two_files, Query(exact_or)); + ASSERT_EQ(two_files.size(), 2); +} + +TEST_P(AppendDvTest, MissingIndexesScanLivePhysicalPositions) { + ASSERT_OK_AND_ASSIGN(std::vector deleted, Query(Id(1))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + PrepareDeletions(deleted, last_query_snapshot_id_)); + ASSERT_OK(Commit(messages)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, PlanSnapshot(std::nullopt)); + size_t matched_files = 0; + int64_t live_rows = 0; + for (const auto& split : plan->Splits()) { + auto source = std::dynamic_pointer_cast(split); + ASSERT_TRUE(source); + std::vector> files; + for (const auto& file : source->DataFiles()) { + auto without_index = std::make_shared(*file); + without_index->embedded_index.reset(); + without_index->extra_files.clear(); + files.push_back(std::move(without_index)); + } + DataSplitImpl::Builder builder(source->Partition(), source->Bucket(), source->BucketPath(), + std::move(files)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr without_indexes, + builder.WithSnapshot(*plan->SnapshotId()) + .WithDataDeletionFiles(source->DeletionFiles()) + .RawConvertible(true) + .Build()); + auto read_options = options_; + read_options[Options::READ_BATCH_SIZE] = "1"; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read, + FileIndexRead::Create(helper_->TablePath(), "main", read_options, without_indexes, + std::make_shared(), GetDefaultPool())); + ASSERT_OK_AND_ASSIGN(std::vector matches, read->Query(Id(1))); + ASSERT_TRUE(matches.empty()); + ASSERT_OK_AND_ASSIGN(matches, read->Query(Id(2))); + matched_files += matches.size(); + if (!matches.empty()) { + ASSERT_EQ(matches[0].positions, RoaringBitmap32::From({1})); + } + ASSERT_OK_AND_ASSIGN(matches, read->Query(PredicateBuilder::GreaterThan( + 0, "id", FieldType::INT, Literal(0)))); + for (const auto& match : matches) { + live_rows += match.positions.Cardinality(); + ASSERT_EQ(match.positions, match.file.row_count == 3 ? RoaringBitmap32::From({1, 2}) + : RoaringBitmap32::From({0, 1})); + } + } + ASSERT_EQ(matched_files, 1); + ASSERT_EQ(live_rows, 4); +} + +TEST_P(AppendDvTest, MissingDeclaredIndexPropagatesError) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, PlanSnapshot(std::nullopt)); + ASSERT_FALSE(plan->Splits().empty()); + auto source = std::dynamic_pointer_cast(plan->Splits()[0]); + ASSERT_TRUE(source); + ASSERT_FALSE(source->DataFiles().empty()); + auto file = std::make_shared(*source->DataFiles()[0]); + file->embedded_index.reset(); + file->extra_files = {std::string("missing.index")}; + DataSplitImpl::Builder builder(source->Partition(), source->Bucket(), source->BucketPath(), + {file}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_index, + builder.WithSnapshot(*plan->SnapshotId()).RawConvertible(true).Build()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, NewIndexRead(missing_index)); + ASSERT_NOK(read->Query(PredicateBuilder::GreaterThan(0, "id", FieldType::INT, Literal(0)))); +} + +TEST_P(AppendDvTest, ReusesSplitReadersAcrossPredicates) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, PlanSnapshot(std::nullopt)); + ASSERT_FALSE(plan->Splits().empty()); + auto fs = std::make_shared(); + ASSERT_NOK(FileIndexRead::Create(helper_->TablePath(), "main", options_, plan->Splits()[0], + nullptr, GetDefaultPool())); + ASSERT_NOK(FileIndexRead::Create(helper_->TablePath(), "main", options_, plan->Splits()[0], fs, + nullptr)); + ASSERT_NOK(FileIndexRead::Create(helper_->TablePath(), "../bad", options_, plan->Splits()[0], + fs, GetDefaultPool())); + size_t first_count = 0; + size_t second_count = 0; + for (const auto& split : plan->Splits()) { + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read, + FileIndexRead::Create(helper_->TablePath(), "main", {}, split, fs, GetDefaultPool())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr warmup, + PredicateBuilder::Or({Id(1), Id(4)})); + ASSERT_OK_AND_ASSIGN(std::vector warmed, read->Query(warmup)); + ASSERT_FALSE(warmed.empty()); + ASSERT_OK_AND_ASSIGN(std::vector first, read->Query(Id(1))); + const int64_t opened = fs->index_open_count; + if (std::get<1>(GetParam()) == 0) { + ASSERT_GT(opened, 0); + } + ASSERT_OK_AND_ASSIGN(std::vector second, read->Query(Id(2))); + ASSERT_EQ(fs->index_open_count, opened); + first_count += first.size(); + second_count += second.size(); + if (!first.empty()) { + ASSERT_EQ(first[0].positions, RoaringBitmap32::From({0})); + ASSERT_EQ(second.size(), 1); + ASSERT_EQ(second[0].positions, RoaringBitmap32::From({1})); + first[0].positions.Add(2); + } + ASSERT_NOK(read->Query(nullptr)); + ASSERT_OK_AND_ASSIGN( + std::vector scanned, + read->Query(PredicateBuilder::GreaterThan(0, "id", FieldType::INT, Literal(0)))); + ASSERT_FALSE(scanned.empty()); + auto inexact = + PredicateBuilder::Equal(1, "value", FieldType::INT, Literal(first.empty() ? 40 : 10)); + ASSERT_OK_AND_ASSIGN(scanned, read->Query(inexact)); + ASSERT_EQ(scanned.size(), 1); + ASSERT_EQ(scanned[0].positions, RoaringBitmap32::From({0})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr conjunction, + PredicateBuilder::And({Id(first.empty() ? 4 : 1), Id(first.empty() ? 5 : 3), inexact})); + ASSERT_OK_AND_ASSIGN(std::vector empty, read->Query(conjunction)); + ASSERT_TRUE(empty.empty()); + ASSERT_OK_AND_ASSIGN(conjunction, + PredicateBuilder::And({Id(first.empty() ? 4 : 1), inexact})); + ASSERT_OK_AND_ASSIGN(scanned, read->Query(conjunction)); + ASSERT_EQ(scanned.size(), 1); + ASSERT_EQ(scanned[0].positions, RoaringBitmap32::From({0})); + const int64_t opened_after_scan = fs->index_open_count; + ASSERT_OK_AND_ASSIGN(std::vector repeated, read->Query(Id(1))); + ASSERT_EQ(repeated.size(), first.size()); + if (!repeated.empty()) { + ASSERT_EQ(repeated[0].positions, RoaringBitmap32::From({0})); + } + ASSERT_EQ(fs->index_open_count, opened_after_scan); + } + ASSERT_EQ(first_count, 1); + ASSERT_EQ(second_count, 1); +} + +TEST_P(AppendDvTest, EmptySplitReturnsEmptyMatches) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, PlanSnapshot(std::nullopt)); + ASSERT_FALSE(plan->Splits().empty()); + auto source = std::dynamic_pointer_cast(plan->Splits()[0]); + ASSERT_TRUE(source); + DataSplitImpl::Builder builder(source->Partition(), source->Bucket(), source->BucketPath(), {}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr empty_split, + builder.WithSnapshot(*plan->SnapshotId()).RawConvertible(true).Build()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, NewIndexRead(empty_split)); + ASSERT_OK_AND_ASSIGN(std::vector matches, read->Query(Id(1))); + ASSERT_TRUE(matches.empty()); + ASSERT_NOK(read->Query(nullptr)); +} + +TEST_P(AppendDvTest, StatisticsPruneBeforeOpeningIndexes) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, PlanSnapshot(std::nullopt)); + auto fs = std::make_shared(); + for (const auto& split : plan->Splits()) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + FileIndexRead::Create(helper_->TablePath(), "main", options_, split, + fs, GetDefaultPool())); + const int64_t opened = fs->open_count; + ASSERT_OK_AND_ASSIGN( + std::vector matches, + read->Query(PredicateBuilder::GreaterThan(0, "id", FieldType::INT, Literal(99)))); + ASSERT_TRUE(matches.empty()); + ASSERT_OK_AND_ASSIGN(matches, read->Query(PredicateBuilder::Equal( + 1, "value", FieldType::INT, Literal(999)))); + ASSERT_TRUE(matches.empty()); + ASSERT_EQ(fs->open_count, opened); + ASSERT_OK_AND_ASSIGN(matches, read->Query(PredicateBuilder::GreaterThan( + 0, "id", FieldType::INT, Literal(0)))); + ASSERT_FALSE(matches.empty()); + } +} + +TEST_P(AppendDvTest, UnknownStatisticsUseIndexesAndScan) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, PlanSnapshot(std::nullopt)); + size_t match_count = 0; + size_t scan_match_count = 0; + for (const auto& split : plan->Splits()) { + auto source = std::dynamic_pointer_cast(split); + ASSERT_TRUE(source); + std::vector> files; + for (const auto& file : source->DataFiles()) { + auto without_stats = std::make_shared(*file); + without_stats->value_stats = SimpleStats::EmptyStats(); + without_stats->value_stats_cols = std::nullopt; + files.push_back(std::move(without_stats)); + } + DataSplitImpl::Builder builder(source->Partition(), source->Bucket(), source->BucketPath(), + std::move(files)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr without_stats, + builder.WithSnapshot(*plan->SnapshotId()).RawConvertible(true).Build()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, NewIndexRead(without_stats)); + ASSERT_OK_AND_ASSIGN(std::vector matches, read->Query(Id(2))); + match_count += matches.size(); + ASSERT_OK_AND_ASSIGN( + matches, read->Query(PredicateBuilder::Equal(1, "value", FieldType::INT, Literal(10)))); + scan_match_count += matches.size(); + if (!matches.empty()) { + ASSERT_EQ(matches[0].positions, RoaringBitmap32::From({0})); + } + } + ASSERT_EQ(match_count, 1); + ASSERT_EQ(scan_match_count, 1); +} + +TEST_P(AppendDvTest, AddedColumnIsAllNullInHistoricalFiles) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, PlanSnapshot(std::nullopt)); + auto fs = std::make_shared(); + SchemaManager manager(fs, helper_->TablePath()); + ASSERT_OK_AND_ASSIGN(std::optional> schema, manager.Latest()); + ASSERT_TRUE(schema); + auto fields = (*schema)->Fields(); + const int32_t field_index = static_cast(fields.size()); + const int32_t field_id = (*schema)->HighestFieldId() + 1; + fields.emplace_back(field_id, arrow::field("added", arrow::int32())); + ASSERT_OK(TestHelper::WriteNextSchema(fs, helper_->TablePath(), fields, field_id, + (*schema)->Options())); + for (const auto& split : plan->Splits()) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + FileIndexRead::Create(helper_->TablePath(), "main", options_, split, + fs, GetDefaultPool())); + auto equal = PredicateBuilder::Equal(field_index, "added", FieldType::INT, Literal(1)); + ASSERT_OK_AND_ASSIGN(std::vector matches, read->Query(equal)); + ASSERT_TRUE(matches.empty()); + const int64_t opened = fs->open_count; + ASSERT_OK_AND_ASSIGN(matches, read->Query(PredicateBuilder::IsNotNull(field_index, "added", + FieldType::INT))); + ASSERT_TRUE(matches.empty()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr conjunction, + PredicateBuilder::And({equal, Id(2)})); + ASSERT_OK_AND_ASSIGN(matches, read->Query(conjunction)); + ASSERT_TRUE(matches.empty()); + ASSERT_EQ(fs->open_count, opened); + ASSERT_OK_AND_ASSIGN( + matches, read->Query(PredicateBuilder::IsNull(field_index, "added", FieldType::INT))); + ASSERT_FALSE(matches.empty()); + for (const auto& match : matches) { + ASSERT_EQ(match.positions.Cardinality(), match.file.row_count); + } + } + ASSERT_OK_AND_ASSIGN(std::vector unchanged, Query(Id(2))); + ASSERT_EQ(unchanged.size(), 1); + ASSERT_EQ(unchanged[0].positions, RoaringBitmap32::From({1})); +} + +TEST_P(AppendDvTest, BoundSplitRetainsHistoricalDeletions) { + ASSERT_OK_AND_ASSIGN(std::vector targets, Query(Id(1))); + ASSERT_EQ(targets.size(), 1); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + PrepareDeletions(targets, last_query_snapshot_id_)); + ASSERT_OK(Commit(messages)); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, PlanSnapshot(std::nullopt)); + auto fs = std::make_shared(); + bool found = false; + for (const auto& split : plan->Splits()) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + FileIndexRead::Create(helper_->TablePath(), "main", options_, split, + fs, GetDefaultPool())); + ASSERT_OK_AND_ASSIGN(std::vector deleted, read->Query(Id(1))); + ASSERT_TRUE(deleted.empty()); + const auto opened = fs->file_open_counts; + const int64_t indexes_opened = fs->index_open_count; + ASSERT_OK_AND_ASSIGN(std::vector live, read->Query(Id(2))); + ASSERT_EQ(fs->index_open_count, indexes_opened); + if (live.empty()) { + continue; + } + found = true; + ASSERT_OK_AND_ASSIGN(messages, PrepareDeletions(live, *plan->SnapshotId())); + ASSERT_OK(Commit(messages)); + + ASSERT_OK_AND_ASSIGN(live, read->Query(Id(2))); + ASSERT_EQ(live.size(), 1); + ASSERT_EQ(live[0].positions, RoaringBitmap32::From({1})); + ASSERT_OK_AND_ASSIGN(deleted, read->Query(Id(1))); + ASSERT_TRUE(deleted.empty()); + ASSERT_EQ(fs->index_open_count, indexes_opened); + auto source = std::dynamic_pointer_cast(split); + ASSERT_TRUE(source); + bool checked_dv = false; + for (const auto& deletion : source->DeletionFiles()) { + if (deletion) { + ASSERT_TRUE(opened.count(deletion->path)); + ASSERT_EQ(fs->file_open_counts.at(deletion->path), opened.at(deletion->path)); + checked_dv = true; + } + } + ASSERT_TRUE(checked_dv); + } + ASSERT_TRUE(found); + ASSERT_OK_AND_ASSIGN(std::vector latest, Query(Id(2))); + ASSERT_TRUE(latest.empty()); +} + +TEST_P(AppendDvTest, MergeHistoryAndAppendInOneCommit) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr first_deletes, + PredicateBuilder::Or({Id(1), Id(4)})); + ASSERT_OK_AND_ASSIGN(std::vector targets, Query(first_deletes)); + ASSERT_EQ(targets.size(), 2); + const int64_t base_snapshot_id = last_query_snapshot_id_; + auto duplicate_targets = targets; + duplicate_targets.insert(duplicate_targets.end(), targets.begin(), targets.end()); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + PrepareDeletions(duplicate_targets, base_snapshot_id)); + ASSERT_EQ(messages.size(), 1); + + ASSERT_OK_AND_ASSIGN(std::string bytes, + CommitMessage::SerializeList(messages, GetDefaultPool())); + ASSERT_OK_AND_ASSIGN( + std::vector> decoded, + CommitMessage::DeserializeList(CommitMessage::CurrentVersion(), bytes.data(), bytes.size(), + GetDefaultPool())); + ASSERT_OK(Commit(decoded)); + ASSERT_OK_AND_ASSIGN(std::vector deleted, Query(first_deletes)); + ASSERT_TRUE(deleted.empty()); + ASSERT_OK_AND_ASSIGN(std::vector historical, + Query(first_deletes, base_snapshot_id)); + ASSERT_EQ(historical.size(), 2); + + // Restore historical DV, not the latest snapshot's DV. + ASSERT_OK_AND_ASSIGN(std::vector> historical_messages, + PrepareDeletions({historical[0]}, base_snapshot_id)); + ASSERT_EQ(historical_messages.size(), 1); + + // Rewriting a shared DV container must preserve untouched files. + ASSERT_OK_AND_ASSIGN(std::vector update, Query(Id(2))); + ASSERT_EQ(update.size(), 1); + ASSERT_EQ(update[0].file.delete_row_count, std::optional(1)); + ASSERT_OK_AND_ASSIGN(messages, PrepareDeletions(update)); + WriteContextBuilder builder(helper_->TablePath(), "external_update"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr append_context, + builder.SetOptions(options_).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr append, + FileStoreWrite::Create(std::move(append_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr replacement, + TestHelper::MakeRecordBatch(arrow::struct_(schema_->fields()), + R"([[2,200,"a"]])", {{"pt", "a"}}, 0, {})); + ASSERT_OK(append->Write(std::move(replacement))); + ASSERT_OK_AND_ASSIGN(std::vector> appended, + append->PrepareCommit(/*wait_compaction=*/true, next_commit_)); + messages.insert(messages.end(), appended.begin(), appended.end()); + ASSERT_OK(Commit(messages)); + ASSERT_OK(append->Close()); + ASSERT_OK_AND_ASSIGN(deleted, Query(first_deletes)); + ASSERT_TRUE(deleted.empty()); + ASSERT_OK_AND_ASSIGN(std::vector replacement_rows, Query(Id(2))); + ASSERT_EQ(replacement_rows.size(), 1); + ASSERT_EQ(replacement_rows[0].file.delete_row_count, std::optional(0)); + ASSERT_NE(replacement_rows[0].file.file_path, update[0].file.file_path); + ASSERT_EQ(*replacement_rows[0].positions.Begin(), 0); + ASSERT_OK_AND_ASSIGN(std::vector unchanged, Query(Id(5))); + ASSERT_EQ(unchanged.size(), 1); + ASSERT_EQ(unchanged[0].file.delete_row_count, std::optional(1)); + ASSERT_EQ(*unchanged[0].positions.Begin(), 1); + + // Committed deletions are idempotent. + ASSERT_OK_AND_ASSIGN(messages, PrepareDeletions({targets[0]}, std::nullopt)); + ASSERT_TRUE(messages.empty()); +} + +TEST_P(AppendDvTest, RejectInvalidTargetsBeforeWriting) { + ASSERT_OK_AND_ASSIGN(std::vector matches, Query(Id(2))); + ASSERT_EQ(matches.size(), 1); + FileRowSelection invalid = matches[0]; + invalid.positions.Add(invalid.file.row_count); + ASSERT_NOK(PrepareDeletions({matches[0], invalid}, last_query_snapshot_id_)); + invalid = matches[0]; + ++invalid.file.schema_id; + ASSERT_NOK(PrepareDeletions({invalid}, last_query_snapshot_id_)); + invalid = matches[0]; + invalid.partition["pt"] = "other"; + ASSERT_NOK(PrepareDeletions({invalid}, last_query_snapshot_id_)); + invalid = matches[0]; + invalid.file.file_path += ".other"; + ASSERT_NOK(PrepareDeletions({invalid}, last_query_snapshot_id_)); + invalid = matches[0]; + ++invalid.bucket; + ASSERT_NOK(PrepareDeletions({invalid}, last_query_snapshot_id_)); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + PrepareDeletions({}, last_query_snapshot_id_)); + ASSERT_TRUE(messages.empty()); + ASSERT_NOK(PrepareDeletions({}, -1)); + ASSERT_NOK(PrepareDeletions({}, 9999)); +} + +TEST_P(AppendDvTest, PartitionAndBucketIsolationAndNulls) { + ASSERT_OK_AND_ASSIGN( + std::unique_ptr other_partition, + TestHelper::MakeRecordBatch(arrow::struct_(schema_->fields()), + R"([[1,70,"b"],[null,80,"b"]])", {{"pt", "b"}}, 0, {})); + ASSERT_OK(helper_->WriteAndCommit(std::move(other_partition), next_commit_++, std::nullopt)); + if (std::get<0>(GetParam()) > 0) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr other_bucket, + TestHelper::MakeRecordBatch(arrow::struct_(schema_->fields()), + R"([[1,90,"a"]])", {{"pt", "a"}}, 1, {})); + ASSERT_OK(helper_->WriteAndCommit(std::move(other_bucket), next_commit_++, std::nullopt)); + } + ASSERT_OK_AND_ASSIGN(std::vector matches, Query(Id(1))); + ASSERT_EQ(matches.size(), std::get<0>(GetParam()) > 0 ? 3 : 2); + std::vector deletions; + for (const auto& match : matches) { + if (match.partition.at("pt") == "b") { + deletions.push_back(match); + } + } + ASSERT_OK_AND_ASSIGN(std::vector> messages, + PrepareDeletions(deletions, last_query_snapshot_id_)); + ASSERT_EQ(messages.size(), 1); + ASSERT_OK(Commit(messages)); + ASSERT_OK_AND_ASSIGN(matches, Query(Id(1))); + ASSERT_EQ(matches.size(), std::get<0>(GetParam()) > 0 ? 2 : 1); + for (const auto& match : matches) { + ASSERT_EQ(match.partition.at("pt"), "a"); + } + ASSERT_OK_AND_ASSIGN(std::vector nulls, + Query(PredicateBuilder::IsNull(0, "id", FieldType::INT))); + ASSERT_EQ(nulls.size(), 1); + ASSERT_EQ(nulls[0].partition.at("pt"), "b"); + ASSERT_EQ(*nulls[0].positions.Begin(), 1); +} + +TEST_P(AppendDvTest, BatchUsesOneSnapshotAcrossPartitionsAndBuckets) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr other_partition, + TestHelper::MakeRecordBatch(arrow::struct_(schema_->fields()), + R"([[1,70,"b"]])", {{"pt", "b"}}, 0, {})); + ASSERT_OK(helper_->WriteAndCommit(std::move(other_partition), next_commit_++, std::nullopt)); + if (std::get<0>(GetParam()) > 0) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr other_bucket, + TestHelper::MakeRecordBatch(arrow::struct_(schema_->fields()), + R"([[1,90,"a"]])", {{"pt", "a"}}, 1, {})); + ASSERT_OK(helper_->WriteAndCommit(std::move(other_bucket), next_commit_++, std::nullopt)); + } + ASSERT_OK_AND_ASSIGN(std::vector targets, Query(Id(1))); + const int64_t base_snapshot_id = last_query_snapshot_id_; + const size_t bucket_count = std::get<0>(GetParam()) > 0 ? 3 : 2; + ASSERT_EQ(targets.size(), bucket_count); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + PrepareDeletions(targets, base_snapshot_id)); + ASSERT_EQ(messages.size(), bucket_count); + ASSERT_OK(Commit(messages)); + ASSERT_OK_AND_ASSIGN(std::vector deleted, Query(Id(1))); + ASSERT_TRUE(deleted.empty()); + + // Every bucket restores the requested history, even after a newer snapshot is committed. + ASSERT_OK_AND_ASSIGN(messages, PrepareDeletions(targets, base_snapshot_id)); + ASSERT_EQ(messages.size(), bucket_count); + ASSERT_OK_AND_ASSIGN(messages, PrepareDeletions(targets, std::nullopt)); + ASSERT_TRUE(messages.empty()); +} + +TEST_P(AppendDvTest, LatestSnapshotRequiresExistingSnapshot) { + auto directory = UniqueTestDirectory::Create(); + ASSERT_TRUE(directory); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr empty_table, + TestHelper::Create(directory->Str(), schema_, {"pt"}, {}, options_, false)); + ASSERT_NOK_WITH_MSG( + DeletionVectorWrite::PrepareCommit(empty_table->TablePath(), "main", options_, std::nullopt, + {}, std::make_shared(), + GetDefaultPool(), CreateDefaultExecutor()), + "existing snapshot"); +} + +TEST_P(AppendDvTest, RejectRealtimeConfiguration) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, PlanSnapshot(std::nullopt)); + ASSERT_FALSE(plan->Splits().empty()); + options_[Options::REALTIME_ENABLED] = "true"; + ASSERT_NOK_WITH_MSG(NewIndexRead(plan->Splits().front()), "ordinary managed append table"); + ASSERT_NOK_WITH_MSG(PrepareDeletions({}, last_query_snapshot_id_), + "ordinary managed append table"); +} + +TEST_P(AppendDvTest, RejectBitmap64Configuration) { + ASSERT_OK_AND_ASSIGN(std::vector matches, Query(Id(2))); + options_["deletion-vectors.bitmap64"] = "true"; + ASSERT_NOK_WITH_MSG(Query(Id(2)), "bitmap64"); + ASSERT_NOK_WITH_MSG(PrepareDeletions(matches, last_query_snapshot_id_), "bitmap64"); +} + +TEST_P(AppendDvTest, RejectHistoricalBitmap64WithBitmap32Option) { + ASSERT_OK_AND_ASSIGN(std::vector matches, Query(Id(2))); + ASSERT_EQ(matches.size(), 1); + ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(options_)); + FileStore store(helper_->TablePath(), options); + ASSERT_OK_AND_ASSIGN(std::shared_ptr paths, + store.NewPathFactory(schema_, {"pt"}, "orc", GetDefaultPool())); + ASSERT_OK_AND_ASSIGN(BinaryRow partition, paths->ToBinaryRow(matches[0].partition)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr index_paths, + paths->CreateIndexFileFactory(partition, matches[0].bucket)); + DeletionVectorsIndexFile index_file(options.GetFileSystem(), std::move(index_paths), + /*bitmap64=*/true, GetDefaultPool()); + auto dv = std::make_shared(); + ASSERT_OK(dv->Delete(0)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr index, + index_file.WriteSingleFile({{PathUtil::GetName(matches[0].file.file_path), dv}})); + auto message = std::make_shared(partition, matches[0].bucket, std::nullopt, + DataIncrement({}, {}, {}, {index}, {}), + CompactIncrement({}, {}, {})); + ASSERT_OK(Commit({message})); + + // Reject historical bitmap64 DV even with bitmap32 options. + ASSERT_NOK_WITH_MSG(Query(Id(2)), "bitmap64"); + ASSERT_NOK_WITH_MSG(PrepareDeletions(matches, last_query_snapshot_id_), "bitmap64"); +} + +TEST(AppendDvInvalidContextTest, NullContextsAndMissingSplit) { + auto fs = std::make_shared(); + ASSERT_NOK(FileIndexRead::Create("", "main", {}, nullptr, fs, GetDefaultPool())); + auto file_system = std::make_shared(); + auto pool = GetDefaultPool(); + std::shared_ptr executor = CreateDefaultExecutor(); + ASSERT_NOK(DeletionVectorWrite::PrepareCommit("", "main", {}, std::nullopt, {}, file_system, + pool, executor)); + ASSERT_NOK(DeletionVectorWrite::PrepareCommit("table", "main", {}, std::nullopt, {}, nullptr, + pool, executor)); + ASSERT_NOK(DeletionVectorWrite::PrepareCommit("table", "main", {}, std::nullopt, {}, + file_system, nullptr, executor)); + ASSERT_NOK(DeletionVectorWrite::PrepareCommit("table", "main", {}, std::nullopt, {}, + file_system, pool, nullptr)); + ASSERT_NOK(DeletionVectorWrite::PrepareCommit("table", "../invalid", {}, std::nullopt, {}, + file_system, pool, executor)); + auto directory = UniqueTestDirectory::Create(); + ASSERT_TRUE(directory); + ASSERT_NOK(FileIndexRead::Create(directory->Str(), "main", {}, nullptr, fs, GetDefaultPool())); +} + +INSTANTIATE_TEST_SUITE_P(BucketAndEncoding, AppendDvTest, + testing::Combine(testing::Values(-1, 2), testing::Values(0, 4096))); + +} // namespace paimon::test diff --git a/src/paimon/core/operation/deletion_vector_write.cpp b/src/paimon/core/operation/deletion_vector_write.cpp new file mode 100644 index 000000000..cecf0adcb --- /dev/null +++ b/src/paimon/core/operation/deletion_vector_write.cpp @@ -0,0 +1,256 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/deletion_vector_write.h" + +#include +#include +#include + +#include "paimon/common/utils/scope_guard.h" +#include "paimon/core/deletionvectors/bucketed_dv_maintainer.h" +#include "paimon/core/io/compact_increment.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/io/data_file_path_factory.h" +#include "paimon/core/io/data_increment.h" +#include "paimon/core/operation/append_dv_context.h" +#include "paimon/core/operation/append_only_file_store_scan.h" +#include "paimon/core/operation/write_restore.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/bucket_mode.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/table/sink/commit_message_impl.h" +#include "paimon/core/utils/field_mapping.h" +#include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/fs/file_system.h" +#include "paimon/scan_context.h" + +namespace paimon { +namespace { + +using BucketKey = std::pair, int32_t>; + +struct BucketDeletions { + BinaryRow partition; + int32_t bucket = 0; + std::vector> indexes; + std::map positions; +}; + +Status ValidateDeletion(const FileRowSelection& deletion, const DataFileMeta& file) { + const auto& reference = deletion.file; + // Split delete_row_count comes from DV cardinality. + if (reference.file_size != file.file_size || reference.row_count != file.row_count || + reference.min_sequence_number != file.min_sequence_number || + reference.max_sequence_number != file.max_sequence_number || + reference.schema_id != file.schema_id || reference.level != file.level || + !(reference.creation_time == file.creation_time)) { + return Status::Invalid("Deletion file metadata does not match the base snapshot"); + } + if (file.row_count > RoaringBitmap32::MAX_VALUE) { + return Status::NotImplemented("DeletionVectorWrite requires bitmap32-addressable files"); + } + for (auto iter = deletion.positions.Begin(); iter != deletion.positions.End(); ++iter) { + if (*iter < 0 || *iter >= file.row_count) { + return Status::Invalid("Deletion position is outside the data file"); + } + } + return Status::OK(); +} + +Result> CollectDeletions( + const AppendDvContext& context, const std::shared_ptr& snapshot_manager, + const Snapshot& snapshot, const std::vector& deletions, + const std::shared_ptr& index_handler, + const std::shared_ptr& executor, const std::shared_ptr& pool) { + FileStore store(context.path_factory->RootPath(), context.options); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_list, + store.NewManifestList(context.path_factory, pool)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr partition_schema, + FieldMapping::GetPartitionSchema(context.arrow_schema, context.schema->PartitionKeys())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_file, + store.NewManifestFile(context.path_factory, partition_schema, pool)); + std::map> grouped; + for (const auto& deletion : deletions) { + grouped[{deletion.partition, deletion.bucket}].push_back(&deletion); + } + std::vector buckets; + for (const auto& [key, requests] : grouped) { + BucketDeletions bucket; + PAIMON_ASSIGN_OR_RAISE(bucket.partition, context.path_factory->ToBinaryRow(key.first)); + bucket.bucket = key.second; + std::vector> partitions; + if (!key.first.empty()) { + partitions.push_back(key.first); + } + auto filter = std::make_shared(nullptr, partitions, bucket.bucket); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + AppendOnlyFileStoreScan::Create(snapshot_manager, context.schema_manager, manifest_list, + manifest_file, context.schema, context.arrow_schema, + filter, context.options, executor, pool)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, + scan->WithSnapshot(snapshot)->CreatePlan()); + std::vector> files; + PAIMON_ASSIGN_OR_RAISE([[maybe_unused]] std::optional total_buckets, + WriteRestore::ExtractDataFiles(plan->Files(), &files)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr paths, + context.path_factory->CreateDataFilePathFactory(bucket.partition, bucket.bucket)); + std::map> live_files; + for (const auto& file : files) { + if (!live_files.emplace(paths->ToPath(file), file).second) { + return Status::Invalid("Base snapshot contains duplicate physical file paths"); + } + } + for (const auto* request : requests) { + auto file = live_files.find(request->file.file_path); + if (file == live_files.end()) { + return Status::Invalid( + "Deletion target is not a live data file in the base snapshot"); + } + PAIMON_RETURN_NOT_OK(ValidateDeletion(*request, *file->second)); + if (!request->positions.IsEmpty()) { + bucket.positions[file->second->file_name] |= request->positions; + } + } + if (!bucket.positions.empty()) { + PAIMON_ASSIGN_OR_RAISE( + bucket.indexes, + index_handler->Scan(snapshot, DeletionVectorsIndexFile::DELETION_VECTORS_INDEX, + bucket.partition, bucket.bucket)); + buckets.push_back(std::move(bucket)); + } + } + return buckets; +} + +Result>> SelectIndexes(const BucketDeletions& bucket, + bool fixed_bucket) { + std::vector> selected; + for (const auto& index : bucket.indexes) { + if (!index->DvRanges()) { + return Status::Invalid("Historical DV index has no file ranges"); + } + if (fixed_bucket || std::any_of(index->DvRanges()->begin(), index->DvRanges()->end(), + [&](const auto& entry) { + return bucket.positions.count(entry.first) != 0; + })) { + selected.push_back(index); + } + } + return selected; +} + +Result> WriteBucket( + const BucketDeletions& bucket, bool fixed_bucket, + const std::shared_ptr& index_handler, std::vector* generated) { + PAIMON_ASSIGN_OR_RAISE(std::vector> indexes, + SelectIndexes(bucket, fixed_bucket)); + std::map> vectors; + PAIMON_ASSIGN_OR_RAISE( + vectors, index_handler->ReadAllDeletionVectors(bucket.partition, bucket.bucket, indexes)); + for (const auto& [name, dv] : vectors) { + if (dynamic_cast(dv.get()) == nullptr) { + return Status::NotImplemented( + "DeletionVectorWrite does not support bitmap64 deletion vectors"); + } + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr maintainer, + BucketedDvMaintainer::Factory(index_handler) + .Create(bucket.partition, bucket.bucket, vectors)); + for (const auto& [name, positions] : bucket.positions) { + PAIMON_RETURN_NOT_OK(maintainer->NotifyNewDeletion(name, positions)); + } + PAIMON_ASSIGN_OR_RAISE(std::optional> new_index, + maintainer->WriteDeletionVectorsIndex()); + if (!new_index) { + return std::shared_ptr(); + } + generated->push_back(maintainer->DvIndexFile()->Path(*new_index)); + if (fixed_bucket) { + indexes.clear(); + } + return std::make_shared( + bucket.partition, bucket.bucket, std::nullopt, + DataIncrement({}, {}, {}, {*new_index}, std::move(indexes)), CompactIncrement({}, {}, {})); +} + +} // namespace + +Result>> DeletionVectorWrite::PrepareCommit( + const std::string& table_path, const std::string& branch, + const std::map& options, + const std::optional& base_snapshot_id, const std::vector& deletions, + const std::shared_ptr& file_system, const std::shared_ptr& pool, + const std::shared_ptr& executor) { + if (!executor) { + return Status::Invalid("DeletionVectorWrite requires an executor"); + } + if (base_snapshot_id && *base_snapshot_id < Snapshot::FIRST_SNAPSHOT_ID) { + return Status::Invalid("Invalid base snapshot id"); + } + PAIMON_ASSIGN_OR_RAISE(AppendDvContext context, + AppendDvContext::Create(table_path, branch, options, file_system, pool)); + if (!context.options.DeletionVectorsEnabled()) { + return Status::Invalid("DeletionVectorWrite requires deletion-vectors.enabled=true"); + } + FileStore store(table_path, context.options); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr snapshot_manager, + store.NewSnapshotManager(nullptr, std::nullopt)); + std::optional snapshot; + if (base_snapshot_id) { + PAIMON_ASSIGN_OR_RAISE(snapshot, snapshot_manager->LoadSnapshot(*base_snapshot_id)); + } else { + PAIMON_ASSIGN_OR_RAISE(snapshot, snapshot_manager->LatestSnapshot()); + } + if (!snapshot) { + return Status::Invalid("DeletionVectorWrite requires an existing snapshot"); + } + std::vector> messages; + if (deletions.empty()) { + return messages; + } + const bool fixed_bucket = context.options.GetBucket() != -1; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr index_handler, + store.NewIndexFileHandler( + fixed_bucket ? BucketMode::HASH_FIXED : BucketMode::BUCKET_UNAWARE, + context.path_factory, pool)); + PAIMON_ASSIGN_OR_RAISE(std::vector buckets, + CollectDeletions(context, snapshot_manager, *snapshot, deletions, + index_handler, executor, pool)); + std::vector generated; + ScopeGuard cleanup([&]() { + for (const auto& path : generated) { + (void)file_system->Delete(path, false); + } + }); + for (const auto& bucket : buckets) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr message, + WriteBucket(bucket, fixed_bucket, index_handler, &generated)); + if (message) { + messages.push_back(std::move(message)); + } + } + cleanup.Release(); + return messages; +} + +} // namespace paimon diff --git a/src/paimon/core/operation/file_index_read.cpp b/src/paimon/core/operation/file_index_read.cpp new file mode 100644 index 000000000..1166a55b2 --- /dev/null +++ b/src/paimon/core/operation/file_index_read.cpp @@ -0,0 +1,248 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/file_index/file_index_read.h" + +#include +#include +#include +#include + +#include "paimon/common/data/binary_row.h" +#include "paimon/common/predicate/predicate_filter.h" +#include "paimon/common/predicate/predicate_validator.h" +#include "paimon/core/deletionvectors/bitmap_deletion_vector.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/operation/append_dv_context.h" +#include "paimon/core/operation/internal_read_context.h" +#include "paimon/core/operation/precise_file_index_read.h" +#include "paimon/core/operation/raw_file_split_read.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/stats/simple_stats_evolutions.h" +#include "paimon/core/table/source/data_split_impl.h" +#include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/read_context.h" + +namespace paimon { +namespace { + +class FileIndexReadImpl final : public FileIndexRead { + public: + static Result> Create( + AppendDvContext context, const std::shared_ptr& split, + const std::shared_ptr& pool); + + Result> Query( + const std::shared_ptr& predicate) override; + + private: + struct File { + std::shared_ptr meta; + DataSplit::SimpleDataFileMeta reference; + std::optional stats; + std::unique_ptr read; + RoaringBitmap32 deleted_positions; + }; + + FileIndexReadImpl(AppendDvContext context, std::shared_ptr file_paths, + DeletionVector::Factory dv_factory, const BinaryRow& partition_row, + std::map partition, int32_t bucket, + std::vector files, std::shared_ptr pool) + : context_(std::move(context)), + pool_(std::move(pool)), + evolutions_(context_.schema, pool_), + file_paths_(std::move(file_paths)), + dv_factory_(std::move(dv_factory)), + partition_row_(partition_row), + partition_(std::move(partition)), + bucket_(bucket), + files_(std::move(files)) {} + + Status LoadStats(File* file); + Status LoadReader(File* file); + Result QueryFile(const std::shared_ptr& predicate, + File* file); + + AppendDvContext context_; + std::shared_ptr pool_; + SimpleStatsEvolutions evolutions_; + std::shared_ptr file_paths_; + std::unique_ptr split_read_; + DeletionVector::Factory dv_factory_; + BinaryRow partition_row_; + std::map partition_; + int32_t bucket_; + std::vector files_; +}; + +Result> FileIndexReadImpl::Create( + AppendDvContext context, const std::shared_ptr& split, + const std::shared_ptr& pool) { + auto data_split = std::dynamic_pointer_cast(split); + if (!data_split || data_split->IsStreaming() || !data_split->RawConvertible() || + !data_split->BeforeFiles().empty()) { + return Status::NotImplemented("FileIndexRead requires a raw-convertible batch DataSplit"); + } + const auto& files = data_split->DataFiles(); + const auto& deletions = data_split->DeletionFiles(); + if (!deletions.empty() && deletions.size() != files.size()) { + return Status::Invalid("FileIndexRead split has mismatched deletion files"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_paths, + context.path_factory->CreateDataFilePathFactory(data_split->Partition(), + data_split->Bucket())); + std::vector> partition_values; + PAIMON_ASSIGN_OR_RAISE(partition_values, + context.path_factory->GeneratePartitionVector(data_split->Partition())); + std::map partition(partition_values.begin(), partition_values.end()); + auto dv_factory = DeletionVector::CreateFactory( + context.options.GetFileSystem(), DeletionVector::CreateDeletionFileMap(files, deletions), + pool); + auto references = data_split->GetFileList(); + std::vector query_files; + query_files.reserve(files.size()); + for (size_t i = 0; i < files.size(); ++i) { + query_files.push_back(File{files[i], std::move(references[i]), std::nullopt, nullptr, {}}); + } + return std::unique_ptr(new FileIndexReadImpl( + std::move(context), std::move(file_paths), std::move(dv_factory), data_split->Partition(), + std::move(partition), data_split->Bucket(), std::move(query_files), pool)); +} + +Status FileIndexReadImpl::LoadStats(File* file) { + if (file->stats) { + return Status::OK(); + } + std::shared_ptr schema = context_.schema; + if (file->meta->schema_id != schema->Id()) { + PAIMON_ASSIGN_OR_RAISE(schema, context_.schema_manager->ReadSchema(file->meta->schema_id)); + } + auto evolution = evolutions_.GetOrCreate(schema); + auto dense_fields = file->meta->value_stats_cols; + if (file->meta->value_stats.MinValues().GetFieldCount() == 0) { + dense_fields = std::vector{}; + } + PAIMON_ASSIGN_OR_RAISE( + SimpleStatsEvolution::EvolutionStats stats, + evolution->Evolution(file->meta->value_stats, file->meta->row_count, dense_fields)); + file->stats.emplace(std::move(stats)); + return Status::OK(); +} + +Status FileIndexReadImpl::LoadReader(File* file) { + if (file->read) { + return Status::OK(); + } + if (!split_read_) { + ReadContextBuilder builder(context_.path_factory->RootPath()); + builder.SetOptions(context_.options.ToMap()) + .WithBranch(context_.options.GetBranch()) + .WithFileSystem(context_.options.GetFileSystem()) + .WithMemoryPool(pool_); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_context, builder.Finish()); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr internal_context, + InternalReadContext::Create(read_context, context_.schema, context_.options.ToMap())); + split_read_ = std::make_unique(context_.path_factory, internal_context, + pool_, read_context->GetExecutor()); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, + PreciseFileIndexRead::Create(*split_read_, partition_row_, file->meta, + nullptr, file_paths_, {})); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr dv, dv_factory_(file->meta->file_name)); + if (dv) { + auto bitmap = dynamic_cast(dv.get()); + if (!bitmap) { + return Status::NotImplemented( + "FileIndexRead does not support bitmap64 deletion vectors"); + } + file->deleted_positions = *bitmap->GetBitmap(); + } + file->read = std::move(read); + return Status::OK(); +} + +Result FileIndexReadImpl::QueryFile( + const std::shared_ptr& predicate, File* file) { + if (file->meta->row_count < 0) { + return Status::Invalid("Data file row count is negative"); + } + if (file->meta->row_count == 0) { + return RoaringBitmap32(); + } + PAIMON_RETURN_NOT_OK(LoadStats(file)); + const auto& stats = *file->stats; + PAIMON_ASSIGN_OR_RAISE( + bool may_match, predicate->Test(context_.arrow_schema, file->meta->row_count, + *stats.min_values, *stats.max_values, *stats.null_counts)); + if (!may_match) { + return RoaringBitmap32(); + } + if (file->meta->row_count > RoaringBitmap32::MAX_VALUE) { + return Status::NotImplemented("FileIndexRead requires bitmap32 physical row positions"); + } + PAIMON_RETURN_NOT_OK(LoadReader(file)); + RoaringBitmap32 candidates; + candidates.AddRange(0, static_cast(file->meta->row_count)); + candidates -= file->deleted_positions; + if (candidates.IsEmpty()) { + return candidates; + } + return file->read->ReadRowPositions(predicate, candidates); +} + +Result> FileIndexReadImpl::Query( + const std::shared_ptr& predicate) { + if (!predicate) { + return Status::Invalid("FileIndexRead requires a non-null query predicate"); + } + PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithSchema( + *context_.arrow_schema, predicate, /*validate_field_idx=*/true)); + auto filter = std::dynamic_pointer_cast(predicate); + if (!filter) { + return Status::NotImplemented("FileIndexRead requires a scalar predicate"); + } + std::vector result; + for (auto& file : files_) { + PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 matches, QueryFile(filter, &file)); + if (!matches.IsEmpty()) { + result.push_back( + FileRowSelection{file.reference, partition_, bucket_, std::move(matches)}); + } + } + return result; +} + +} // namespace + +Result> FileIndexRead::Create( + const std::string& table_path, const std::string& branch, + const std::map& options, const std::shared_ptr& split, + const std::shared_ptr& file_system, const std::shared_ptr& pool) { + if (!split) { + return Status::Invalid("FileIndexRead requires a split"); + } + PAIMON_ASSIGN_OR_RAISE(AppendDvContext context, + AppendDvContext::Create(table_path, branch, options, file_system, pool)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, + FileIndexReadImpl::Create(std::move(context), split, pool)); + return std::unique_ptr(std::move(read)); +} + +} // namespace paimon diff --git a/src/paimon/core/operation/precise_file_index_read.cpp b/src/paimon/core/operation/precise_file_index_read.cpp index c538c530a..3f459462e 100644 --- a/src/paimon/core/operation/precise_file_index_read.cpp +++ b/src/paimon/core/operation/precise_file_index_read.cpp @@ -194,18 +194,14 @@ Result> PreciseFileIndexRead::Create( for (size_t i = 0; i < data_fields.size(); ++i) { data_by_id.emplace(data_fields[i].Id(), std::make_pair(i, data_fields[i])); } - std::map, std::shared_ptr> mapped_leaves; - // Precheck the entire tree before an empty candidate set can short-circuit it. - PAIMON_RETURN_NOT_OK(PreparePredicate(split_read.context_->GetPredicate(), read_by_name, - data_by_id, split_read.context_->GetPartitionKeys(), - indexes.get(), &mapped_leaves)); std::shared_ptr vector_search; std::shared_ptr full_text_search; PAIMON_RETURN_NOT_OK(PrepareSearch(split_read.context_, field_names, file->file_name, indexes.get(), &vector_search, &full_text_search)); - return std::unique_ptr(new PreciseFileIndexRead( - split_read, partition, file, field_mapping_builder, path_factory, extra_format_options, - std::move(indexes), mapped_leaves, vector_search, full_text_search)); + return std::unique_ptr( + new PreciseFileIndexRead(split_read, partition, file, field_mapping_builder, path_factory, + extra_format_options, std::move(indexes), std::move(read_by_name), + std::move(data_by_id), vector_search, full_text_search)); } PreciseFileIndexRead::PreciseFileIndexRead( @@ -213,8 +209,8 @@ PreciseFileIndexRead::PreciseFileIndexRead( const std::shared_ptr& file, const FieldMappingBuilder* field_mapping_builder, const std::shared_ptr& path_factory, const std::map& extra_format_options, - std::unique_ptr indexes, - const std::map, std::shared_ptr>& mapped_leaves, + std::unique_ptr indexes, std::map read_by_name, + std::map> data_by_id, const std::shared_ptr& vector_search, const std::shared_ptr& full_text_search) : split_read_(split_read), @@ -224,10 +220,30 @@ PreciseFileIndexRead::PreciseFileIndexRead( path_factory_(path_factory), extra_format_options_(extra_format_options), indexes_(std::move(indexes)), - mapped_leaves_(mapped_leaves), + read_by_name_(std::move(read_by_name)), + data_by_id_(std::move(data_by_id)), vector_search_(vector_search), full_text_search_(full_text_search) {} +PreciseFileIndexRead::~PreciseFileIndexRead() = default; + +Result PreciseFileIndexRead::ReadRowPositions( + const std::shared_ptr& predicate, const RoaringBitmap32& candidates) const { + PAIMON_ASSIGN_OR_RAISE(PredicateMapping mapped_leaves, PreparePredicateMapping(predicate)); + PAIMON_ASSIGN_OR_RAISE(Evaluation evaluated, Evaluate(predicate, candidates, mapped_leaves)); + return std::move(evaluated.rows); +} + +Result PreciseFileIndexRead::PreparePredicateMapping( + const std::shared_ptr& predicate) const { + PredicateMapping mapped_leaves; + // Validate every branch before evaluation can short-circuit. + PAIMON_RETURN_NOT_OK(PreparePredicate(predicate, read_by_name_, data_by_id_, + split_read_.context_->GetPartitionKeys(), indexes_.get(), + &mapped_leaves)); + return mapped_leaves; +} + Result PreciseFileIndexRead::CreateCandidates( const DeletionVector::Factory& dv_factory, const std::optional>& row_ranges) const { @@ -253,27 +269,30 @@ Result PreciseFileIndexRead::CreateCandidates( } Result PreciseFileIndexRead::Evaluate( - const std::shared_ptr& predicate, const RoaringBitmap32& candidates) const { + const std::shared_ptr& predicate, const RoaringBitmap32& candidates, + const PredicateMapping& mapped_leaves) const { if (!predicate || candidates.IsEmpty()) { return Evaluation{candidates, {}}; } if (auto leaf = std::dynamic_pointer_cast(predicate)) { - return EvaluateLeaf(leaf, candidates); + return EvaluateLeaf(leaf, candidates, mapped_leaves); } if (auto compound = std::dynamic_pointer_cast(predicate)) { - return EvaluateCompound(compound, candidates); + return EvaluateCompound(compound, candidates, mapped_leaves); } return Status::Invalid(fmt::format("unsupported file predicate {}", predicate->ToString())); } Result PreciseFileIndexRead::EvaluateCompound( - const std::shared_ptr& predicate, const RoaringBitmap32& candidates) const { + const std::shared_ptr& predicate, const RoaringBitmap32& candidates, + const PredicateMapping& mapped_leaves) const { const Function::Type type = predicate->GetFunction().GetType(); if (type == Function::Type::OR) { Evaluation result; for (const auto& child : predicate->Children()) { // Every branch must see the same input, including rows matched by earlier branches. - PAIMON_ASSIGN_OR_RAISE(Evaluation child_result, Evaluate(child, candidates)); + PAIMON_ASSIGN_OR_RAISE(Evaluation child_result, + Evaluate(child, candidates, mapped_leaves)); result.rows |= child_result.rows; result.scores.insert(child_result.scores.begin(), child_result.scores.end()); } @@ -291,7 +310,8 @@ Result PreciseFileIndexRead::EvaluateCompound( std::vector children; children.reserve(predicate->Children().size()); for (const auto& child : predicate->Children()) { - PAIMON_ASSIGN_OR_RAISE(std::optional selectivity, EstimateSelectivity(child)); + PAIMON_ASSIGN_OR_RAISE(std::optional selectivity, + EstimateSelectivity(child, mapped_leaves)); children.push_back({child, selectivity}); } std::stable_sort(children.begin(), children.end(), [](const Child& lhs, const Child& rhs) { @@ -303,7 +323,8 @@ Result PreciseFileIndexRead::EvaluateCompound( Evaluation result{candidates, {}}; for (const auto& child : children) { - PAIMON_ASSIGN_OR_RAISE(Evaluation child_result, Evaluate(child.predicate, result.rows)); + PAIMON_ASSIGN_OR_RAISE(Evaluation child_result, + Evaluate(child.predicate, result.rows, mapped_leaves)); result.rows = std::move(child_result.rows); result.scores.insert(child_result.scores.begin(), child_result.scores.end()); if (result.rows.IsEmpty()) { @@ -314,12 +335,13 @@ Result PreciseFileIndexRead::EvaluateCompound( } Result> PreciseFileIndexRead::EstimateSelectivity( - const std::shared_ptr& predicate) const { + const std::shared_ptr& predicate, const PredicateMapping& mapped_leaves) const { if (!predicate) { return std::optional(); } if (auto leaf = std::dynamic_pointer_cast(predicate)) { - PAIMON_ASSIGN_OR_RAISE(std::optional estimate, EstimateLeafSelectivity(leaf)); + PAIMON_ASSIGN_OR_RAISE(std::optional estimate, + EstimateLeafSelectivity(leaf, mapped_leaves)); if (estimate && (!std::isfinite(*estimate) || *estimate < 0.0 || *estimate > 1.0)) { return std::optional(); } @@ -338,7 +360,8 @@ Result> PreciseFileIndexRead::EstimateSelectivity( type == Function::Type::AND ? std::nullopt : std::make_optional(0.0); // These are scheduling hints. Continue visiting all children to propagate errors. for (const auto& child : compound->Children()) { - PAIMON_ASSIGN_OR_RAISE(std::optional child_estimate, EstimateSelectivity(child)); + PAIMON_ASSIGN_OR_RAISE(std::optional child_estimate, + EstimateSelectivity(child, mapped_leaves)); if (type == Function::Type::AND) { if (child_estimate) { if (estimate.has_value()) { @@ -357,9 +380,9 @@ Result> PreciseFileIndexRead::EstimateSelectivity( } Result> PreciseFileIndexRead::EstimateLeafSelectivity( - const std::shared_ptr& predicate) const { - auto iter = mapped_leaves_.find(predicate); - if (iter == mapped_leaves_.end()) { + const std::shared_ptr& predicate, const PredicateMapping& mapped_leaves) const { + auto iter = mapped_leaves.find(predicate); + if (iter == mapped_leaves.end()) { return Status::Invalid("File predicate mapping was not prepared"); } if (!iter->second) { @@ -369,15 +392,16 @@ Result> PreciseFileIndexRead::EstimateLeafSelectivity( } Result PreciseFileIndexRead::EvaluateLeaf( - const std::shared_ptr& predicate, const RoaringBitmap32& candidates) const { + const std::shared_ptr& predicate, const RoaringBitmap32& candidates, + const PredicateMapping& mapped_leaves) const { bool is_full_text = predicate->GetFunction().GetType() == Function::Type::FULL_TEXT_MATCH; bool needs_scores = is_full_text && split_read_.context_->NeedsScores( {InternalReadContext::IndexScoreSource::Type::FULL_TEXT_MATCH, predicate->FieldName()}); std::shared_ptr index_result = FileIndexResult::Remain(); - auto iter = mapped_leaves_.find(predicate); - if (iter == mapped_leaves_.end()) { + auto iter = mapped_leaves.find(predicate); + if (iter == mapped_leaves.end()) { return Status::Invalid("File predicate mapping was not prepared"); } if (iter->second) { @@ -549,9 +573,11 @@ Result>> PreciseFileIndexRead::CreateOu Result> PreciseFileIndexRead::CreateReader( const DeletionVector::Factory& dv_factory, const std::optional>& row_ranges) const { + PAIMON_ASSIGN_OR_RAISE(PredicateMapping mapped_leaves, + PreparePredicateMapping(split_read_.context_->GetPredicate())); PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 candidates, CreateCandidates(dv_factory, row_ranges)); - PAIMON_ASSIGN_OR_RAISE(Evaluation evaluated, - Evaluate(split_read_.context_->GetPredicate(), candidates)); + PAIMON_ASSIGN_OR_RAISE(Evaluation evaluated, Evaluate(split_read_.context_->GetPredicate(), + candidates, mapped_leaves)); if (evaluated.rows.IsEmpty()) { return std::unique_ptr(); } diff --git a/src/paimon/core/operation/precise_file_index_read.h b/src/paimon/core/operation/precise_file_index_read.h index 3c373620c..69e947b4a 100644 --- a/src/paimon/core/operation/precise_file_index_read.h +++ b/src/paimon/core/operation/precise_file_index_read.h @@ -26,6 +26,7 @@ #include #include +#include "paimon/common/types/data_field.h" #include "paimon/core/deletionvectors/deletion_vector.h" #include "paimon/reader/file_batch_reader.h" #include "paimon/result.h" @@ -56,12 +57,21 @@ class PreciseFileIndexRead { const std::shared_ptr& path_factory, const std::map& extra_format_options); + ~PreciseFileIndexRead(); + + /// Returns exact predicate matches among the supplied live physical rows. + Result ReadRowPositions(const std::shared_ptr& predicate, + const RoaringBitmap32& candidates) const; + /// Finishes evaluation synchronously. The returned reader does not retain this instance. Result> CreateReader( const DeletionVector::Factory& dv_factory, const std::optional>& row_ranges) const; private: + using PredicateMapping = + std::map, std::shared_ptr>; + struct Evaluation { RoaringBitmap32 rows; // Sources retain physical row positions until all filtering and root TopK are complete. @@ -74,24 +84,30 @@ class PreciseFileIndexRead { const std::shared_ptr& path_factory, const std::map& extra_format_options, std::unique_ptr indexes, - const std::map, - std::shared_ptr>& mapped_leaves, + std::map read_by_name, + std::map> data_by_id, const std::shared_ptr& vector_search, const std::shared_ptr& full_text_search); Result CreateCandidates( const DeletionVector::Factory& dv_factory, const std::optional>& row_ranges) const; + Result PreparePredicateMapping( + const std::shared_ptr& predicate) const; Result Evaluate(const std::shared_ptr& predicate, - const RoaringBitmap32& candidates) const; + const RoaringBitmap32& candidates, + const PredicateMapping& mapped_leaves) const; Result EvaluateCompound(const std::shared_ptr& predicate, - const RoaringBitmap32& candidates) const; + const RoaringBitmap32& candidates, + const PredicateMapping& mapped_leaves) const; Result EvaluateLeaf(const std::shared_ptr& predicate, - const RoaringBitmap32& candidates) const; - Result> EstimateSelectivity( - const std::shared_ptr& predicate) const; + const RoaringBitmap32& candidates, + const PredicateMapping& mapped_leaves) const; + Result> EstimateSelectivity(const std::shared_ptr& predicate, + const PredicateMapping& mapped_leaves) const; Result> EstimateLeafSelectivity( - const std::shared_ptr& predicate) const; + const std::shared_ptr& predicate, + const PredicateMapping& mapped_leaves) const; /// Opens a fresh mapped reader for selected physical rows without re-evaluating indexes, /// predicates or deletion vectors. Result> CreateSelectedFileReader( @@ -112,7 +128,10 @@ class PreciseFileIndexRead { std::shared_ptr path_factory_; std::map extra_format_options_; std::unique_ptr indexes_; - std::map, std::shared_ptr> mapped_leaves_; + // Read field name -> field definition, including the ID used to map predicates to file fields. + std::map read_by_name_; + // Field ID -> (zero-based position in the file schema, file field definition). + std::map> data_by_id_; std::shared_ptr vector_search_; std::shared_ptr full_text_search_; }; diff --git a/src/paimon/core/operation/precise_file_index_read_test.cpp b/src/paimon/core/operation/precise_file_index_read_test.cpp index 640179ef9..b63b7fc18 100644 --- a/src/paimon/core/operation/precise_file_index_read_test.cpp +++ b/src/paimon/core/operation/precise_file_index_read_test.cpp @@ -230,6 +230,53 @@ class PreciseFileIndexReadTest : public ::testing::Test { std::make_shared(); }; +TEST_F(PreciseFileIndexReadTest, TestReadRowPositionsAcrossPredicates) { + ReadContextBuilder builder(SearchTablePath()); + builder.EnablePreciseFileIndexEvaluate(true); + builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}) + .SetPredicate(PredicateBuilder::GreaterThan(3, "f3", FieldType::DOUBLE, Literal(13.5))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr split_read, CreateSplitRead(context)); + auto file = PrepareDataFile(); + auto path_factory = std::make_shared(); + ASSERT_OK( + path_factory->Init(SearchTablePath() + "/f1=10/f2=0/bucket-0", "orc", "data-", nullptr)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr field_mapping_builder, + FieldMappingBuilder::Create(split_read->context_->GetReadSchema(), + split_read->context_->GetPartitionKeys(), nullptr)); + const BinaryRow partition = BinaryRowGenerator::GenerateRow({10, 0}, pool_.get()); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read, + PreciseFileIndexRead::Create(*split_read, partition, file, field_mapping_builder.get(), + path_factory, {})); + const RoaringBitmap32 candidates = RoaringBitmap32::From({0, 1, 2}); + auto first = PredicateBuilder::Equal(3, "f3", FieldType::DOUBLE, Literal(12.1)); + ASSERT_OK_AND_ASSIGN(RoaringBitmap32 rows, read->ReadRowPositions(first, candidates)); + ASSERT_EQ(rows, RoaringBitmap32::From({0})); + ASSERT_OK_AND_ASSIGN( + rows, read->ReadRowPositions(PredicateBuilder::Equal(1, "f1", FieldType::INT, Literal(10)), + candidates)); + ASSERT_EQ(rows, candidates); + ASSERT_OK_AND_ASSIGN( + rows, read->ReadRowPositions( + PredicateBuilder::LessOrEqual(3, "f3", FieldType::DOUBLE, Literal(13.1)), + RoaringBitmap32::From({1, 2}))); + ASSERT_EQ(rows, RoaringBitmap32::From({1})); + ASSERT_OK_AND_ASSIGN(rows, read->ReadRowPositions(first, candidates)); + ASSERT_EQ(rows, RoaringBitmap32::From({0})); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + read->CreateReader({}, std::nullopt)); + ASSERT_TRUE(reader); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + ReadResultCollector::CollectResult(std::move(reader))); + CheckSearchResult(result, + {arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::int32()), + arrow::field("f2", arrow::int32()), arrow::field("f3", arrow::float64())}, + R"([["Tony",10,0,14.1]])"); +} + TEST_F(PreciseFileIndexReadTest, TestMixedPredicateNestedAndOrWithPartitionConstants) { const std::string first_query = R"({"match":{"query":"first"}})"; const std::string second_query = R"({"match":{"query":"second"}})"; diff --git a/src/paimon/core/table/source/data_split_impl.cpp b/src/paimon/core/table/source/data_split_impl.cpp index be0ef4030..95cef7a66 100644 --- a/src/paimon/core/table/source/data_split_impl.cpp +++ b/src/paimon/core/table/source/data_split_impl.cpp @@ -72,17 +72,22 @@ int64_t DataSplitImpl::RowCount() const { std::vector DataSplitImpl::GetFileList() const { std::vector result_files; result_files.reserve(data_files_.size()); - for (const auto& file : data_files_) { + for (size_t i = 0; i < data_files_.size(); ++i) { + const auto& file = data_files_[i]; std::string result_file_path; if (!file->external_path) { result_file_path = PathUtil::JoinPath(bucket_path_, file->file_name); } else { result_file_path = file->external_path.value(); } + std::optional delete_row_count = file->delete_row_count; + if (i < data_deletion_files_.size() && data_deletion_files_[i]) { + delete_row_count = data_deletion_files_[i]->cardinality; + } result_files.emplace_back(result_file_path, file->file_size, file->row_count, file->min_sequence_number, file->max_sequence_number, file->schema_id, file->level, file->creation_time, - file->delete_row_count); + delete_row_count); } return result_files; } diff --git a/src/paimon/core/table/source/data_split_test.cpp b/src/paimon/core/table/source/data_split_test.cpp index bb631be9a..8da7a0e26 100644 --- a/src/paimon/core/table/source/data_split_test.cpp +++ b/src/paimon/core/table/source/data_split_test.cpp @@ -1190,6 +1190,10 @@ TEST(DataSplitTest, TestPartialMergedRowCountRawConvertibleWithoutDeletionFiles) // Java parity: rawConvertible and deletion files absent should return row_count sum. ASSERT_OK_AND_ASSIGN(std::optional merged_row_count, data_split->MergedRowCount()); ASSERT_EQ(std::optional(7), merged_row_count); + auto files = data_split->GetFileList(); + ASSERT_EQ(files.size(), 2); + ASSERT_EQ(files[0].delete_row_count, std::optional(0)); + ASSERT_EQ(files[1].delete_row_count, std::optional(0)); } TEST(DataSplitTest, TestPartialMergedRowCountRawConvertibleWithCardinality) { @@ -1245,6 +1249,13 @@ TEST(DataSplitTest, TestPartialMergedRowCountRawConvertibleWithCardinality) { // Java parity: null + cardinality + cardinality => (7 + (2 - 2) + (3 - 1)) = 9. ASSERT_OK_AND_ASSIGN(std::optional merged_row_count, data_split->MergedRowCount()); ASSERT_EQ(std::optional(9), merged_row_count); + auto files = data_split->GetFileList(); + ASSERT_EQ(files.size(), 3); + ASSERT_EQ(files[0].delete_row_count, std::optional(0)); + ASSERT_EQ(files[1].delete_row_count, std::optional(2)); + ASSERT_EQ(files[2].delete_row_count, std::optional(1)); + ASSERT_EQ(file_meta2->delete_row_count, std::optional(0)); + ASSERT_EQ(file_meta3->delete_row_count, std::optional(0)); } TEST(DataSplitTest, TestPartialMergedRowCountMixedCardinalityReturnsNullopt) { @@ -1288,6 +1299,11 @@ TEST(DataSplitTest, TestPartialMergedRowCountMixedCardinalityReturnsNullopt) { // If any deletion file misses cardinality, metadata count is unknown and must not be partial. ASSERT_OK_AND_ASSIGN(std::optional merged_row_count, data_split->MergedRowCount()); ASSERT_EQ(std::nullopt, merged_row_count); + auto files = data_split->GetFileList(); + ASSERT_EQ(files.size(), 2); + ASSERT_EQ(files[0].delete_row_count, std::optional(2)); + ASSERT_EQ(files[1].delete_row_count, std::nullopt); + ASSERT_EQ(file_meta2->delete_row_count, std::optional(0)); } TEST(DataSplitTest, TestPartialMergedRowCountUnknownDeleteRowCountDoesNotBlockRawCount) { From 9413b3c64c6f026d0b8362fc50c510ee04a143e6 Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Sat, 14 Nov 2026 10:25:44 +0800 Subject: [PATCH 2/2] fix ci --- src/paimon/core/operation/append_dv_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/paimon/core/operation/append_dv_test.cpp b/src/paimon/core/operation/append_dv_test.cpp index 999783232..f29cb0a9b 100644 --- a/src/paimon/core/operation/append_dv_test.cpp +++ b/src/paimon/core/operation/append_dv_test.cpp @@ -410,7 +410,7 @@ TEST_P(AppendDvTest, AddedColumnIsAllNullInHistoricalFiles) { ASSERT_OK_AND_ASSIGN(std::optional> schema, manager.Latest()); ASSERT_TRUE(schema); auto fields = (*schema)->Fields(); - const int32_t field_index = static_cast(fields.size()); + const auto field_index = static_cast(fields.size()); const int32_t field_id = (*schema)->HighestFieldId() + 1; fields.emplace_back(field_id, arrow::field("added", arrow::int32())); ASSERT_OK(TestHelper::WriteNextSchema(fs, helper_->TablePath(), fields, field_id,