diff --git a/docs/source/api/file_index.rst b/docs/source/api/file_index.rst index c9fedf377..cac005c8b 100644 --- a/docs/source/api/file_index.rst +++ b/docs/source/api/file_index.rst @@ -39,6 +39,15 @@ Interface :members: :undoc-members: +.. doxygenstruct:: paimon::FileIndexInput + :members: + +.. doxygenstruct:: paimon::DataFileIndexInput + :members: + +.. doxygenclass:: paimon::FileIndexWrite + :members: + .. doxygenclass:: paimon::FileIndexReader :members: :undoc-members: diff --git a/docs/source/user_guide/file_index.rst b/docs/source/user_guide/file_index.rst index 44955689e..702a68879 100644 --- a/docs/source/user_guide/file_index.rst +++ b/docs/source/user_guide/file_index.rst @@ -18,6 +18,40 @@ File Index ========== +Attach indexes to existing data files +------------------------------------ + +``FileIndexWrite::PrepareCommit`` supports managed append tables with fixed buckets. +It merges serialized payloads into one external index file per data file and returns +commit messages. Inputs use original file field names and physical row positions, +including DV-deleted rows. Given a serialized bitmap stream ``payload``: + +.. code-block:: cpp + + #include "paimon/file_index/file_index_write.h" + + std::vector inputs = { + {"data-example.parquet", {{"original_field_name", "bitmap", payload}}}}; + PAIMON_ASSIGN_OR_RAISE( + std::vector> messages, + paimon::FileIndexWrite::PrepareCommit( + table_path, branch, options, base_snapshot_id, inputs, + file_system, pool, executor)); + +Use the payloads' snapshot as ``base_snapshot_id``; ``std::nullopt`` selects latest +once. Null file system, pool or executor selects internal defaults. Input streams +are borrowed synchronously and are not closed. + +Existing targets are reused without reading or writing index payloads, and commit +messages are still prepared. Otherwise, index files are published from UUID temporary +files using ``Rename``. If the target appears meanwhile, it is reused and the temporary +file is cleaned up best-effort. Existing contents are not validated. +Concurrent publication requires atomic rename without overwrite from the file system. + +Commit the messages with ``FileStoreCommit`` to the same table and branch. +Serialize backfill commits with compaction/overwrite per partition/bucket, +keeping base metadata valid through commit. + Full-text predicates and root TopK ---------------------------------- diff --git a/include/paimon/file_index/file_index_write.h b/include/paimon/file_index/file_index_write.h new file mode 100644 index 000000000..bc7e6fe65 --- /dev/null +++ b/include/paimon/file_index/file_index_write.h @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "paimon/commit_message.h" +#include "paimon/result.h" +#include "paimon/visibility.h" + +namespace paimon { +class Executor; +class FileSystem; +class InputStream; +class MemoryPool; + +/// Serialized index payload using the data file's original field name. +struct PAIMON_EXPORT FileIndexInput { + std::string field_name; + std::string index_type; + /// Raw payload read from [0, Length()); nullptr denotes an empty index. + /// Non-null inputs must be nonempty. Borrowed synchronously, never closed or retained. + std::shared_ptr input; +}; + +/// Index updates for one live data file in the base snapshot. +struct PAIMON_EXPORT DataFileIndexInput { + std::string data_file_name; + std::vector indexes; +}; + +/// Prepares file-index updates for fixed-bucket append tables. +/// Serialize backfill commits with compaction/overwrite per partition/bucket; +/// keep base metadata valid through commit. +class PAIMON_EXPORT FileIndexWrite { + public: + FileIndexWrite() = delete; + ~FileIndexWrite() = delete; + + /// Writes one external container per file, preserving indexes except supplied field/type pairs. + /// Ignores the embedding threshold; preserves other extra files and historical index files. + /// Reuses existing targets without reading or writing index payloads; still prepares messages. + /// Otherwise publishes UUID temporary files via Rename. If the target appears meanwhile, + /// reuses it and cleans up the temporary file best-effort. Existing contents are not validated. + /// Concurrent publication requires atomic rename without overwrite from the file system. + /// Payloads must use physical row positions, including DV-deleted rows. Do not use input + /// streams concurrently. Null file system, pool or executor uses the corresponding default. + /// + /// @param branch Target branch; empty selects main. + /// @param options Table option overrides; bucket count cannot change. + /// @param base_snapshot_id Base snapshot; nullopt pins latest once for the batch. + /// @param files Unique live file names and field/type pairs; empty index lists are skipped. + /// @return Messages grouped by partition/bucket for commit to the same table and branch. + /// New files are cleaned up best-effort on failure; callers handle commit recovery and orphans. + static Result>> PrepareCommit( + const std::string& table_path, const std::string& branch, + const std::map& options, + const std::optional& base_snapshot_id, + const std::vector& files, + const std::shared_ptr& file_system, const std::shared_ptr& pool, + const std::shared_ptr& executor); +}; + +} // namespace paimon diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 88a9a36f8..4e5e690bf 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -407,6 +407,7 @@ set(PAIMON_CORE_SRCS core/operation/commit/retry_waiter.cpp core/operation/commit_context.cpp core/operation/expire_snapshots.cpp + core/operation/file_index_write.cpp core/operation/file_store_commit.cpp core/operation/file_store_commit_impl.cpp core/operation/file_store_scan.cpp @@ -1026,6 +1027,7 @@ if(PAIMON_BUILD_TESTS) core/operation/commit_metrics_test.cpp core/operation/commit_context_test.cpp core/operation/expire_snapshots_test.cpp + core/operation/file_index_write_test.cpp core/operation/file_store_commit_impl_test.cpp core/operation/file_store_commit_test.cpp core/operation/append_only_file_store_scan_test.cpp diff --git a/src/paimon/common/file_index/file_index_format_utils.cpp b/src/paimon/common/file_index/file_index_format_utils.cpp index d42344e34..a23c4438a 100644 --- a/src/paimon/common/file_index/file_index_format_utils.cpp +++ b/src/paimon/common/file_index/file_index_format_utils.cpp @@ -29,6 +29,9 @@ namespace paimon { Result FileIndexFormatUtils::ReadIndexEntries( const std::shared_ptr& input, const std::shared_ptr& pool) { + if (!input || !pool) { + return Status::Invalid("File index metadata requires an input stream and memory pool"); + } DataInputStream in(input); PAIMON_ASSIGN_OR_RAISE(int64_t magic, in.ReadValue()); if (magic != FileIndexFormat::MAGIC) { diff --git a/src/paimon/core/io/data_file_path_factory.cpp b/src/paimon/core/io/data_file_path_factory.cpp index 1721436df..bf8e94e24 100644 --- a/src/paimon/core/io/data_file_path_factory.cpp +++ b/src/paimon/core/io/data_file_path_factory.cpp @@ -18,10 +18,13 @@ #include "paimon/core/io/data_file_path_factory.h" +#include #include #include +#include "fmt/format.h" #include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/string_utils.h" #include "paimon/common/utils/uuid.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/macros.h" @@ -66,6 +69,26 @@ std::string DataFilePathFactory::ToFileIndexPath(const std::string& file_path) c return PathUtil::JoinPath(parent, PathUtil::GetName(file_path) + INDEX_PATH_SUFFIX); } +std::string DataFilePathFactory::CreateNewFileIndexFilePath(const std::string& file_path) { + std::string file_name = PathUtil::GetName(file_path); + size_t dot = file_name.rfind('.'); + size_t dash = file_name.rfind('-'); + std::string parent = PathUtil::GetParentDirPath(file_path); + if (dash != std::string::npos && dash < dot) { + auto number = + StringUtils::StringToValue(file_name.substr(dash + 1, dot - dash - 1)); + if (number) { + int32_t next = *number == std::numeric_limits::max() + ? std::numeric_limits::min() + : *number + 1; + return PathUtil::JoinPath(parent, fmt::format("{}{}{}", file_name.substr(0, dash + 1), + next, INDEX_PATH_SUFFIX)); + } + } + return PathUtil::JoinPath(parent, + fmt::format("{}-1{}", file_name.substr(0, dot), INDEX_PATH_SUFFIX)); +} + std::string DataFilePathFactory::ToAlignedPath(const std::string& file_name, const std::shared_ptr& aligned) const { auto external_path = aligned->ExternalPathDir(); diff --git a/src/paimon/core/io/data_file_path_factory.h b/src/paimon/core/io/data_file_path_factory.h index 54f4168ef..4ef106f46 100644 --- a/src/paimon/core/io/data_file_path_factory.h +++ b/src/paimon/core/io/data_file_path_factory.h @@ -84,6 +84,7 @@ class DataFilePathFactory : public PathFactory { } std::string ToFileIndexPath(const std::string& file_path) const; + static std::string CreateNewFileIndexFilePath(const std::string& file_path); std::string ToAlignedPath(const std::string& file_name, const std::shared_ptr& aligned) const; diff --git a/src/paimon/core/io/data_file_path_factory_test.cpp b/src/paimon/core/io/data_file_path_factory_test.cpp index 380acf47a..15f26f89e 100644 --- a/src/paimon/core/io/data_file_path_factory_test.cpp +++ b/src/paimon/core/io/data_file_path_factory_test.cpp @@ -19,7 +19,9 @@ #include "paimon/core/io/data_file_path_factory.h" #include +#include #include +#include #include "gtest/gtest.h" #include "paimon/common/data/binary_row.h" @@ -115,6 +117,25 @@ TEST_F(DataFilePathFactoryTest, TestToFileIndexPath) { ASSERT_EQ(index_path, "/tmp/example.txt.index"); } +TEST_F(DataFilePathFactoryTest, TestCreateNewFileIndexFilePath) { + for (const auto& [before, after] : std::vector>{ + {"example.txt.index", "example.txt-1.index"}, + {"data-uuid-0.orc.index", "data-uuid-0.orc-1.index"}, + {"data-uuid-0.orc-1.index", "data-uuid-0.orc-2.index"}, + {"example-9.index", "example-10.index"}, + {"example.index", "example-1.index"}, + {"example-other.index", "example-other-1.index"}, + {"example-2147483647.index", "example--2147483648.index"}, + {"example-2147483648.index", "example-2147483648-1.index"}}) { + for (const std::string& parent : + std::vector{"/tmp", "file:/external/bucket-0"}) { + ASSERT_EQ(PathUtil::JoinPath(parent, after), + DataFilePathFactory::CreateNewFileIndexFilePath( + PathUtil::JoinPath(parent, before))); + } + } +} + TEST_F(DataFilePathFactoryTest, TestToAlignedPath) { auto file_meta = std::make_shared( "data-0.txt", /*file_size=*/645, /*row_count=*/5, BinaryRow::EmptyRow(), diff --git a/src/paimon/core/manifest/file_entry.h b/src/paimon/core/manifest/file_entry.h index 30705a853..81a660abf 100644 --- a/src/paimon/core/manifest/file_entry.h +++ b/src/paimon/core/manifest/file_entry.h @@ -17,18 +17,23 @@ */ #pragma once +#include #include #include +#include +#include #include #include #include #include #include "fmt/format.h" +#include "fmt/ranges.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/utils/linked_hash_map.h" #include "paimon/common/utils/murmurhash_utils.h" #include "paimon/core/manifest/file_kind.h" +#include "paimon/memory/bytes.h" #include "paimon/status.h" namespace paimon { @@ -40,16 +45,25 @@ class FileEntry { /// file. struct Identifier { Identifier(const BinaryRow& _partition, int32_t _bucket, int32_t _level, - const std::string& _file_name, const std::optional& _external_path) + const std::string& _file_name, + const std::vector>& _extra_files, + const std::shared_ptr& _embedded_index, + const std::optional& _external_path) : partition(_partition), bucket(_bucket), level(_level), file_name(_file_name), + extra_files(_extra_files), + embedded_index(_embedded_index), external_path(_external_path) {} bool operator==(const Identifier& other) const { return partition == other.partition && bucket == other.bucket && level == other.level && - file_name == other.file_name && external_path == other.external_path; + file_name == other.file_name && extra_files == other.extra_files && + (embedded_index == other.embedded_index || + (embedded_index && other.embedded_index && + *embedded_index == *other.embedded_index)) && + external_path == other.external_path; } size_t HashCode() const { @@ -61,31 +75,61 @@ class FileEntry { hash_ = MurmurHashUtils::HashUnsafeBytes(reinterpret_cast(&level), /*offset=*/0, sizeof(level), /*seed=*/hash_); + auto hash_bytes = [this](const char* data, size_t length) { + while (length > 0) { + int32_t chunk_size = static_cast( + std::min(length, std::numeric_limits::max())); + hash_ = MurmurHashUtils::HashUnsafeBytes(data, /*offset=*/0, chunk_size, + /*seed=*/hash_); + data += chunk_size; + length -= chunk_size; + } + }; + hash_bytes(file_name.data(), file_name.size()); + for (const auto& extra_file : extra_files) { + bool present = extra_file.has_value(); + hash_ = MurmurHashUtils::HashUnsafeBytes(&present, /*offset=*/0, + sizeof(present), /*seed=*/hash_); + if (extra_file) { + hash_bytes(extra_file->data(), extra_file->size()); + } + } + bool has_embedded_index = embedded_index != nullptr; hash_ = MurmurHashUtils::HashUnsafeBytes( - reinterpret_cast(file_name.data()), - /*offset=*/0, file_name.size(), - /*seed=*/hash_); + &has_embedded_index, /*offset=*/0, sizeof(has_embedded_index), /*seed=*/hash_); + if (embedded_index) { + hash_bytes(embedded_index->data(), embedded_index->size()); + } + bool has_external_path = external_path.has_value(); + hash_ = MurmurHashUtils::HashUnsafeBytes(&has_external_path, /*offset=*/0, + sizeof(has_external_path), /*seed=*/hash_); if (external_path) { - hash_ = MurmurHashUtils::HashUnsafeBytes( - reinterpret_cast(external_path.value().data()), - /*offset=*/0, external_path.value().size(), - /*seed=*/hash_); + hash_bytes(external_path->data(), external_path->size()); } } return hash_; } std::string ToString() const { + std::vector extra_file_names; + for (const auto& extra_file : extra_files) { + extra_file_names.push_back(extra_file.value_or("null")); + } + std::string embedded_index_str = + embedded_index ? fmt::format("{} bytes", embedded_index->size()) : "null"; std::string external_path_str = external_path == std::nullopt ? "null" : external_path.value(); - return fmt::format("{{{}, {}, {}, {}, {}}}", partition.ToString(), bucket, level, - file_name, external_path_str); + return fmt::format("{{{}, {}, {}, {}, [{}], {}, {}}}", partition.ToString(), bucket, + level, file_name, fmt::join(extra_file_names, ", "), + embedded_index_str, external_path_str); } BinaryRow partition; int32_t bucket; int32_t level; std::string file_name; + std::vector> extra_files; + std::shared_ptr embedded_index; std::optional external_path; private: diff --git a/src/paimon/core/manifest/file_entry_test.cpp b/src/paimon/core/manifest/file_entry_test.cpp index 83f592961..6fd52628e 100644 --- a/src/paimon/core/manifest/file_entry_test.cpp +++ b/src/paimon/core/manifest/file_entry_test.cpp @@ -21,12 +21,16 @@ #include #include #include +#include #include #include "gtest/gtest.h" #include "paimon/common/data/binary_row_writer.h" #include "paimon/common/data/binary_string.h" +#include "paimon/core/io/data_file_meta.h" #include "paimon/core/manifest/file_kind.h" +#include "paimon/core/manifest/manifest_entry.h" +#include "paimon/memory/bytes.h" #include "paimon/memory/memory_pool.h" #include "paimon/testing/utils/testharness.h" @@ -65,7 +69,7 @@ class MockFileEntry : public FileEntry { return external_path_; } Identifier CreateIdentifier() const override { - return Identifier(partition_, bucket_, level_, file_name_, external_path_); + return Identifier(partition_, bucket_, level_, file_name_, {}, nullptr, external_path_); } const BinaryRow& MinKey() const override { assert(false); @@ -190,4 +194,71 @@ TEST_F(FileEntryTest, TestMergeEntriesDeleteAndAdd) { ASSERT_NOK(FileEntry::MergeEntries(entries, &merged_entries)); } +TEST_F(FileEntryTest, TestIndexMetadataIdentifiers) { + auto pool = GetDefaultPool(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr file, + DataFileMeta::ForAppend("file1", 1024, 10, SimpleStats::EmptyStats(), 1, 1, + 0, FileSource::Append(), std::nullopt, + std::nullopt, std::nullopt, std::nullopt)); + const BinaryRow partition = GetPartition("0"); + auto identifier = [&](const std::shared_ptr& meta) { + return ManifestEntry(FileKind::Add(), partition, 0, 1, meta).CreateIdentifier(); + }; + file->extra_files = {"shared.aux", "old.index"}; + file->embedded_index = std::make_shared(std::string("\0\xff", 2), pool.get()); + auto equal_file = file->CopyWithExtraFiles(file->extra_files); + equal_file->embedded_index = std::make_shared(std::string("\0\xff", 2), pool.get()); + ASSERT_NE(file->embedded_index, equal_file->embedded_index); + ASSERT_EQ(identifier(file), identifier(equal_file)); + ASSERT_EQ(identifier(file).HashCode(), identifier(equal_file).HashCode()); + + std::unordered_set identifiers = {identifier(file)}; + ASSERT_FALSE(identifiers.insert(identifier(equal_file)).second); + ASSERT_TRUE( + identifiers.insert(identifier(file->CopyWithExtraFiles({"shared.aux", "new.index"}))) + .second); + ASSERT_TRUE( + identifiers.insert(identifier(file->CopyWithExtraFiles({"old.index", "shared.aux"}))) + .second); + ASSERT_TRUE(identifiers.insert(identifier(file->CopyWithExtraFiles({std::nullopt}))).second); + ASSERT_TRUE(identifiers.insert(identifier(file->CopyWithExtraFiles({""}))).second); + + auto changed_file = file->CopyWithExtraFiles(file->extra_files); + changed_file->embedded_index = std::make_shared(std::string("\0\xfe", 2), pool.get()); + ASSERT_TRUE(identifiers.insert(identifier(changed_file)).second); + changed_file->embedded_index = Bytes::EmptyBytes(); + ASSERT_TRUE(identifiers.insert(identifier(changed_file)).second); + changed_file->embedded_index.reset(); + ASSERT_TRUE(identifiers.insert(identifier(changed_file)).second); +} + +TEST_F(FileEntryTest, TestMergeIndexMetadataRewrite) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr before, + DataFileMeta::ForAppend("file1", 1024, 10, SimpleStats::EmptyStats(), 1, 1, + 0, FileSource::Append(), std::nullopt, + std::nullopt, std::nullopt, std::nullopt)); + const BinaryRow partition = GetPartition("0"); + for (bool embedded : {false, true}) { + before->extra_files = embedded ? std::vector>{} + : std::vector>{"old.index"}; + before->embedded_index = + embedded ? std::make_shared("old", GetDefaultPool().get()) : nullptr; + auto after = before->CopyWithExtraFiles({"new.index"}); + after->embedded_index.reset(); + std::vector delta = {{FileKind::Delete(), partition, 0, 1, before}, + {FileKind::Add(), partition, 0, 1, after}}; + std::vector merged; + ASSERT_OK(FileEntry::MergeEntries(delta, &merged)); + ASSERT_EQ(2u, merged.size()); + + std::vector history = {{FileKind::Add(), partition, 0, 1, before}}; + history.insert(history.end(), delta.begin(), delta.end()); + merged.clear(); + ASSERT_OK(FileEntry::MergeEntries(history, &merged)); + ASSERT_EQ(1u, merged.size()); + ASSERT_EQ(FileKind::Add(), merged[0].Kind()); + ASSERT_EQ(*after, *merged[0].File()); + } +} + } // namespace paimon::test diff --git a/src/paimon/core/manifest/manifest_entry.h b/src/paimon/core/manifest/manifest_entry.h index f75461134..1507c9952 100644 --- a/src/paimon/core/manifest/manifest_entry.h +++ b/src/paimon/core/manifest/manifest_entry.h @@ -122,6 +122,7 @@ class ManifestEntry : public FileEntry { FileEntry::Identifier CreateIdentifier() const override { return FileEntry::Identifier(partition_, bucket_, file_->level, file_->file_name, + file_->extra_files, file_->embedded_index, file_->external_path); } diff --git a/src/paimon/core/operation/commit/conflict_detection_test.cpp b/src/paimon/core/operation/commit/conflict_detection_test.cpp index 47e57514e..9f43c28ac 100644 --- a/src/paimon/core/operation/commit/conflict_detection_test.cpp +++ b/src/paimon/core/operation/commit/conflict_detection_test.cpp @@ -261,6 +261,35 @@ TEST_F(ConflictDetectionTest, TestEmptyAndUnrelatedEntriesSkipRowIdChecks) { ASSERT_OK(detection.CheckGlobalIndexRowIdExistence({}, {deleted_index, dv_index})); } +TEST_F(ConflictDetectionTest, TestIndexMetadataRewrite) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr schema, + TableSchema::Create(0, arrow::schema(fields_), {"f1"}, {}, {{Options::BUCKET, "2"}})); + ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(schema->Options())); + ConflictDetection detection(schema, options, nullptr, nullptr, nullptr, nullptr, "test-user", + "test-table", nullptr); + auto original = CreateManifestEntry("file1", FileKind::Add()); + for (bool embedded : {false, true}) { + original.File()->extra_files = embedded + ? std::vector>{} + : std::vector>{"old.index"}; + original.File()->embedded_index = + embedded ? std::make_shared("old", GetDefaultPool().get()) : nullptr; + auto indexed = original.File()->CopyWithExtraFiles({"new.index"}); + indexed->embedded_index.reset(); + std::vector changes = { + {FileKind::Delete(), original.Partition(), 0, 2, original.File()}, + {FileKind::Add(), original.Partition(), 0, 2, indexed}}; + ASSERT_OK(CheckConflicts(detection, {original}, changes, Snapshot::CommitKind::Compact())); + + auto stale = original.File()->CopyWithExtraFiles({"stale.index"}); + changes[0] = ManifestEntry(FileKind::Delete(), original.Partition(), 0, 2, stale); + ASSERT_NOK_WITH_MSG( + CheckConflicts(detection, {original}, changes, Snapshot::CommitKind::Compact()), + "Trying to delete file file1"); + } +} + TEST_F(ConflictDetectionTest, TestRowIdChecksAreDisabledWithoutDataEvolution) { ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, TableSchema::Create(0, arrow::schema(fields_), {"f1"}, {}, {})); diff --git a/src/paimon/core/operation/commit/realtime_deduplicate_compaction_changes_provider.cpp b/src/paimon/core/operation/commit/realtime_deduplicate_compaction_changes_provider.cpp index da115ca2b..157320b68 100644 --- a/src/paimon/core/operation/commit/realtime_deduplicate_compaction_changes_provider.cpp +++ b/src/paimon/core/operation/commit/realtime_deduplicate_compaction_changes_provider.cpp @@ -338,11 +338,8 @@ Status RealtimeDeduplicateCompactionChangesProvider::MaterializeBucketDeletions( update.final_file_names.insert(entry.FileName()); } } - // TODO(lxy): Align FileEntry::Identifier with Java by including extra_files and - // embedded_index to support file-index rebuild commits through normal DELETE/ADD - // entries. For unchanged physical files and row positions, preserve the latest DV - // and skip redundant offset reads, lookups, and DV writes as a performance - // optimization. + // TODO(lxy): For index-only rewrites, preserve the latest DV and skip redundant offset + // reads, lookups, and DV writes. // Load latest bucket DVs and collect deleted offsets from the original inputs. std::map> baseline; PAIMON_ASSIGN_OR_RAISE(baseline, diff --git a/src/paimon/core/operation/file_index_write.cpp b/src/paimon/core/operation/file_index_write.cpp new file mode 100644 index 000000000..92b74ec7f --- /dev/null +++ b/src/paimon/core/operation/file_index_write.cpp @@ -0,0 +1,428 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/file_index/file_index_write.h" + +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/common/file_index/file_index_format_utils.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/scope_guard.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/core_options.h" +#include "paimon/core/io/compact_increment.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/io/data_file_path_factory.h" +#include "paimon/core/io/data_increment.h" +#include "paimon/core/manifest/manifest_entry.h" +#include "paimon/core/operation/append_only_file_store_scan.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/bucket_mode.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/table/sink/commit_message_impl.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/field_mapping.h" +#include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/defs.h" +#include "paimon/executor.h" +#include "paimon/file_index/file_index_format.h" +#include "paimon/fs/file_system.h" +#include "paimon/io/byte_array_input_stream.h" +#include "paimon/memory/bytes.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/scan_context.h" +#include "paimon/status.h" + +namespace paimon { +namespace { + +// Field name -> index type -> payload writer; an empty writer denotes an empty index. +using IndexPayloads = + std::map>; + +using FileIndexRequests = std::map; + +struct FileIndexWriteContext { + Snapshot snapshot; + std::shared_ptr schema; + std::shared_ptr schema_manager; + std::shared_ptr snapshot_manager; + CoreOptions options; +}; + +struct BucketUpdates { + BinaryRow partition; + int32_t bucket; + int32_t total_buckets; + std::vector> before; + std::vector> after; +}; + +FileIndexFormat::Writer::PayloadWriter CopyPayload(const std::shared_ptr& input, + int64_t offset, int64_t length, + const std::shared_ptr& pool) { + return [input, offset, length, pool](OutputStream* output) -> Result { + if (length == 0) { + return true; + } + constexpr int64_t kCopyBufferSize = 1024 * 1024; + auto buffer = Bytes::AllocateBytes(std::min(length, kCopyBufferSize), pool.get()); + PAIMON_RETURN_NOT_OK(input->Seek(offset, FS_SEEK_SET)); + int64_t remaining = length; + while (remaining > 0) { + int64_t size = std::min(remaining, kCopyBufferSize); + PAIMON_ASSIGN_OR_RAISE(int64_t read, input->Read(buffer->data(), size)); + if (read <= 0 || read > size) { + return Status::IOError("Could not read the complete file index payload"); + } + PAIMON_ASSIGN_OR_RAISE(int64_t written, output->Write(buffer->data(), read)); + if (written != read) { + return Status::IOError("Could not write the complete file index payload"); + } + remaining -= read; + } + return true; + }; +} + +Result ReadExistingIndexes(const std::shared_ptr& file, + const std::optional& source_path, + const std::shared_ptr& fs, + const std::shared_ptr& pool, + std::shared_ptr* source) { + if (file->embedded_index) { + *source = std::make_shared(file->embedded_index->data(), + file->embedded_index->size()); + } else if (source_path) { + PAIMON_ASSIGN_OR_RAISE(*source, fs->Open(*source_path)); + } else { + return IndexPayloads(); + } + const auto& input = *source; + PAIMON_ASSIGN_OR_RAISE(FileIndexFormatUtils::FileIndexEntries entries, + FileIndexFormatUtils::ReadIndexEntries(input, pool)); + PAIMON_ASSIGN_OR_RAISE(int64_t file_length, input->Length()); + IndexPayloads payloads; + for (const auto& [column, indexes] : entries) { + for (const auto& [type, range] : indexes) { + const auto& [offset, length] = range; + if (offset == FileIndexFormat::EMPTY_INDEX_FLAG && length == 0) { + payloads[column][type] = {}; + } else { + if (offset < 0 || length < 0 || offset > file_length || + length > file_length - offset) { + return Status::Invalid("Invalid existing file index payload range"); + } + payloads[column][type] = CopyPayload(input, offset, length, pool); + } + } + } + return payloads; +} + +Result> WriteFileIndexes( + const DataFileIndexInput& request, const std::shared_ptr& file, + const std::shared_ptr& paths, int32_t format_version, + const std::shared_ptr& fs, const std::shared_ptr& pool, + std::vector* generated) { + std::optional source_path; + std::vector> extras; + for (const auto& extra : file->extra_files) { + if (extra && StringUtils::EndsWith(*extra, DataFilePathFactory::INDEX_PATH_SUFFIX)) { + if (source_path) { + return Status::Invalid("Found more than one index file for one data file"); + } + source_path = paths->ToAlignedPath(*extra, file); + } else { + extras.push_back(extra); + } + } + std::string index_path = source_path + ? DataFilePathFactory::CreateNewFileIndexFilePath(*source_path) + : paths->ToFileIndexPath(paths->ToPath(file)); + PAIMON_ASSIGN_OR_RAISE(bool use_existing, fs->Exists(index_path)); + extras.emplace_back(PathUtil::GetName(index_path)); + auto indexed_file = file->CopyWithExtraFiles(extras); + indexed_file->embedded_index.reset(); + if (use_existing) { + return indexed_file; + } + std::shared_ptr source; + ScopeGuard close_source([&source]() { + if (source) { + [[maybe_unused]] Status ignored = source->Close(); + } + }); + PAIMON_ASSIGN_OR_RAISE(IndexPayloads payloads, + ReadExistingIndexes(file, source_path, fs, pool, &source)); + for (const auto& index : request.indexes) { + if (!index.input) { + payloads[index.field_name][index.index_type] = {}; + } else { + PAIMON_ASSIGN_OR_RAISE(int64_t length, index.input->Length()); + if (length <= 0) { + return Status::Invalid("A non-null file index input must have positive length"); + } + payloads[index.field_name][index.index_type] = + CopyPayload(index.input, 0, length, pool); + } + } + PAIMON_ASSIGN_OR_RAISE(std::string temporary_path, PathUtil::CreateTempPath(index_path)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr output, + fs->Create(temporary_path, /*overwrite=*/false)); + generated->push_back(temporary_path); + ScopeGuard close_output([&output]() { [[maybe_unused]] Status ignored = output->Close(); }); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, + FileIndexFormat::CreateWriter(output, format_version, pool)); + for (const auto& [column, indexes] : payloads) { + for (const auto& [type, payload] : indexes) { + PAIMON_RETURN_NOT_OK(writer->WriteIndex(column, type, payload)); + } + } + PAIMON_RETURN_NOT_OK(writer->Finish()); + PAIMON_RETURN_NOT_OK(writer->Close()); + close_output.Release(); + PAIMON_ASSIGN_OR_RAISE(use_existing, fs->Exists(index_path)); + if (!use_existing) { + Status renamed = fs->Rename(temporary_path, index_path); + if (!renamed.ok()) { + auto destination_exists = fs->Exists(index_path); + if (!renamed.IsExist() && !(destination_exists.ok() && destination_exists.value())) { + return renamed; + } + use_existing = true; + } else { + generated->back() = index_path; + } + } + if (use_existing) { + // Reuse the existing target without claiming it for failure cleanup. + [[maybe_unused]] Status ignored = fs->Delete(temporary_path, /*recursive=*/false); + generated->pop_back(); + } + return indexed_file; +} + +Result> FindFiles( + const FileStore& store, const std::shared_ptr& schema, + const std::shared_ptr& schema_manager, + const std::shared_ptr& snapshot_manager, const Snapshot& snapshot, + const std::shared_ptr& paths, + const std::map& requests, + const std::shared_ptr& executor, const std::shared_ptr& pool) { + auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr partition_schema, + FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_list, + store.NewManifestList(paths, pool)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_file, + store.NewManifestFile(paths, partition_schema, pool)); + auto filter = std::make_shared( + nullptr, std::vector>{}, std::nullopt); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, + AppendOnlyFileStoreScan::Create( + snapshot_manager, schema_manager, manifest_list, manifest_file, + schema, arrow_schema, filter, store.GetOptions(), executor, pool)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, + scan->WithSnapshot(snapshot)->CreatePlan()); + std::map files; + for (const auto& entry : plan->Files()) { + if (requests.count(entry.FileName()) == 0) { + continue; + } + if (!files.emplace(entry.FileName(), entry).second) { + return Status::Invalid( + fmt::format("Ambiguous data file name in the base snapshot: {}", entry.FileName())); + } + if (entry.Bucket() < 0 || entry.TotalBuckets() <= 0 || + entry.Bucket() >= entry.TotalBuckets()) { + return Status::Invalid("FileIndexWrite requires data files in fixed buckets"); + } + } + for (const auto& request : requests) { + if (files.count(request.first) == 0) { + return Status::Invalid(fmt::format( + "File index target is not live in the base snapshot: {}", request.first)); + } + } + return files; +} + +Result LoadWriteContext(const std::string& table_path, + const std::string& branch, + const std::map& options, + const std::optional& base_snapshot_id, + const std::shared_ptr& file_system) { + if (table_path.empty()) { + return Status::Invalid("FileIndexWrite requires a table path"); + } + if (base_snapshot_id && *base_snapshot_id < Snapshot::FIRST_SNAPSHOT_ID) { + return Status::Invalid("Invalid base snapshot id"); + } + PAIMON_ASSIGN_OR_RAISE( + std::string resolved_branch, + BranchManager::ResolveBranch(std::nullopt, options, branch, "FileIndexWrite")); + std::shared_ptr fs = file_system; + if (!fs) { + PAIMON_ASSIGN_OR_RAISE(CoreOptions initial_options, CoreOptions::FromMap(options)); + fs = initial_options.GetFileSystem(); + } + auto snapshot_manager = std::make_shared(fs, table_path, resolved_branch); + std::optional snapshot; + if (base_snapshot_id) { + PAIMON_ASSIGN_OR_RAISE(snapshot, snapshot_manager->LoadSnapshot(*base_snapshot_id)); + } else { + PAIMON_ASSIGN_OR_RAISE(snapshot, snapshot_manager->LatestSnapshot()); + } + if (!snapshot) { + return Status::Invalid("FileIndexWrite requires an existing snapshot"); + } + auto schema_manager = std::make_shared(fs, table_path, resolved_branch); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + schema_manager->ReadSchema(snapshot->SchemaId())); + if (!schema->PrimaryKeys().empty() || schema->NumBuckets() <= 0 || + ResolveBucketMode(schema->NumBuckets(), schema) != BucketMode::HASH_FIXED) { + return Status::Invalid("FileIndexWrite only supports append tables with fixed buckets"); + } + std::map overrides = options; + overrides[Options::BRANCH] = resolved_branch; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr configured_schema, + schema->MergeOptions(overrides)); + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, + CoreOptions::FromMap(configured_schema->Options(), file_system)); + if (core_options.GetBucket() != schema->NumBuckets()) { + return Status::Invalid("FileIndexWrite cannot override the table's bucket count"); + } + int32_t format_version = core_options.FileIndexFormatVersion(); + if (format_version != FileIndexFormat::V_1 && format_version != FileIndexFormat::V_2) { + return Status::Invalid("Unsupported file index format version"); + } + return FileIndexWriteContext{std::move(*snapshot), std::move(schema), std::move(schema_manager), + std::move(snapshot_manager), std::move(core_options)}; +} + +Result CollectIndexRequests(const std::vector& files) { + std::set file_names; + FileIndexRequests requests; + for (const auto& file : files) { + PAIMON_RETURN_NOT_OK( + PathUtil::CheckSinglePathComponent("data file name", file.data_file_name)); + if (!file_names.insert(file.data_file_name).second) { + return Status::Invalid("Duplicate data file name in file index inputs"); + } + std::set> keys; + for (const auto& index : file.indexes) { + if (index.field_name.empty() || index.index_type.empty()) { + return Status::Invalid("File index field name and index type must not be empty"); + } + if (!keys.emplace(index.field_name, index.index_type).second) { + return Status::Invalid("Duplicate field and index type in file index inputs"); + } + } + if (!file.indexes.empty()) { + requests.emplace(file.data_file_name, &file); + } + } + return requests; +} + +Result>> WriteIndexesAndBuildMessages( + const FileIndexRequests& requests, const std::map& targets, + const std::shared_ptr& paths, int32_t format_version, + const std::shared_ptr& fs, const std::shared_ptr& memory_pool) { + std::vector> messages; + std::vector generated; + ScopeGuard cleanup([&]() { + for (const auto& path : generated) { + [[maybe_unused]] Status ignored = fs->Delete(path, /*recursive=*/false); + } + }); + std::vector buckets; + std::unordered_map> bucket_positions; + for (const auto& [name, entry] : targets) { + auto& partition_buckets = bucket_positions[entry.Partition()]; + auto [position, inserted] = partition_buckets.emplace(entry.Bucket(), buckets.size()); + if (inserted) { + buckets.push_back({entry.Partition(), entry.Bucket(), entry.TotalBuckets(), {}, {}}); + } + auto& bucket = buckets[position->second]; + if (bucket.total_buckets != entry.TotalBuckets()) { + return Status::Invalid("Inconsistent total bucket counts in file index targets"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_paths, + paths->CreateDataFilePathFactory(entry.Partition(), entry.Bucket())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr indexed_file, + WriteFileIndexes(*requests.at(name), entry.File(), file_paths, + format_version, fs, memory_pool, &generated)); + bucket.before.push_back(entry.File()); + bucket.after.push_back(std::move(indexed_file)); + } + for (auto& bucket : buckets) { + messages.push_back(std::make_shared( + bucket.partition, bucket.bucket, bucket.total_buckets, DataIncrement({}, {}, {}), + CompactIncrement(std::move(bucket.before), std::move(bucket.after), {}))); + } + cleanup.Release(); + return messages; +} + +} // namespace + +Result>> FileIndexWrite::PrepareCommit( + const std::string& table_path, const std::string& branch, + const std::map& options, + const std::optional& base_snapshot_id, const std::vector& files, + const std::shared_ptr& file_system, const std::shared_ptr& pool, + const std::shared_ptr& executor) { + PAIMON_ASSIGN_OR_RAISE( + FileIndexWriteContext context, + LoadWriteContext(table_path, branch, options, base_snapshot_id, file_system)); + PAIMON_ASSIGN_OR_RAISE(FileIndexRequests requests, CollectIndexRequests(files)); + if (requests.empty()) { + return std::vector>{}; + } + std::shared_ptr memory_pool = pool ? pool : GetDefaultPool(); + std::shared_ptr scan_executor = executor; + if (!scan_executor) { + scan_executor = CreateDefaultExecutor(); + } + FileStore store(table_path, context.options); + auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(context.schema->Fields()); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr paths, + store.NewPathFactory(arrow_schema, context.schema->PartitionKeys(), + context.options.GetFileFormat()->Identifier(), memory_pool)); + std::map targets; + PAIMON_ASSIGN_OR_RAISE( + targets, FindFiles(store, context.schema, context.schema_manager, context.snapshot_manager, + context.snapshot, paths, requests, scan_executor, memory_pool)); + return WriteIndexesAndBuildMessages(requests, targets, paths, + context.options.FileIndexFormatVersion(), + context.options.GetFileSystem(), memory_pool); +} + +} // namespace paimon diff --git a/src/paimon/core/operation/file_index_write_test.cpp b/src/paimon/core/operation/file_index_write_test.cpp new file mode 100644 index 000000000..105e9e6e9 --- /dev/null +++ b/src/paimon/core/operation/file_index_write_test.cpp @@ -0,0 +1,1315 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/file_index/file_index_write.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "arrow/c/helpers.h" +#include "arrow/ipc/json_simple.h" +#include "fmt/format.h" +#include "gtest/gtest.h" +#include "paimon/commit_context.h" +#include "paimon/common/file_index/file_index_format_utils.h" +#include "paimon/common/io/data_output_stream.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/scope_guard.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/core_options.h" +#include "paimon/core/io/compact_increment.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/io/data_file_path_factory.h" +#include "paimon/core/io/data_increment.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/sink/commit_message_impl.h" +#include "paimon/core/table/source/data_split_impl.h" +#include "paimon/core/utils/file_store_path_factory_cache.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/defs.h" +#include "paimon/executor.h" +#include "paimon/file_index/file_index_format.h" +#include "paimon/file_index/file_indexer.h" +#include "paimon/file_index/file_indexer_factory.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/io/byte_array_input_stream.h" +#include "paimon/io/data_input_stream.h" +#include "paimon/memory/bytes.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/predicate/predicate_builder.h" +#include "paimon/read_context.h" +#include "paimon/scan_context.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/testing/mock/mock_file_system.h" +#include "paimon/testing/utils/deletion_vector_test_helper.h" +#include "paimon/testing/utils/read_result_collector.h" +#include "paimon/testing/utils/test_helper.h" +#include "paimon/testing/utils/testharness.h" +#include "paimon/write_context.h" + +namespace paimon::test { +namespace { + +using Messages = std::vector>; +// Outer key: original data-file field name; value: indexes for that field. +// Inner key: index type (e.g. bitmap or bsi); value: raw index payload bytes. +// std::nullopt represents an explicit empty index. +using PayloadBytes = std::map>>; + +class PayloadInput : public MockInputStream { + public: + explicit PayloadInput(std::string bytes) : bytes_(std::move(bytes)) {} + + using MockInputStream::Read; + + Status Seek(int64_t offset, SeekOrigin origin) override { + if (fail_seek) { + return Status::IOError("injected payload seek failure"); + } + int64_t start = + origin == FS_SEEK_SET + ? 0 + : (origin == FS_SEEK_CUR ? position_ : static_cast(bytes_.size())); + if (offset < -start || offset > static_cast(bytes_.size()) - start) { + return Status::Invalid("payload seek out of bounds"); + } + position_ = start + offset; + return Status::OK(); + } + + Result GetPos() const override { + return position_; + } + + Result Length() const override { + if (fail_length) { + return Status::IOError("injected payload length failure"); + } + return bytes_.size(); + } + + Result Read(char* buffer, int64_t size) override { + if (fail_read) { + return Status::IOError("injected payload read failure"); + } + if (early_eof) { + return 0; + } + int64_t count = std::min({size, max_read, static_cast(bytes_.size()) - position_}); + std::memcpy(buffer, bytes_.data() + position_, count); + position_ += count; + return count; + } + + Status Close() override { + ++close_count; + return Status::OK(); + } + + const std::string& BytesValue() const { + return bytes_; + } + + bool fail_length = false; + bool fail_seek = false; + bool fail_read = false; + bool early_eof = false; + int64_t max_read = std::numeric_limits::max(); + int32_t close_count = 0; + + private: + std::string bytes_; + int64_t position_ = 0; +}; + +enum class OutputFailure { NONE, CREATE, WRITE, FLUSH, CLOSE, RENAME }; + +class IndexOutput : public OutputStream { + public: + IndexOutput(std::unique_ptr output, OutputFailure failure) + : output_(std::move(output)), failure_(failure) {} + + Result Write(const char* buffer, int64_t size) override { + if (failure_ == OutputFailure::WRITE) { + return Status::IOError("injected index write failure"); + } + return output_->Write(buffer, size); + } + + Result GetPos() const override { + return output_->GetPos(); + } + + Result GetUri() const override { + return output_->GetUri(); + } + + Status Flush() override { + if (failure_ == OutputFailure::FLUSH) { + return Status::IOError("injected index flush failure"); + } + return output_->Flush(); + } + + Status Close() override { + PAIMON_RETURN_NOT_OK(output_->Close()); + if (failure_ == OutputFailure::CLOSE) { + return Status::IOError("injected index close failure"); + } + return Status::OK(); + } + + private: + std::unique_ptr output_; + OutputFailure failure_; +}; + +class IndexFileSystem : public LocalFileSystem { + public: + Result> Create(const std::string& path, + bool overwrite) const override { + if (!StringUtils::EndsWith(path, ".index") && + !(StringUtils::EndsWith(path, ".tmp") && path.find(".index.") != std::string::npos)) { + return LocalFileSystem::Create(path, overwrite); + } + ++create_attempts; + if (failure == OutputFailure::CREATE) { + return Status::IOError("injected index create failure"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr output, + LocalFileSystem::Create(path, overwrite)); + created.push_back(path); + return std::unique_ptr( + std::make_unique(std::move(output), failure)); + } + + Status Rename(const std::string& src, const std::string& dst) const override { + rename_attempts.emplace_back(src, dst); + if (before_rename) { + PAIMON_RETURN_NOT_OK(before_rename(src, dst)); + } + if (failure == OutputFailure::RENAME) { + return Status::IOError("injected index rename failure"); + } + PAIMON_RETURN_NOT_OK(LocalFileSystem::Rename(src, dst)); + published.push_back(dst); + return Status::OK(); + } + + Status Delete(const std::string& path, bool recursive = true) const override { + delete_attempts.push_back(path); + if (fail_delete) { + return Status::IOError("injected index delete failure"); + } + return LocalFileSystem::Delete(path, recursive); + } + + OutputFailure failure = OutputFailure::NONE; + std::function before_rename; + bool fail_delete = false; + mutable int32_t create_attempts = 0; + mutable std::vector created; + mutable std::vector published; + mutable std::vector> rename_attempts; + mutable std::vector delete_attempts; +}; + +} // namespace + +class FileIndexWriteTest : public ::testing::Test { + protected: + void SetUp() override { + directory_ = UniqueTestDirectory::Create(); + ASSERT_TRUE(directory_); + fs_ = std::make_shared(); + pool_ = GetDefaultPool(); + executor_ = CreateDefaultExecutor(); + schema_ = arrow::schema({arrow::field("part", arrow::int32(), false), + arrow::field("value", arrow::int32(), false), + arrow::field("other", arrow::int32())}); + } + + Status CreateTable(const std::map& overrides = {}, + const std::vector& primary_keys = {}, + const std::string& branch = "main") { + table_path_ = PathUtil::JoinPath(directory_->Str(), fmt::format("table-{}", next_table_++)); + branch_ = branch; + options_ = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::BUCKET, "2"}, + {Options::WRITE_ONLY, "true"}, + {Options::FILE_INDEX_READ_ENABLED, "true"}}; + for (const auto& [key, value] : overrides) { + options_[key] = value; + } + if (options_[Options::BUCKET] != "-1") { + options_.try_emplace(Options::BUCKET_KEY, "value"); + } + options_[Options::BRANCH] = branch; + SchemaManager manager(fs_, table_path_, branch); + PAIMON_ASSIGN_OR_RAISE([[maybe_unused]] std::unique_ptr created, + manager.CreateTable(schema_, {"part"}, primary_keys, options_)); + return Status::OK(); + } + + Status Commit(const Messages& messages) { + CommitContextBuilder builder(table_path_, "file-index-test"); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, + builder.SetOptions(options_).Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, + FileStoreCommit::Create(std::move(context))); + return commit->Commit(messages, next_commit_++); + } + + Status Abort(const Messages& messages) const { + CommitContextBuilder builder(table_path_, "file-index-test"); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, + builder.SetOptions(options_).Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, + FileStoreCommit::Create(std::move(context))); + return commit->Abort(messages); + } + + Result WriteData(const std::string& json = "[[10,1,10],[10,2,20],[10,1,30]]", + int32_t bucket = 0, int32_t partition = 10) { + WriteContextBuilder builder(table_path_, "file-index-test"); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, + builder.SetOptions(options_).Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, + FileStoreWrite::Create(std::move(context))); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(schema_->fields()), json, + {{"part", std::to_string(partition)}}, bucket, {})); + PAIMON_RETURN_NOT_OK(writer->Write(std::move(batch))); + PAIMON_ASSIGN_OR_RAISE(Messages messages, + writer->PrepareCommit(/*wait_compaction=*/true, next_commit_)); + PAIMON_RETURN_NOT_OK(Commit(messages)); + PAIMON_RETURN_NOT_OK(writer->Close()); + return messages; + } + + Result> NewFile(const Messages& messages) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr message, OnlyMessage(messages)); + const auto& files = message->GetNewFilesIncrement().NewFiles(); + if (files.size() != 1) { + return Status::Invalid("expected one new data file"); + } + return files[0]; + } + + Result> UpdatedFile(const Messages& messages) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr message, OnlyMessage(messages)); + const auto& files = message->GetCompactIncrement().CompactAfter(); + if (files.size() != 1) { + return Status::Invalid("expected one updated data file"); + } + return files[0]; + } + + Result> OnlyMessage(const Messages& messages) const { + if (messages.size() != 1) { + return Status::Invalid("expected one commit message"); + } + auto message = std::dynamic_pointer_cast(messages[0]); + if (!message) { + return Status::Invalid("expected CommitMessageImpl"); + } + return message; + } + + Result LatestSnapshot() const { + SnapshotManager manager(fs_, table_path_, branch_); + PAIMON_ASSIGN_OR_RAISE(std::optional snapshot, manager.LatestSnapshot()); + if (!snapshot) { + return Status::Invalid("expected a snapshot"); + } + return *snapshot; + } + + Result Prepare(const std::vector& inputs, + const std::map& overrides = {}, + const std::optional& snapshot = std::nullopt) const { + return FileIndexWrite::PrepareCommit(table_path_, branch_, overrides, snapshot, inputs, fs_, + pool_, executor_); + } + + Result> MakePayload( + const std::string& field = "value", const std::string& type = "bitmap", + const std::string& json = "[[1],[2],[1]]") const { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexer, + FileIndexerFactory::Get(type, {{"version", "2"}})); + if (!indexer) { + return Status::Invalid("missing test indexer"); + } + auto schema = arrow::schema({arrow::field(field, arrow::int32())}); + ::ArrowSchema c_schema; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &c_schema)); + ScopeGuard close_schema([&c_schema]() { ArrowSchemaRelease(&c_schema); }); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr writer, + indexer->CreateWriter(&c_schema, pool_)); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr array, + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema->fields()), json)); + ::ArrowArray c_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array)); + ScopeGuard close_array([&c_array]() { ArrowArrayRelease(&c_array); }); + PAIMON_RETURN_NOT_OK(writer->AddBatch(&c_array)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr bytes, writer->SerializedBytes()); + return std::make_shared(std::string(bytes->data(), bytes->size())); + } + + Result> Paths(int32_t partition = 10, + int32_t bucket = 0) const { + PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, LatestSnapshot()); + SchemaManager manager(fs_, table_path_, branch_); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + manager.ReadSchema(snapshot.SchemaId())); + PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(schema->Options(), fs_)); + FileStorePathFactoryCache cache(table_path_, schema, options, pool_); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr paths, + cache.GetOrCreatePathFactory(options.GetFileFormat()->Identifier())); + PAIMON_ASSIGN_OR_RAISE(BinaryRow partition_row, + paths->ToBinaryRow({{"part", std::to_string(partition)}})); + return paths->CreateDataFilePathFactory(partition_row, bucket); + } + + Result> OpenIndex(const std::shared_ptr& file, + int32_t partition = 10, + int32_t bucket = 0) const { + if (file->embedded_index) { + return std::shared_ptr(std::make_shared( + file->embedded_index->data(), file->embedded_index->size())); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr paths, + Paths(partition, bucket)); + for (const auto& extra : file->extra_files) { + if (extra && StringUtils::EndsWith(*extra, ".index")) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr input, + fs_->Open(paths->ToAlignedPath(*extra, file))); + return input; + } + } + return Status::Invalid("expected an index container"); + } + + Result ReadPayloads(const std::shared_ptr& file, + int32_t partition = 10, int32_t bucket = 0) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr input, + OpenIndex(file, partition, bucket)); + ScopeGuard close([&input]() { [[maybe_unused]] Status ignored = input->Close(); }); + PAIMON_ASSIGN_OR_RAISE(FileIndexFormatUtils::FileIndexEntries entries, + FileIndexFormatUtils::ReadIndexEntries(input, pool_)); + PayloadBytes bytes; + DataInputStream in(input); + for (const auto& [field, indexes] : entries) { + for (const auto& [type, range] : indexes) { + if (range.first == FileIndexFormat::EMPTY_INDEX_FLAG) { + bytes[field][type] = std::nullopt; + } else { + std::string payload(range.second, '\0'); + PAIMON_RETURN_NOT_OK(in.Seek(range.first)); + PAIMON_RETURN_NOT_OK(in.Read(payload.data(), range.second)); + bytes[field][type] = std::move(payload); + } + } + } + return bytes; + } + + Result>> Scan( + const std::optional& snapshot = std::nullopt) const { + ScanContextBuilder builder(table_path_); + builder.SetOptions(options_).WithStreamingMode(false); + if (snapshot) { + builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(*snapshot)); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, + TableScan::Create(std::move(context))); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scan->CreatePlan()); + return plan->Splits(); + } + + Result> ReadValues( + const std::optional& snapshot = std::nullopt, const std::string& field = "value", + const std::shared_ptr& predicate = nullptr) const { + PAIMON_ASSIGN_OR_RAISE(std::vector> splits, Scan(snapshot)); + ReadContextBuilder builder(table_path_); + builder.SetOptions(options_).SetReadFieldNames({field}); + if (predicate) { + builder.SetPredicate(predicate); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, + TableRead::Create(std::move(context))); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, read->CreateReader(splits)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr rows, + ReadResultCollector::CollectResult(std::move(reader))); + std::vector values; + for (const auto& chunk : rows->chunks()) { + const auto* batch = dynamic_cast(chunk.get()); + if (!batch) { + return Status::Invalid("expected struct rows"); + } + auto column = batch->GetFieldByName(field); + const auto* ints = dynamic_cast(column.get()); + if (!ints) { + return Status::Invalid("expected int values"); + } + for (int64_t row = 0; row < ints->length(); ++row) { + values.push_back(ints->Value(row)); + } + } + std::sort(values.begin(), values.end()); + return values; + } + + Status RewriteMetadata(const std::shared_ptr& before, + const std::shared_ptr& after) { + PAIMON_ASSIGN_OR_RAISE(std::vector> splits, Scan()); + if (splits.size() != 1) { + return Status::Invalid("expected one split for metadata rewrite"); + } + auto split = std::dynamic_pointer_cast(splits[0]); + if (!split) { + return Status::Invalid("expected a data split"); + } + return Commit({std::make_shared( + split->Partition(), split->Bucket(), split->TotalBuckets(), DataIncrement({}, {}, {}), + CompactIncrement({before}, {after}, {}))}); + } + + Status WriteContainer(const std::shared_ptr& file, const std::string& name, + const PayloadBytes& payloads, int32_t version = 1) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr paths, Paths()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr output, + fs_->Create(paths->ToAlignedPath(name, file), false)); + ScopeGuard close([&output]() { [[maybe_unused]] Status ignored = output->Close(); }); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, + FileIndexFormat::CreateWriter(output, version, pool_)); + for (const auto& [field, indexes] : payloads) { + for (const auto& [type, payload] : indexes) { + FileIndexFormat::Writer::PayloadWriter copy; + if (payload) { + auto bytes = std::make_shared(*payload, pool_.get()); + copy = [bytes](OutputStream* destination) -> Result { + PAIMON_RETURN_NOT_OK(DataOutputStream(destination).WriteBytes(bytes)); + return true; + }; + } + PAIMON_RETURN_NOT_OK(writer->WriteIndex(field, type, copy)); + } + } + PAIMON_RETURN_NOT_OK(writer->Finish()); + PAIMON_RETURN_NOT_OK(writer->Close()); + close.Release(); + return Status::OK(); + } + + void AssertRemoved(const std::vector& paths) const { + for (const auto& path : paths) { + ASSERT_OK_AND_ASSIGN(bool exists, fs_->Exists(path)); + ASSERT_FALSE(exists) << path; + } + } + + std::unique_ptr directory_; + std::shared_ptr fs_; + std::shared_ptr pool_; + std::shared_ptr executor_; + std::shared_ptr schema_; + std::map options_; + std::string table_path_; + std::string branch_ = "main"; + int32_t next_table_ = 0; + int64_t next_commit_ = 0; +}; + +// Parameters: old/new container format versions; old index location (0=none, 1=embedded, +// 2=external). +class VersionedFileIndexWriteTest + : public FileIndexWriteTest, + public ::testing::WithParamInterface> {}; + +TEST_P(VersionedFileIndexWriteTest, MergeAndCommit) { + const auto& [old_version, new_version, old_location] = GetParam(); + std::map table_options = { + {Options::FILE_INDEX_FORMAT_VERSION, std::to_string(old_version)}}; + if (old_location != 0) { + table_options["file-index.bitmap.columns"] = "value"; + table_options["file-index.bitmap.value.version"] = "1"; + table_options["file-index.bsi.columns"] = "other"; + table_options[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = old_location == 1 ? "1MB" : "0B"; + } + ASSERT_OK(CreateTable(table_options)); + ASSERT_OK_AND_ASSIGN(Messages written, WriteData()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr before, NewFile(written)); + ASSERT_OK_AND_ASSIGN(Snapshot base, LatestSnapshot()); + PayloadBytes old_payloads; + if (old_location != 0) { + ASSERT_EQ(old_location == 1, before->embedded_index != nullptr); + ASSERT_OK_AND_ASSIGN(old_payloads, ReadPayloads(before)); + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr bitmap, MakePayload()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr bsi, MakePayload("value", "bsi")); + ASSERT_OK_AND_ASSIGN(std::shared_ptr other, + MakePayload("other", "bitmap", "[[10],[20],[30]]")); + ASSERT_OK_AND_ASSIGN( + Messages updates, + Prepare( + {{before->file_name, + {{"value", "bitmap", bitmap}, {"value", "bsi", bsi}, {"other", "bitmap", other}}}}, + {{Options::FILE_INDEX_FORMAT_VERSION, std::to_string(new_version)}, + {Options::FILE_INDEX_IN_MANIFEST_THRESHOLD, "1GB"}}, + base.Id())); + ASSERT_OK_AND_ASSIGN(Snapshot uncommitted, LatestSnapshot()); + ASSERT_EQ(base.Id(), uncommitted.Id()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr update, OnlyMessage(updates)); + ASSERT_TRUE(update->GetNewFilesIncrement().IsEmpty()); + ASSERT_EQ(2, update->TotalBuckets()); + const auto& increment = update->GetCompactIncrement(); + ASSERT_EQ(1u, increment.CompactBefore().size()); + ASSERT_EQ(1u, increment.CompactAfter().size()); + ASSERT_TRUE(increment.ChangelogFiles().empty()); + ASSERT_TRUE(increment.NewIndexFiles().empty()); + ASSERT_TRUE(increment.DeletedIndexFiles().empty()); + auto after = increment.CompactAfter()[0]; + ASSERT_FALSE(after->embedded_index); + ASSERT_EQ(1u, after->extra_files.size()); + ASSERT_EQ(before->file_name + (old_location == 2 ? "-1.index" : ".index"), + after->extra_files[0].value()); + auto restored = after->CopyWithExtraFiles(before->extra_files); + restored->embedded_index = before->embedded_index; + ASSERT_EQ(*before, *restored); + ASSERT_OK_AND_ASSIGN(PayloadBytes payloads, ReadPayloads(after)); + ASSERT_EQ(bitmap->BytesValue(), payloads.at("value").at("bitmap")); + ASSERT_EQ(bsi->BytesValue(), payloads.at("value").at("bsi")); + ASSERT_EQ(other->BytesValue(), payloads.at("other").at("bitmap")); + if (old_location != 0) { + ASSERT_NE(old_payloads.at("value").at("bitmap"), payloads.at("value").at("bitmap")); + ASSERT_EQ(old_payloads.at("other").at("bsi"), payloads.at("other").at("bsi")); + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr index, OpenIndex(after)); + DataInputStream in(index); + ASSERT_OK_AND_ASSIGN(int64_t magic, in.ReadValue()); + ASSERT_EQ(FileIndexFormat::MAGIC, magic); + ASSERT_OK_AND_ASSIGN(int32_t version, in.ReadValue()); + ASSERT_EQ(new_version, version); + ASSERT_OK(index->Close()); + ASSERT_OK(Commit(updates)); + ASSERT_OK_AND_ASSIGN(Snapshot committed, LatestSnapshot()); + ASSERT_EQ(base.Id() + 1, committed.Id()); + ASSERT_EQ(Snapshot::CommitKind::Compact(), committed.GetCommitKind()); + ASSERT_OK_AND_ASSIGN(auto splits, Scan()); + ASSERT_EQ(1u, splits.size()); + auto split = std::dynamic_pointer_cast(splits[0]); + ASSERT_TRUE(split); + ASSERT_EQ(1u, split->DataFiles().size()); + ASSERT_EQ(*after, *split->DataFiles()[0]); + ASSERT_OK_AND_ASSIGN( + auto filtered, ReadValues(std::nullopt, "value", + PredicateBuilder::Equal(0, "value", FieldType::INT, Literal(1)))); + ASSERT_EQ((std::vector{1, 1}), filtered); + if (old_location != 0) { + ASSERT_OK_AND_ASSIGN(auto historical_payloads, ReadPayloads(before)); + ASSERT_EQ(old_payloads, historical_payloads); + } +} + +INSTANTIATE_TEST_SUITE_P(AllContainers, VersionedFileIndexWriteTest, + ::testing::Combine(::testing::Values(1, 2), ::testing::Values(1, 2), + ::testing::Values(0, 1, 2))); + +TEST_F(FileIndexWriteTest, GroupsFilesByPartitionAndBucket) { + ASSERT_OK(CreateTable()); + std::vector requests; + std::unordered_map, std::set> expected; + for (const auto& [partition, bucket] : + std::vector>{{10, 0}, {10, 0}, {10, 1}, {20, 0}}) { + ASSERT_OK_AND_ASSIGN(Messages written, + WriteData(fmt::format("[[{},1,10],[{},2,20],[{},1,30]]", partition, + partition, partition), + bucket, partition)); + ASSERT_OK_AND_ASSIGN(auto file, NewFile(written)); + ASSERT_OK_AND_ASSIGN(auto message, OnlyMessage(written)); + expected[{message->Partition(), message->Bucket()}].insert(file->file_name); + requests.push_back( + {file->file_name, {{"value", "opaque", std::make_shared("payload")}}}); + } + ASSERT_OK_AND_ASSIGN(Messages updates, Prepare(requests)); + ASSERT_EQ(expected.size(), updates.size()); + for (const auto& raw : updates) { + auto message = std::dynamic_pointer_cast(raw); + ASSERT_TRUE(message); + std::set names; + ASSERT_EQ(2, message->TotalBuckets()); + for (const auto& file : message->GetCompactIncrement().CompactAfter()) { + names.insert(file->file_name); + ASSERT_EQ(1u, file->extra_files.size()); + } + auto key = std::make_pair(message->Partition(), message->Bucket()); + ASSERT_EQ(expected.at(key), names); + expected.erase(key); + } + ASSERT_TRUE(expected.empty()); + ASSERT_OK(Commit(updates)); + ASSERT_OK_AND_ASSIGN(auto values, ReadValues()); + ASSERT_EQ(12u, values.size()); + ASSERT_EQ(8, std::count(values.begin(), values.end(), 1)); + ASSERT_EQ(4, std::count(values.begin(), values.end(), 2)); +} + +TEST_F(FileIndexWriteTest, PreservesOpaqueIndexesAndExtraFiles) { + ASSERT_OK(CreateTable()); + ASSERT_OK_AND_ASSIGN(Messages written, WriteData()); + ASSERT_OK_AND_ASSIGN(auto file, NewFile(written)); + const PayloadBytes existing = { + {"value", {{"unregistered", "opaque bytes"}, {"empty-unregistered", std::nullopt}}}}; + ASSERT_OK(WriteContainer(file, "old.index", existing)); + ASSERT_OK_AND_ASSIGN(auto paths, Paths()); + const std::string aux_path = paths->ToAlignedPath("shared.aux", file); + ASSERT_OK(fs_->WriteFile(aux_path, "auxiliary data", false)); + auto before = file->CopyWithExtraFiles({"old.index", "shared.aux"}); + ASSERT_OK(RewriteMetadata(file, before)); + ASSERT_OK_AND_ASSIGN(auto bitmap, MakePayload()); + ASSERT_OK_AND_ASSIGN(Messages updates, + Prepare({{before->file_name, + {{"value", "bitmap", bitmap}, {"other", "opaque", nullptr}}}})); + ASSERT_OK_AND_ASSIGN(auto after, UpdatedFile(updates)); + ASSERT_EQ(2u, after->extra_files.size()); + ASSERT_EQ("shared.aux", after->extra_files[0]); + ASSERT_OK_AND_ASSIGN(PayloadBytes actual, ReadPayloads(after)); + ASSERT_EQ(bitmap->BytesValue(), actual.at("value").at("bitmap")); + ASSERT_EQ(existing.at("value").at("unregistered"), actual.at("value").at("unregistered")); + ASSERT_EQ(std::nullopt, actual.at("value").at("empty-unregistered")); + ASSERT_EQ(std::nullopt, actual.at("other").at("opaque")); + ASSERT_OK(Commit(updates)); + ASSERT_OK_AND_ASSIGN(bool old_exists, fs_->Exists(paths->ToAlignedPath("old.index", before))); + ASSERT_TRUE(old_exists); + std::string content; + ASSERT_OK(fs_->ReadFile(aux_path, &content)); + ASSERT_EQ("auxiliary data", content); +} + +TEST_F(FileIndexWriteTest, WritesIndexBesideExternalDataFile) { + ASSERT_OK(CreateTable({{Options::DATA_FILE_EXTERNAL_PATHS, + "file://" + PathUtil::JoinPath(directory_->Str(), "external")}, + {Options::DATA_FILE_EXTERNAL_PATHS_STRATEGY, "round-robin"}})); + ASSERT_OK_AND_ASSIGN(Messages written, WriteData()); + ASSERT_OK_AND_ASSIGN(auto file, NewFile(written)); + ASSERT_TRUE(file->external_path); + ASSERT_OK_AND_ASSIGN(Snapshot base, LatestSnapshot()); + ASSERT_OK_AND_ASSIGN(auto payload, MakePayload()); + ASSERT_OK(payload->Seek(1, FS_SEEK_SET)); + for (int32_t generation = 0; generation < 3; ++generation) { + ASSERT_OK_AND_ASSIGN(Messages updates, + Prepare({{file->file_name, {{"value", "bitmap", payload}}}})); + ASSERT_OK_AND_ASSIGN(auto after, UpdatedFile(updates)); + ASSERT_EQ(file->external_path, after->external_path); + ASSERT_EQ(1u, after->extra_files.size()); + ASSERT_TRUE(after->extra_files[0]); + std::string expected_name = generation == 0 + ? file->file_name + ".index" + : fmt::format("{}-{}.index", file->file_name, generation); + ASSERT_EQ(expected_name, *after->extra_files[0]); + const std::string index_path = PathUtil::JoinPath( + PathUtil::GetParentDirPath(*file->external_path), *after->extra_files[0]); + ASSERT_OK_AND_ASSIGN(bool exists, fs_->Exists(index_path)); + ASSERT_TRUE(exists); + ASSERT_OK(Commit(updates)); + if (generation > 0) { + ASSERT_OK_AND_ASSIGN(PayloadBytes old_payloads, ReadPayloads(file)); + ASSERT_EQ(payload->BytesValue(), old_payloads.at("value").at("bitmap")); + } + file = after; + } + ASSERT_OK_AND_ASSIGN(auto values, ReadValues()); + ASSERT_EQ((std::vector{1, 1, 2}), values); + ASSERT_OK_AND_ASSIGN(auto historical, ReadValues(base.Id())); + ASSERT_EQ(values, historical); + ASSERT_OK_AND_ASSIGN( + auto filtered, ReadValues(std::nullopt, "value", + PredicateBuilder::Equal(0, "value", FieldType::INT, Literal(1)))); + ASSERT_EQ((std::vector{1, 1}), filtered); + ASSERT_EQ(0, payload->close_count); + ASSERT_EQ(1, payload.use_count()); +} + +TEST_F(FileIndexWriteTest, HonorsHistoricalSnapshotAndLatestSelection) { + ASSERT_OK(CreateTable()); + ASSERT_OK_AND_ASSIGN(Messages first_write, WriteData()); + ASSERT_OK_AND_ASSIGN(auto first, NewFile(first_write)); + ASSERT_OK_AND_ASSIGN(Snapshot base, LatestSnapshot()); + ASSERT_OK_AND_ASSIGN(Messages second_write, WriteData()); + ASSERT_OK_AND_ASSIGN(auto second, NewFile(second_write)); + ASSERT_OK_AND_ASSIGN(auto payload, MakePayload()); + const DataFileIndexInput first_input{first->file_name, {{"value", "bitmap", payload}}}; + const DataFileIndexInput second_input{second->file_name, {{"value", "bitmap", payload}}}; + ASSERT_NOK_WITH_MSG(Prepare({second_input}, {}, base.Id()), "not live in the base snapshot"); + ASSERT_OK_AND_ASSIGN(Messages explicit_updates, Prepare({first_input}, {}, base.Id())); + ASSERT_OK_AND_ASSIGN(auto explicit_message, OnlyMessage(explicit_updates)); + ASSERT_EQ(1u, explicit_message->GetCompactIncrement().CompactBefore().size()); + ASSERT_EQ(*first, *explicit_message->GetCompactIncrement().CompactBefore()[0]); + ASSERT_OK_AND_ASSIGN(Messages latest_updates, Prepare({second_input})); + explicit_updates.insert(explicit_updates.end(), latest_updates.begin(), latest_updates.end()); + ASSERT_OK(Commit(explicit_updates)); + ASSERT_OK_AND_ASSIGN(auto old_values, ReadValues(base.Id())); + ASSERT_EQ((std::vector{1, 1, 2}), old_values); + ASSERT_OK_AND_ASSIGN(auto current_values, ReadValues()); + ASSERT_EQ((std::vector{1, 1, 1, 1, 2, 2}), current_values); +} + +TEST_F(FileIndexWriteTest, IsolatesBranchCommits) { + ASSERT_OK(CreateTable()); + ASSERT_OK_AND_ASSIGN(Messages written, WriteData()); + ASSERT_OK_AND_ASSIGN(auto file, NewFile(written)); + ASSERT_OK_AND_ASSIGN(Snapshot base, LatestSnapshot()); + SchemaManager main_schema(fs_, table_path_); + ASSERT_OK_AND_ASSIGN(auto schema, main_schema.ReadSchema(base.SchemaId())); + ASSERT_OK_AND_ASSIGN(std::string schema_json, schema->ToJsonString()); + SchemaManager branch_schema(fs_, table_path_, "dev"); + ASSERT_OK(fs_->AtomicStore(PathUtil::JoinPath(branch_schema.SchemaDirectory(), "schema-0"), + schema_json)); + SnapshotManager branch_snapshots(fs_, table_path_, "dev"); + ASSERT_OK_AND_ASSIGN(std::string snapshot_json, base.ToJsonString()); + ASSERT_OK(fs_->AtomicStore(branch_snapshots.SnapshotPath(base.Id()), snapshot_json)); + branch_ = "dev"; + options_[Options::BRANCH] = branch_; + ASSERT_OK_AND_ASSIGN(auto payload, MakePayload()); + ASSERT_OK_AND_ASSIGN( + Messages updates, + Prepare({{file->file_name, {{"value", "bitmap", payload}}}}, {}, base.Id())); + ASSERT_OK(Commit(updates)); + ASSERT_OK_AND_ASSIGN(Snapshot dev, LatestSnapshot()); + ASSERT_EQ(base.Id() + 1, dev.Id()); + ASSERT_OK_AND_ASSIGN(auto dev_splits, Scan()); + ASSERT_EQ(1u, dev_splits.size()); + auto dev_split = std::dynamic_pointer_cast(dev_splits[0]); + ASSERT_TRUE(dev_split); + ASSERT_EQ(1u, dev_split->DataFiles().size()); + ASSERT_EQ(1u, dev_split->DataFiles()[0]->extra_files.size()); + auto predicate = PredicateBuilder::Equal(0, "value", FieldType::INT, Literal(1)); + ASSERT_OK_AND_ASSIGN(auto dev_values, ReadValues(std::nullopt, "value", predicate)); + ASSERT_EQ((std::vector{1, 1}), dev_values); + branch_ = "main"; + options_[Options::BRANCH] = branch_; + ASSERT_OK_AND_ASSIGN(Snapshot main, LatestSnapshot()); + ASSERT_EQ(base.Id(), main.Id()); + ASSERT_OK_AND_ASSIGN(auto main_splits, Scan()); + ASSERT_EQ(1u, main_splits.size()); + auto main_split = std::dynamic_pointer_cast(main_splits[0]); + ASSERT_TRUE(main_split); + ASSERT_EQ(1u, main_split->DataFiles().size()); + ASSERT_EQ(*file, *main_split->DataFiles()[0]); + ASSERT_OK_AND_ASSIGN(auto main_values, ReadValues(std::nullopt, "value", predicate)); + ASSERT_EQ((std::vector{1, 1, 2}), main_values); +} + +TEST_F(FileIndexWriteTest, UsesOriginalFieldNameAfterRename) { + ASSERT_OK(CreateTable()); + ASSERT_OK_AND_ASSIGN(Messages written, WriteData()); + ASSERT_OK_AND_ASSIGN(auto file, NewFile(written)); + SchemaManager manager(fs_, table_path_); + ASSERT_OK_AND_ASSIGN(auto schema, manager.ReadSchema(file->schema_id)); + auto fields = schema->Fields(); + fields[1] = DataField(fields[1].Id(), arrow::field("renamed_value", arrow::int32(), false)); + auto renamed_options = schema->Options(); + renamed_options[Options::BUCKET_KEY] = "renamed_value"; + ASSERT_OK(TestHelper::WriteNextSchema(fs_, table_path_, fields, schema->HighestFieldId(), + renamed_options)); + options_[Options::BUCKET_KEY] = "renamed_value"; + ASSERT_OK_AND_ASSIGN(auto payload, MakePayload()); + ASSERT_OK_AND_ASSIGN(Messages updates, + Prepare({{file->file_name, {{"value", "bitmap", payload}}}})); + ASSERT_OK_AND_ASSIGN(auto after, UpdatedFile(updates)); + ASSERT_EQ(file->schema_id, after->schema_id); + ASSERT_OK_AND_ASSIGN(PayloadBytes payloads, ReadPayloads(after)); + ASSERT_EQ(1u, payloads.count("value")); + ASSERT_EQ(0u, payloads.count("renamed_value")); + ASSERT_OK(Commit(updates)); + ASSERT_OK_AND_ASSIGN( + auto values, + ReadValues(std::nullopt, "renamed_value", + PredicateBuilder::Equal(0, "renamed_value", FieldType::INT, Literal(1)))); + ASSERT_EQ((std::vector{1, 1}), values); +} + +TEST_F(FileIndexWriteTest, PreservesDeletionVectorsAndPhysicalPositions) { + ASSERT_OK(CreateTable({{Options::DELETION_VECTORS_ENABLED, "true"}})); + ASSERT_OK_AND_ASSIGN(Messages written, WriteData()); + ASSERT_OK_AND_ASSIGN(auto file, NewFile(written)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr deletion, + DeletionVectorTestHelper::CreateDeletionVectorCommitMessage( + fs_, table_path_, "parquet", written[0], {{file->file_name, {0}}}, pool_)); + ASSERT_OK(Commit({deletion})); + ASSERT_OK_AND_ASSIGN(auto payload, MakePayload()); + ASSERT_OK_AND_ASSIGN(Messages updates, + Prepare({{file->file_name, {{"value", "bitmap", payload}}}})); + ASSERT_OK(Commit(updates)); + ASSERT_OK_AND_ASSIGN(auto values, ReadValues()); + ASSERT_EQ((std::vector{1, 2}), values); + ASSERT_OK_AND_ASSIGN( + auto filtered, ReadValues(std::nullopt, "value", + PredicateBuilder::Equal(0, "value", FieldType::INT, Literal(1)))); + ASSERT_EQ((std::vector{1}), filtered); +} + +TEST_F(FileIndexWriteTest, OptionalDependenciesAndCallerOverrides) { + ASSERT_OK(CreateTable({{Options::FILE_INDEX_FORMAT_VERSION, "1"}})); + ASSERT_OK_AND_ASSIGN(Messages written, WriteData()); + ASSERT_OK_AND_ASSIGN(auto file, NewFile(written)); + auto fs = std::make_shared(); + auto payload = std::make_shared("opaque payload"); + // Bits 1/2/4 supply filesystem/pool/executor respectively. + // 0: all null; 7: all supplied; 6: filesystem null; 5: pool null; 3: executor null. + for (int32_t mask : {0, 7, 6, 5, 3}) { + SCOPED_TRACE(mask); + size_t creates = fs->created.size(); + ASSERT_OK_AND_ASSIGN( + Messages updates, + FileIndexWrite::PrepareCommit(table_path_, "", {}, std::nullopt, + {{file->file_name, {{"value", "opaque", payload}}}}, + mask & 1 ? fs : nullptr, mask & 2 ? pool_ : nullptr, + mask & 4 ? executor_ : nullptr)); + ASSERT_EQ(creates + ((mask & 1) != 0), fs->created.size()); + ASSERT_OK(Abort(updates)); + } + ASSERT_OK_AND_ASSIGN( + Messages updates, + FileIndexWrite::PrepareCommit( + table_path_, "", + {{Options::FILE_INDEX_FORMAT_VERSION, "2"}, {Options::FILE_SYSTEM, "unregistered-fs"}}, + std::nullopt, {{file->file_name, {{"value", "opaque", payload}}}}, fs, pool_, + executor_)); + ASSERT_OK_AND_ASSIGN(auto after, UpdatedFile(updates)); + ASSERT_OK_AND_ASSIGN(auto input, OpenIndex(after)); + DataInputStream in(input); + ASSERT_OK(in.Seek(8)); + ASSERT_OK_AND_ASSIGN(int32_t version, in.ReadValue()); + ASSERT_EQ(2, version); + ASSERT_OK(input->Close()); + ASSERT_OK(Abort(updates)); + ASSERT_EQ(0, payload->close_count); + ASSERT_OK_AND_ASSIGN(auto values, ReadValues()); + ASSERT_EQ((std::vector{1, 1, 2}), values); +} + +TEST_F(FileIndexWriteTest, EmptyAndInvalidInputs) { + ASSERT_OK(CreateTable()); + ASSERT_NOK_WITH_MSG(Prepare({}), "requires an existing snapshot"); + ASSERT_OK_AND_ASSIGN(Messages written, WriteData()); + ASSERT_OK_AND_ASSIGN(auto file, NewFile(written)); + ASSERT_OK_AND_ASSIGN(Messages empty, Prepare({})); + ASSERT_TRUE(empty.empty()); + ASSERT_OK_AND_ASSIGN(Messages skipped, Prepare({{file->file_name, {}}})); + ASSERT_TRUE(skipped.empty()); + auto payload = std::make_shared("payload"); + DataFileIndexInput valid{file->file_name, {{"value", "opaque", payload}}}; + const std::vector, std::string>> invalid = { + {{valid, valid}, "Duplicate data file name"}, + {{{file->file_name, {{"value", "opaque", payload}, {"value", "opaque", payload}}}}, + "Duplicate field"}, + {{{file->file_name, {{"", "opaque", payload}}}}, "must not be empty"}, + {{{file->file_name, {{"value", "", payload}}}}, "must not be empty"}, + {{{"", valid.indexes}}, "data file name"}, + {{{"../outside.orc", valid.indexes}}, "data file name"}, + {{{"missing.orc", valid.indexes}}, "not live in the base snapshot"}, + {{{file->file_name, {{"value", "opaque", std::make_shared("")}}}}, + "positive length"}}; + for (const auto& [inputs, error] : invalid) { + SCOPED_TRACE(error); + ASSERT_NOK_WITH_MSG(Prepare(inputs), error); + } + ASSERT_NOK_WITH_MSG(Prepare({valid}, {}, 0), "Invalid base snapshot id"); + ASSERT_NOK(Prepare({valid}, {}, 999999)); + ASSERT_NOK_WITH_MSG(Prepare({valid}, {{Options::BUCKET, "3"}}), "cannot override"); + ASSERT_NOK(Prepare({valid}, {{Options::FILE_INDEX_FORMAT_VERSION, "3"}})); + ASSERT_NOK(Prepare({valid}, {{Options::BRANCH, "dev"}})); + ASSERT_NOK(FileIndexWrite::PrepareCommit(table_path_, "../dev", {}, std::nullopt, {valid}, fs_, + pool_, executor_)); + ASSERT_NOK(FileIndexWrite::PrepareCommit("", "main", {}, std::nullopt, {valid}, nullptr, + nullptr, nullptr)); + ASSERT_EQ(0, payload->close_count); +} + +TEST_F(FileIndexWriteTest, RejectsUnsupportedTableModes) { + for (bool primary_key : {false, true}) { + ASSERT_OK(CreateTable( + {{Options::BUCKET, primary_key ? "2" : "-1"}}, + primary_key ? std::vector{"part", "value"} : std::vector{})); + ASSERT_OK_AND_ASSIGN(Messages written, WriteData()); + ASSERT_OK_AND_ASSIGN(auto file, NewFile(written)); + ASSERT_NOK_WITH_MSG(Prepare({{file->file_name, {{"value", "opaque", nullptr}}}}), + "only supports append tables with fixed buckets"); + } +} + +TEST_F(FileIndexWriteTest, RejectsAmbiguousFileNames) { + ASSERT_OK(CreateTable()); + ASSERT_OK_AND_ASSIGN(Messages first_write, WriteData()); + ASSERT_OK_AND_ASSIGN(Messages second_write, WriteData("[[20,1,10]]", 0, 20)); + ASSERT_OK_AND_ASSIGN(auto first, NewFile(first_write)); + ASSERT_OK_AND_ASSIGN(auto second, NewFile(second_write)); + ASSERT_OK_AND_ASSIGN(auto paths, Paths(20, 0)); + auto duplicate = std::make_shared(*second); + duplicate->file_name = first->file_name; + std::string data; + ASSERT_OK(fs_->ReadFile(paths->ToPath(second), &data)); + ASSERT_OK(fs_->WriteFile(paths->ToPath(duplicate), data, false)); + ASSERT_OK_AND_ASSIGN(auto message, OnlyMessage(second_write)); + ASSERT_OK(Commit( + {std::make_shared(message->Partition(), 0, 2, DataIncrement({}, {}, {}), + CompactIncrement({second}, {duplicate}, {}))})); + ASSERT_NOK_WITH_MSG(Prepare({{first->file_name, {{"value", "opaque", nullptr}}}}), + "Ambiguous data file name"); +} + +TEST_F(FileIndexWriteTest, ShortReadsPreservePayload) { + ASSERT_OK(CreateTable()); + ASSERT_OK_AND_ASSIGN(Messages written, WriteData()); + ASSERT_OK_AND_ASSIGN(auto file, NewFile(written)); + auto payload = std::make_shared(std::string(1024 * 1024 + 17, 'x')); + payload->max_read = 65521; + ASSERT_OK_AND_ASSIGN(Messages updates, + Prepare({{file->file_name, {{"value", "opaque", payload}}}}, + {{Options::FILE_INDEX_FORMAT_VERSION, "2"}})); + ASSERT_OK_AND_ASSIGN(auto after, UpdatedFile(updates)); + ASSERT_OK_AND_ASSIGN(PayloadBytes actual, ReadPayloads(after)); + ASSERT_EQ(payload->BytesValue(), actual.at("value").at("opaque")); + ASSERT_EQ(0, payload->close_count); + ASSERT_OK(Commit(updates)); +} + +TEST_F(FileIndexWriteTest, PublishesCompleteTemporaryIndex) { + ASSERT_OK(CreateTable()); + ASSERT_OK_AND_ASSIGN(Messages written, WriteData()); + ASSERT_OK_AND_ASSIGN(auto file, NewFile(written)); + ASSERT_OK_AND_ASSIGN(auto paths, Paths()); + auto payload = std::make_shared("new payload"); + for (int32_t version : {1, 2}) { + auto fs = std::make_shared(); + fs->before_rename = [&](const std::string& src, const std::string& dst) -> Status { + EXPECT_NE(src, dst); + EXPECT_TRUE(StringUtils::EndsWith(src, ".tmp")); + EXPECT_EQ(PathUtil::GetParentDirPath(src), PathUtil::GetParentDirPath(dst)); + PAIMON_ASSIGN_OR_RAISE(bool exists, fs_->Exists(dst)); + EXPECT_FALSE(exists); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr input, fs_->Open(src)); + ScopeGuard close([&input]() { [[maybe_unused]] Status ignored = input->Close(); }); + PAIMON_ASSIGN_OR_RAISE(FileIndexFormatUtils::FileIndexEntries entries, + FileIndexFormatUtils::ReadIndexEntries(input, pool_)); + EXPECT_FALSE(entries.empty()); + return Status::OK(); + }; + ASSERT_OK_AND_ASSIGN( + Messages updates, + FileIndexWrite::PrepareCommit( + table_path_, branch_, + {{Options::FILE_INDEX_FORMAT_VERSION, std::to_string(version)}}, std::nullopt, + {{file->file_name, {{"value", "opaque", payload}}}}, fs, pool_, executor_)); + ASSERT_EQ(1u, fs->created.size()); + ASSERT_EQ(1u, fs->published.size()); + ASSERT_EQ(1u, fs->rename_attempts.size()); + ASSERT_EQ(paths->ToFileIndexPath(paths->ToPath(file)), fs->published[0]); + ASSERT_EQ(fs->created[0], fs->rename_attempts[0].first); + ASSERT_EQ(fs->published[0], fs->rename_attempts[0].second); + AssertRemoved(fs->created); + ASSERT_OK_AND_ASSIGN(auto after, UpdatedFile(updates)); + ASSERT_OK_AND_ASSIGN(PayloadBytes actual, ReadPayloads(after)); + ASSERT_EQ(payload->BytesValue(), actual.at("value").at("opaque")); + ASSERT_OK(Abort(updates)); + AssertRemoved(fs->published); + } +} + +TEST_F(FileIndexWriteTest, ReusesExistingIndexAndPreparesWholeBatch) { + ASSERT_OK(CreateTable()); + ASSERT_OK_AND_ASSIGN(Messages first_write, WriteData()); + ASSERT_OK_AND_ASSIGN(Messages second_write, WriteData()); + ASSERT_OK_AND_ASSIGN(auto first, NewFile(first_write)); + ASSERT_OK_AND_ASSIGN(auto second, NewFile(second_write)); + if (second->file_name < first->file_name) { + std::swap(first, second); + } + ASSERT_OK_AND_ASSIGN(Snapshot base, LatestSnapshot()); + ASSERT_OK_AND_ASSIGN(auto paths, Paths()); + const std::string target = paths->ToFileIndexPath(paths->ToPath(first)); + ASSERT_OK(WriteContainer(first, PathUtil::GetName(target), + {{"value", {{"opaque", "existing payload"}}}})); + std::string existing_bytes; + ASSERT_OK(fs_->ReadFile(target, &existing_bytes)); + auto fs = std::make_shared(); + auto unused_payload = std::make_shared("unused payload"); + unused_payload->fail_length = true; + unused_payload->fail_read = true; + auto payload = std::make_shared("payload"); + ASSERT_OK_AND_ASSIGN( + Messages updates, + FileIndexWrite::PrepareCommit(table_path_, branch_, {}, std::nullopt, + {{first->file_name, {{"value", "opaque", unused_payload}}}, + {second->file_name, {{"value", "opaque", payload}}}}, + fs, pool_, executor_)); + ASSERT_EQ(1, fs->create_attempts); + ASSERT_EQ(1u, fs->created.size()); + ASSERT_EQ(1u, fs->published.size()); + ASSERT_EQ(1u, fs->rename_attempts.size()); + ASSERT_TRUE(fs->delete_attempts.empty()); + ASSERT_EQ(0, unused_payload->close_count); + AssertRemoved(fs->created); + ASSERT_OK_AND_ASSIGN(auto update, OnlyMessage(updates)); + const auto& increment = update->GetCompactIncrement(); + ASSERT_EQ(2u, increment.CompactBefore().size()); + ASSERT_EQ(2u, increment.CompactAfter().size()); + ASSERT_EQ(*first, *increment.CompactBefore()[0]); + ASSERT_EQ(*second, *increment.CompactBefore()[1]); + ASSERT_EQ(first->file_name, increment.CompactAfter()[0]->file_name); + ASSERT_EQ(second->file_name, increment.CompactAfter()[1]->file_name); + ASSERT_OK_AND_ASSIGN(PayloadBytes reused, ReadPayloads(increment.CompactAfter()[0])); + ASSERT_EQ("existing payload", reused.at("value").at("opaque")); + ASSERT_OK_AND_ASSIGN(PayloadBytes actual, ReadPayloads(increment.CompactAfter()[1])); + ASSERT_EQ(payload->BytesValue(), actual.at("value").at("opaque")); + ASSERT_EQ(0, std::count(fs->delete_attempts.begin(), fs->delete_attempts.end(), target)); + std::string content; + ASSERT_OK(fs_->ReadFile(target, &content)); + ASSERT_EQ(existing_bytes, content); + ASSERT_OK_AND_ASSIGN(Snapshot unchanged, LatestSnapshot()); + ASSERT_EQ(base.Id(), unchanged.Id()); + ASSERT_OK(Commit(updates)); +} + +TEST_F(FileIndexWriteTest, PreparesCommitForTargetsCreatedDuringRename) { + for (const Status& error : {Status::Exist("native exist"), Status::Invalid("native invalid"), + Status::IOError("native IO error")}) { + SCOPED_TRACE(error.ToString()); + ASSERT_OK(CreateTable()); + ASSERT_OK_AND_ASSIGN(Messages written, WriteData()); + ASSERT_OK_AND_ASSIGN(auto file, NewFile(written)); + ASSERT_OK_AND_ASSIGN(auto paths, Paths()); + const std::string target = paths->ToFileIndexPath(paths->ToPath(file)); + auto payload = std::make_shared("payload"); + auto fs = std::make_shared(); + fs->fail_delete = true; + fs->before_rename = [&](const std::string&, const std::string& dst) -> Status { + PAIMON_RETURN_NOT_OK( + WriteContainer(file, PathUtil::GetName(dst), + {{"value", {{"opaque", "another writer's payload"}}}})); + return error; + }; + ASSERT_OK_AND_ASSIGN( + Messages updates, + FileIndexWrite::PrepareCommit(table_path_, branch_, {}, std::nullopt, + {{file->file_name, {{"value", "opaque", payload}}}}, fs, + pool_, executor_)); + ASSERT_OK_AND_ASSIGN(auto after, UpdatedFile(updates)); + ASSERT_EQ(file->file_name, after->file_name); + ASSERT_EQ(1u, after->extra_files.size()); + ASSERT_EQ(PathUtil::GetName(target), after->extra_files[0]); + ASSERT_EQ(1u, fs->rename_attempts.size()); + ASSERT_TRUE(fs->published.empty()); + ASSERT_EQ(1u, fs->created.size()); + ASSERT_EQ(fs->created, fs->delete_attempts); + ASSERT_OK_AND_ASSIGN(bool temporary_exists, fs_->Exists(fs->created[0])); + ASSERT_TRUE(temporary_exists); + fs->fail_delete = false; + ASSERT_OK(fs->Delete(fs->created[0], false)); + ASSERT_OK_AND_ASSIGN(PayloadBytes actual, ReadPayloads(after)); + ASSERT_EQ("another writer's payload", actual.at("value").at("opaque")); + ASSERT_OK(Commit(updates)); + } +} + +TEST_F(FileIndexWriteTest, PayloadFailuresRemoveAllGeneratedIndexes) { + ASSERT_OK(CreateTable({{"file-index.bitmap.columns", "value"}, + {Options::FILE_INDEX_IN_MANIFEST_THRESHOLD, "0B"}})); + ASSERT_OK_AND_ASSIGN(Messages first, WriteData()); + ASSERT_OK_AND_ASSIGN(Messages second, WriteData()); + ASSERT_OK_AND_ASSIGN(auto first_file, NewFile(first)); + ASSERT_OK_AND_ASSIGN(auto second_file, NewFile(second)); + std::vector> files{first_file, second_file}; + std::sort(files.begin(), files.end(), + [](const auto& lhs, const auto& rhs) { return lhs->file_name < rhs->file_name; }); + ASSERT_OK_AND_ASSIGN(auto paths, Paths()); + ASSERT_OK_AND_ASSIGN(Snapshot base, LatestSnapshot()); + for (int32_t version : {1, 2}) { + for (int32_t failure = 0; failure < 4; ++failure) { + SCOPED_TRACE(fmt::format("version={}, failure={}", version, failure)); + auto good = std::make_shared("good payload"); + auto bad = std::make_shared("bad payload"); + bad->fail_length = failure == 0; + bad->fail_seek = failure == 1; + bad->fail_read = failure == 2; + bad->early_eof = failure == 3; + auto fs = std::make_shared(); + ASSERT_NOK(FileIndexWrite::PrepareCommit( + table_path_, branch_, + {{Options::FILE_INDEX_FORMAT_VERSION, std::to_string(version)}}, base.Id(), + {{files[0]->file_name, {{"value", "opaque", good}}}, + {files[1]->file_name, {{"value", "opaque", bad}}}}, + fs, pool_, executor_)); + ASSERT_FALSE(fs->created.empty()); + AssertRemoved(fs->created); + AssertRemoved(fs->published); + ASSERT_EQ(0, good->close_count); + ASSERT_EQ(0, bad->close_count); + ASSERT_OK_AND_ASSIGN(Snapshot unchanged, LatestSnapshot()); + ASSERT_EQ(base.Id(), unchanged.Id()); + for (const auto& file : files) { + ASSERT_OK_AND_ASSIGN(bool exists, fs_->Exists(paths->ToPath(file))); + ASSERT_TRUE(exists); + ASSERT_OK_AND_ASSIGN(PayloadBytes original, ReadPayloads(file)); + ASSERT_EQ(1u, original.at("value").count("bitmap")); + } + } + } +} + +TEST_F(FileIndexWriteTest, OutputFailuresRemovePartialIndexes) { + ASSERT_OK(CreateTable()); + ASSERT_OK_AND_ASSIGN(Messages written, WriteData()); + ASSERT_OK_AND_ASSIGN(auto file, NewFile(written)); + auto payload = std::make_shared("payload"); + for (int32_t version : {1, 2}) { + for (auto failure : {OutputFailure::CREATE, OutputFailure::WRITE, OutputFailure::FLUSH, + OutputFailure::CLOSE, OutputFailure::RENAME}) { + auto fs = std::make_shared(); + fs->failure = failure; + ASSERT_NOK(FileIndexWrite::PrepareCommit( + table_path_, branch_, + {{Options::FILE_INDEX_FORMAT_VERSION, std::to_string(version)}}, std::nullopt, + {{file->file_name, {{"value", "opaque", payload}}}}, fs, pool_, executor_)); + ASSERT_EQ(1, fs->create_attempts); + if (failure != OutputFailure::CREATE) { + ASSERT_EQ(1u, fs->created.size()); + } + AssertRemoved(fs->created); + ASSERT_TRUE(fs->published.empty()); + if (failure == OutputFailure::RENAME) { + ASSERT_EQ(1u, fs->rename_attempts.size()); + } else { + ASSERT_TRUE(fs->rename_attempts.empty()); + } + } + } + ASSERT_EQ(0, payload->close_count); + ASSERT_OK_AND_ASSIGN(auto values, ReadValues()); + ASSERT_EQ((std::vector{1, 1, 2}), values); +} + +TEST_F(FileIndexWriteTest, CleanupFailuresPreserveOriginalError) { + ASSERT_OK(CreateTable()); + ASSERT_OK_AND_ASSIGN(Messages first_write, WriteData()); + ASSERT_OK_AND_ASSIGN(Messages second_write, WriteData()); + ASSERT_OK_AND_ASSIGN(auto first, NewFile(first_write)); + ASSERT_OK_AND_ASSIGN(auto second, NewFile(second_write)); + if (second->file_name < first->file_name) { + std::swap(first, second); + } + auto good = std::make_shared("good payload"); + auto bad = std::make_shared("bad payload"); + bad->fail_read = true; + auto fs = std::make_shared(); + fs->fail_delete = true; + ASSERT_NOK_WITH_MSG( + FileIndexWrite::PrepareCommit(table_path_, branch_, {}, std::nullopt, + {{first->file_name, {{"value", "opaque", good}}}, + {second->file_name, {{"value", "opaque", bad}}}}, + fs, pool_, executor_), + "injected payload read failure"); + ASSERT_EQ(2u, fs->created.size()); + ASSERT_EQ(1u, fs->published.size()); + const std::vector cleanup_paths = {fs->published[0], fs->created[1]}; + ASSERT_EQ(cleanup_paths, fs->delete_attempts); + fs->fail_delete = false; + AssertRemoved({fs->created[0]}); + for (const auto& path : cleanup_paths) { + ASSERT_OK_AND_ASSIGN(bool exists, fs_->Exists(path)); + ASSERT_TRUE(exists); + ASSERT_OK(fs->Delete(path, false)); + } + ASSERT_OK_AND_ASSIGN(auto values, ReadValues()); + ASSERT_EQ((std::vector{1, 1, 1, 1, 2, 2}), values); +} + +TEST_F(FileIndexWriteTest, RejectsMalformedExistingIndexes) { + for (int32_t malformed = 0; malformed < 3; ++malformed) { + SCOPED_TRACE(malformed); + ASSERT_OK(CreateTable()); + ASSERT_OK_AND_ASSIGN(Messages written, WriteData()); + ASSERT_OK_AND_ASSIGN(auto file, NewFile(written)); + ASSERT_OK_AND_ASSIGN(auto paths, Paths()); + const std::string index_path = paths->ToAlignedPath("old.index", file); + if (malformed == 0) { + ASSERT_OK(fs_->WriteFile(index_path, "invalid container", false)); + } else { + ASSERT_OK(WriteContainer(file, "old.index", {{"value", {{"opaque", "payload"}}}})); + if (malformed == 1) { + std::string bytes; + ASSERT_OK(fs_->ReadFile(index_path, &bytes)); + auto input = std::make_shared(bytes.data(), bytes.size()); + DataInputStream in(input); + ASSERT_OK(in.Seek(12)); + ASSERT_OK_AND_ASSIGN(int32_t header_length, in.ReadValue()); + ASSERT_GE(header_length, 12); + ASSERT_LE(static_cast(header_length), bytes.size()); + bytes[header_length - 12] = 0x7f; + ASSERT_OK(fs_->WriteFile(index_path, bytes, true)); + } + } + auto before = file->CopyWithExtraFiles( + malformed == 2 ? std::vector>{"old.index", "duplicate.index"} + : std::vector>{"old.index"}); + if (malformed == 2) { + ASSERT_OK( + fs_->WriteFile(paths->ToAlignedPath("duplicate.index", file), "duplicate", false)); + } + ASSERT_OK(RewriteMetadata(file, before)); + auto fs = std::make_shared(); + ASSERT_NOK(FileIndexWrite::PrepareCommit( + table_path_, branch_, {}, std::nullopt, + {{file->file_name, {{"other", "opaque", nullptr}}}}, fs, pool_, executor_)); + ASSERT_TRUE(fs->created.empty()); + ASSERT_OK_AND_ASSIGN(bool exists, fs_->Exists(paths->ToPath(file))); + ASSERT_TRUE(exists); + } +} + +} // namespace paimon::test diff --git a/src/paimon/core/operation/file_store_commit_impl.cpp b/src/paimon/core/operation/file_store_commit_impl.cpp index ee8adfee8..02ef7d6bf 100644 --- a/src/paimon/core/operation/file_store_commit_impl.cpp +++ b/src/paimon/core/operation/file_store_commit_impl.cpp @@ -217,6 +217,13 @@ Status FileStoreCommitImpl::Abort( const DataIncrement& new_files_increment = msg->GetNewFilesIncrement(); const CompactIncrement& compact_increment = msg->GetCompactIncrement(); + // Preserve paths reused by metadata-only rewrites. + std::unordered_set retained_paths; + for (const auto& file : compact_increment.CompactBefore()) { + for (const auto& path : data_file_path_factory->CollectFiles(file)) { + retained_paths.insert(path); + } + } std::vector> data_files_to_delete; auto append_data_files = [&data_files_to_delete](const std::vector>& files) { @@ -224,7 +231,6 @@ Status FileStoreCommitImpl::Abort( }; append_data_files(new_files_increment.NewFiles()); append_data_files(new_files_increment.ChangelogFiles()); - append_data_files(compact_increment.CompactAfter()); append_data_files(compact_increment.ChangelogFiles()); for (const auto& file : data_files_to_delete) { // Best-effort cleanup: ignore delete failures, aligning with Java deleteQuietly. @@ -232,6 +238,13 @@ Status FileStoreCommitImpl::Abort( [[maybe_unused]] Status status = fs_->Delete(path, /*recursive=*/false); } } + for (const auto& file : compact_increment.CompactAfter()) { + for (const std::string& path : data_file_path_factory->CollectFiles(file)) { + if (retained_paths.count(path) == 0) { + [[maybe_unused]] Status status = fs_->Delete(path, /*recursive=*/false); + } + } + } std::vector> index_files_to_delete; auto append_index_files = [&index_files_to_delete]( diff --git a/src/paimon/core/operation/file_store_commit_impl_test.cpp b/src/paimon/core/operation/file_store_commit_impl_test.cpp index be1482052..a591fd828 100644 --- a/src/paimon/core/operation/file_store_commit_impl_test.cpp +++ b/src/paimon/core/operation/file_store_commit_impl_test.cpp @@ -3570,6 +3570,58 @@ TEST_F(FileStoreCommitImplTest, TestAbortDeletesDataAndIndexFiles) { } } +TEST_F(FileStoreCommitImplTest, AbortPreservesReusedPathsAndDeletesNewCompactionInputs) { + CommitContextBuilder builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, + builder.AddOption(Options::FILE_SYSTEM, "local").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + FileStoreCommit::Create(std::move(context))); + auto* commit_impl = dynamic_cast(commit.get()); + ASSERT_TRUE(commit_impl); + const BinaryRow partition = CreateIntRow(10); + ASSERT_OK_AND_ASSIGN(std::shared_ptr paths, + commit_impl->path_factory_->CreateDataFilePathFactory(partition, 0)); + for (bool external : {false, true}) { + auto before = CreateAppendDataFileMeta("abort-reused-data", 1); + before->extra_files = {"old.index", "shared.aux"}; + if (external) { + before->external_path = PathUtil::JoinPath(table_path_, "external/abort-reused-data"); + } + ASSERT_OK_AND_ASSIGN(auto upgraded, before->Upgrade(before->level + 1)); + auto after = upgraded->CopyWithExtraFiles({"new.index", "shared.aux"}); + auto appended = CreateAppendDataFileMeta("abort-new-input", 1); + appended->extra_files = {"abort-new-input.index"}; + auto append_log = CreateAppendDataFileMeta("abort-append-log", 1); + auto compact_log = CreateAppendDataFileMeta("abort-compact-log", 1); + DataIncrement data_increment({appended}, {}, {append_log}); + CompactIncrement compact_increment({before, appended}, {after}, {compact_log}); + auto message = + std::make_shared(partition, 0, 2, data_increment, compact_increment); + const std::vector retained = {paths->ToPath(before), + paths->ToAlignedPath("old.index", before), + paths->ToAlignedPath("shared.aux", before)}; + const std::vector removed = { + paths->ToAlignedPath("new.index", after), paths->ToPath(appended), + paths->ToAlignedPath("abort-new-input.index", appended), paths->ToPath(append_log), + paths->ToPath(compact_log)}; + for (const auto& group : {retained, removed}) { + for (const auto& path : group) { + ASSERT_OK(file_system_->Mkdirs(PathUtil::GetParentDirPath(path))); + ASSERT_OK(file_system_->WriteFile(path, "", /*overwrite=*/false)); + } + } + ASSERT_OK(commit_impl->Abort({message})); + for (const auto& path : retained) { + ASSERT_OK_AND_ASSIGN(bool exists, file_system_->Exists(path)); + ASSERT_TRUE(exists) << path; + } + for (const auto& path : removed) { + ASSERT_OK_AND_ASSIGN(bool exists, file_system_->Exists(path)); + ASSERT_FALSE(exists) << path; + } + } +} + TEST_F(FileStoreCommitImplTest, AbortIgnoresMissingFilesAndFailsForNonImplMessage) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,