diff --git a/docs/source/user_guide/write.rst b/docs/source/user_guide/write.rst index 97c4dd30b..0acce59a4 100644 --- a/docs/source/user_guide/write.rst +++ b/docs/source/user_guide/write.rst @@ -38,6 +38,21 @@ Parquet, and Avro, thereby improving write throughput. the files it ignored. See :doc:`compaction` for the ``full-compaction`` and ``lookup`` producers. +External Data Paths +------------------- + +Set ``data-file.external-paths`` to comma-separated roots and +``data-file.external-paths.strategy`` to ``round-robin`` to distribute new files +across them. For weighted random selection, set the strategy to ``weight-robin`` and +``data-file.external-paths.weights`` to positive integers in the same order as +the paths. For example, ``10,5,15`` selects three roots with probabilities +``1/3``, ``1/6``, and ``1/2``. These are probabilities per file, not byte quotas. + +Without weights, ``weight-robin`` falls back to round robin. A single external path +always uses that path. With multiple paths and configured weights, the number +of weights must equal the number of paths. The default strategy is ``none``, +which writes to the table directory. + Bucketing Modes --------------- diff --git a/include/paimon/defs.h b/include/paimon/defs.h index 88af98c0d..4db8f1290 100644 --- a/include/paimon/defs.h +++ b/include/paimon/defs.h @@ -495,8 +495,16 @@ struct PAIMON_EXPORT Options { /// written, multiple elements separated by commas. static const char DATA_FILE_EXTERNAL_PATHS[]; /// "data-file.external-paths.strategy" - The strategy of selecting an external path when - /// writing data. Values can be: "none", "specific-fs", "round-robin". Default value is "none". + /// writing data. Values: "none", "specific-fs", "round-robin", "weight-robin". Default is + /// "none". static const char DATA_FILE_EXTERNAL_PATHS_STRATEGY[]; + /// "data-file.external-paths.weights" - The weights for external paths when + /// data-file.external-paths.strategy is set to weight-robin. + /// Format: 'weight1,weight2,...' with weights corresponding to paths in + /// data-file.external-paths by order. + /// Example: '10,5,15' means first path has weight 10, second 5, third 15. + /// Weights must be positive integers. + static const char DATA_FILE_EXTERNAL_PATHS_WEIGHTS[]; /// "data-file.prefix" - Specify the file name prefix of data files. Default value is "data-". static const char DATA_FILE_PREFIX[]; /// "index-file-in-data-file-dir" - Whether index file in data file directory. Default value is @@ -611,7 +619,8 @@ struct PAIMON_EXPORT Options { /// "global-index.thread-num" - The maximum number of concurrent scanner for global index. No /// default value. By default is the number of processors available to the machine. static const char GLOBAL_INDEX_THREAD_NUM[]; - /// "global-index.external-path" - Global index root directory, if not set, the global index + /// "global-index.external-path" - Comma-separated global index root directories. New files + /// and checkpoints use independent round robin. If not set, the global index /// files will be stored under the index directory. static const char GLOBAL_INDEX_EXTERNAL_PATH[]; /// "global-index.search-mode" - Fallback search mode for global index queries, used when the diff --git a/include/paimon/global_index/io/global_index_file_writer.h b/include/paimon/global_index/io/global_index_file_writer.h index 639169cfb..57da9c2aa 100644 --- a/include/paimon/global_index/io/global_index_file_writer.h +++ b/include/paimon/global_index/io/global_index_file_writer.h @@ -29,19 +29,16 @@ class PAIMON_EXPORT GlobalIndexFileWriter { public: virtual ~GlobalIndexFileWriter() = default; - /// Generates a unique file name for a new index file using the given prefix. + /// Generates a unique full path for a new index file using the given prefix. /// @note This function may be called multiple times if the index consists of multiple files. - virtual Result NewFileName(const std::string& prefix) const = 0; + virtual Result NewPath(const std::string& prefix) const = 0; - /// Opens a new output stream for writing index data to the specified file. + /// Opens a new output stream at the supplied full path, without selecting a directory. virtual Result> NewOutputStream( - const std::string& file_name) const = 0; + const std::string& file_path) const = 0; - /// Get the file size of input file name. - virtual Result GetFileSize(const std::string& file_name) const = 0; - - /// Get the index file path of input file name. - virtual std::string ToPath(const std::string& file_name) const = 0; + /// Gets the size of the file at the supplied full path. + virtual Result GetFileSize(const std::string& file_path) const = 0; }; } // namespace paimon diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp index dc88b6b38..a6e68ffaf 100644 --- a/src/paimon/common/defs.cpp +++ b/src/paimon/common/defs.cpp @@ -126,6 +126,7 @@ const char Options::FILE_INDEX_FORMAT_VERSION[] = "file-index.format.version"; const char Options::FILE_INDEX_IN_MANIFEST_THRESHOLD[] = "file-index.in-manifest-threshold"; const char Options::DATA_FILE_EXTERNAL_PATHS[] = "data-file.external-paths"; const char Options::DATA_FILE_EXTERNAL_PATHS_STRATEGY[] = "data-file.external-paths.strategy"; +const char Options::DATA_FILE_EXTERNAL_PATHS_WEIGHTS[] = "data-file.external-paths.weights"; const char Options::DATA_FILE_PREFIX[] = "data-file.prefix"; const char Options::INDEX_FILE_IN_DATA_FILE_DIR[] = "index-file-in-data-file-dir"; const char Options::ROW_TRACKING_ENABLED[] = "row-tracking.enabled"; diff --git a/src/paimon/common/fs/external_path_provider.h b/src/paimon/common/fs/external_path_provider.h index c70aa2f64..be0007b54 100644 --- a/src/paimon/common/fs/external_path_provider.h +++ b/src/paimon/common/fs/external_path_provider.h @@ -19,8 +19,10 @@ #pragma once +#include #include #include +#include #include #include #include @@ -32,32 +34,55 @@ #include "paimon/status.h" namespace paimon { -/// Provider for external data paths. +/// Provider for external file paths. class ExternalPathProvider { public: static Result> Create( const std::vector& external_table_paths, - const std::string& relative_bucket_path) { + const std::string& relative_bucket_path, const std::vector& weights) { if (external_table_paths.empty()) { return Status::Invalid("external table paths cannot be empty"); } - return std::unique_ptr( - new ExternalPathProvider(external_table_paths, relative_bucket_path)); + std::vector cumulative_weights; + if (external_table_paths.size() > 1 && !weights.empty()) { + if (weights.size() != external_table_paths.size()) { + return Status::Invalid("external paths and weights must have the same size"); + } + cumulative_weights.reserve(weights.size()); + uint64_t total = 0; + for (int32_t weight : weights) { + total += weight; + cumulative_weights.push_back(total); + } + } + return std::unique_ptr(new ExternalPathProvider( + external_table_paths, relative_bucket_path, cumulative_weights)); } - /// Get the next external data path. + /// Get the next external file path. /// - /// @return the next external data path + /// @return the next external file path std::string GetNextExternalDataPath(const std::string& file_name) { size_t position = (++position_) % external_table_paths_.size(); + if (!cumulative_weights_.empty()) { + thread_local std::mt19937_64 generator( + std::random_device{}()); // NOLINT(whitespace/braces) + std::uniform_int_distribution distribution(0, cumulative_weights_.back() - 1); + position = std::upper_bound(cumulative_weights_.begin(), cumulative_weights_.end(), + distribution(generator)) - + cumulative_weights_.begin(); + } return PathUtil::JoinPath( PathUtil::JoinPath(external_table_paths_[position], relative_bucket_path_), file_name); } private: ExternalPathProvider(const std::vector& external_table_paths, - const std::string& relative_bucket_path) - : external_table_paths_(external_table_paths), relative_bucket_path_(relative_bucket_path) { + const std::string& relative_bucket_path, + const std::vector& cumulative_weights) + : external_table_paths_(external_table_paths), + relative_bucket_path_(relative_bucket_path), + cumulative_weights_(cumulative_weights) { std::random_device rd; std::mt19937 gen(rd()); std::uniform_int_distribution<> dist(0, external_table_paths_.size() - 1); @@ -68,5 +93,6 @@ class ExternalPathProvider { std::vector external_table_paths_; std::string relative_bucket_path_; std::atomic position_; + std::vector cumulative_weights_; }; } // namespace paimon diff --git a/src/paimon/common/fs/external_path_provider_test.cpp b/src/paimon/common/fs/external_path_provider_test.cpp index 5ef37f2d9..8c529a3f8 100644 --- a/src/paimon/common/fs/external_path_provider_test.cpp +++ b/src/paimon/common/fs/external_path_provider_test.cpp @@ -33,7 +33,8 @@ TEST(ExternalPathProviderTest, TestInvalidExternalDataPath) { std::vector external_table_paths; std::string relative_bucket_path = "p0=1/p1=0/bucket-0"; - auto provider = ExternalPathProvider::Create(external_table_paths, relative_bucket_path); + auto provider = + ExternalPathProvider::Create(external_table_paths, relative_bucket_path, /*weights=*/{}); ASSERT_NOK_WITH_MSG(provider.status(), "external table paths cannot be empty"); } @@ -43,7 +44,8 @@ TEST(ExternalPathProviderTest, TestGetNextExternalDataPath) { std::string relative_bucket_path = "p0=1/p1=0/bucket-0"; ASSERT_OK_AND_ASSIGN(std::unique_ptr provider, - ExternalPathProvider::Create(external_table_paths, relative_bucket_path)); + ExternalPathProvider::Create(external_table_paths, relative_bucket_path, + /*weights=*/{})); ASSERT_EQ(provider->GetNextExternalDataPath("file.orc"), "/tmp/external_path/p0=1/p1=0/bucket-0/file.orc"); } @@ -56,7 +58,8 @@ TEST(ExternalPathProviderTest, TestGetNextExternalDataPath2) { std::string relative_bucket_path = "p0=1/p1=0/bucket-0"; ASSERT_OK_AND_ASSIGN(std::unique_ptr provider, - ExternalPathProvider::Create(external_table_paths, relative_bucket_path)); + ExternalPathProvider::Create(external_table_paths, relative_bucket_path, + /*weights=*/{})); std::set result_data_paths; result_data_paths.insert(provider->GetNextExternalDataPath("file.orc")); @@ -70,6 +73,28 @@ TEST(ExternalPathProviderTest, TestGetNextExternalDataPath2) { })); } +TEST(ExternalPathProviderTest, TestGetNextExternalDataPathWithWeights) { + std::vector external_table_paths = {"/tmp/external_path_a/", + "/tmp/external_path_b/"}; + std::string relative_bucket_path = "p0=1/p1=0/bucket-0"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr provider, + ExternalPathProvider::Create(external_table_paths, relative_bucket_path, + /*weights=*/{9, 1})); + + int32_t first_path_count = 0; + int32_t second_path_count = 0; + for (int32_t i = 0; i < 1000; ++i) { + std::string path = provider->GetNextExternalDataPath("file.orc"); + if (path == "/tmp/external_path_a/p0=1/p1=0/bucket-0/file.orc") { + ++first_path_count; + } else { + ASSERT_EQ(path, "/tmp/external_path_b/p0=1/p1=0/bucket-0/file.orc"); + ++second_path_count; + } + } + ASSERT_GT(first_path_count, second_path_count); +} + TEST(ExternalPathProviderTest, TestGetNextExternalDataPathConcurrently) { std::vector external_table_paths; external_table_paths.emplace_back("/tmp/external_path_a/"); @@ -78,7 +103,8 @@ TEST(ExternalPathProviderTest, TestGetNextExternalDataPathConcurrently) { std::string relative_bucket_path = "p0=1/p1=0/bucket-0"; ASSERT_OK_AND_ASSIGN(std::unique_ptr provider, - ExternalPathProvider::Create(external_table_paths, relative_bucket_path)); + ExternalPathProvider::Create(external_table_paths, relative_bucket_path, + /*weights=*/{})); const std::set expected_data_paths = { "/tmp/external_path_a/p0=1/p1=0/bucket-0/file.orc", diff --git a/src/paimon/common/global_index/bitmap/bitmap_global_index_writer.cpp b/src/paimon/common/global_index/bitmap/bitmap_global_index_writer.cpp index a9c3c2a17..dff896b36 100644 --- a/src/paimon/common/global_index/bitmap/bitmap_global_index_writer.cpp +++ b/src/paimon/common/global_index/bitmap/bitmap_global_index_writer.cpp @@ -142,9 +142,9 @@ Result> BitmapGlobalIndexWriter::Finish() { } SortedIndexFileMeta index_meta(first_key_bytes, last_key_bytes, !null_rows_.IsEmpty()); std::shared_ptr metadata = index_meta.Serialize(pool_.get()); - PAIMON_ASSIGN_OR_RAISE(int64_t file_size, file_writer_->GetFileSize(file_name_)); - return std::vector{GlobalIndexIOMeta( - file_writer_->ToPath(file_name_), file_size, row_count_, std::move(metadata))}; + PAIMON_ASSIGN_OR_RAISE(int64_t file_size, file_writer_->GetFileSize(file_path_)); + return std::vector{ + GlobalIndexIOMeta(file_path_, file_size, row_count_, std::move(metadata))}; } Status BitmapGlobalIndexWriter::FlushCurrentBitmap() { @@ -169,9 +169,9 @@ BitmapGlobalIndexWriter::GetOrCreateStreamingWriter() { if (streaming_writer_ != nullptr) { return streaming_writer_.get(); } - PAIMON_ASSIGN_OR_RAISE(file_name_, file_writer_->NewFileName(kBitmapIdentifier)); + PAIMON_ASSIGN_OR_RAISE(file_path_, file_writer_->NewPath(kBitmapIdentifier)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr output_stream, - file_writer_->NewOutputStream(file_name_)); + file_writer_->NewOutputStream(file_path_)); output_stream_ = std::move(output_stream); PAIMON_ASSIGN_OR_RAISE(streaming_writer_, BitmapGlobalIndexFormat::StreamingWriter::Create( output_stream_, dictionary_block_size_, diff --git a/src/paimon/common/global_index/bitmap/bitmap_global_index_writer.h b/src/paimon/common/global_index/bitmap/bitmap_global_index_writer.h index 3f7e8a2b3..9767985cc 100644 --- a/src/paimon/common/global_index/bitmap/bitmap_global_index_writer.h +++ b/src/paimon/common/global_index/bitmap/bitmap_global_index_writer.h @@ -82,7 +82,7 @@ class BitmapGlobalIndexWriter : public GlobalIndexWriter { std::shared_ptr compression_factory_; std::shared_ptr pool_; - std::string file_name_; + std::string file_path_; std::shared_ptr output_stream_; std::unique_ptr streaming_writer_; int64_t row_count_ = 0; diff --git a/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp b/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp index ce2afefc7..b38bc5a5e 100644 --- a/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp +++ b/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp @@ -50,25 +50,20 @@ class FakeGlobalIndexFileWriter : public GlobalIndexFileWriter { FakeGlobalIndexFileWriter(const std::shared_ptr& fs, const std::string& base_path) : fs_(fs), base_path_(base_path) {} - Result NewFileName(const std::string& prefix) const override { - return prefix + "_" + std::to_string(file_counter_++); + Result NewPath(const std::string& prefix) const override { + return base_path_ + "/" + prefix + "_" + std::to_string(file_counter_++); } Result> NewOutputStream( - const std::string& file_name) const override { - return fs_->Create(base_path_ + "/" + file_name, true); + const std::string& file_path) const override { + return fs_->Create(file_path, true); } - Result GetFileSize(const std::string& file_name) const override { - PAIMON_ASSIGN_OR_RAISE(FileStatus file_status, - fs_->GetFileStatus(base_path_ + "/" + file_name)); + Result GetFileSize(const std::string& file_path) const override { + PAIMON_ASSIGN_OR_RAISE(FileStatus file_status, fs_->GetFileStatus(file_path)); return file_status.GetLen(); } - std::string ToPath(const std::string& file_name) const override { - return base_path_ + "/" + file_name; - } - private: std::shared_ptr fs_; std::string base_path_; diff --git a/src/paimon/common/global_index/btree/btree_global_index_writer.cpp b/src/paimon/common/global_index/btree/btree_global_index_writer.cpp index 614bd366e..278272db2 100644 --- a/src/paimon/common/global_index/btree/btree_global_index_writer.cpp +++ b/src/paimon/common/global_index/btree/btree_global_index_writer.cpp @@ -61,21 +61,21 @@ Result> BTreeGlobalIndexWriter::Create( PAIMON_ASSIGN_OR_RAISE(bloom_filter_builder, BloomFilter::DynamicBuilder(kBloomFilterFpp, pool)); } - PAIMON_ASSIGN_OR_RAISE(std::string index_file_name, - file_writer->NewFileName(BtreeDefs::kIdentifier)); + PAIMON_ASSIGN_OR_RAISE(std::string index_file_path, + file_writer->NewPath(BtreeDefs::kIdentifier)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr output_stream, - file_writer->NewOutputStream(index_file_name)); + file_writer->NewOutputStream(index_file_path)); auto sst_file_writer = std::make_unique(output_stream, bloom_filter_builder, block_size, compression_factory, pool); return std::shared_ptr(new BTreeGlobalIndexWriter( - field_name, arrow_type, key_serializer, file_writer, index_file_name, output_stream, + field_name, arrow_type, key_serializer, file_writer, index_file_path, output_stream, std::move(sst_file_writer), pool)); } BTreeGlobalIndexWriter::BTreeGlobalIndexWriter( const std::string& field_name, const std::shared_ptr& arrow_type, const std::shared_ptr& key_serializer, - const std::shared_ptr& file_writer, const std::string& index_file_name, + const std::shared_ptr& file_writer, const std::string& index_file_path, const std::shared_ptr& output_stream, std::unique_ptr&& sst_writer, const std::shared_ptr& pool) : field_name_(field_name), @@ -83,7 +83,7 @@ BTreeGlobalIndexWriter::BTreeGlobalIndexWriter( key_serializer_(key_serializer), pool_(pool), file_writer_(file_writer), - index_file_name_(index_file_name), + index_file_path_(index_file_path), output_stream_(output_stream), sst_writer_(std::move(sst_writer)) {} @@ -219,9 +219,8 @@ Result> BTreeGlobalIndexWriter::Finish() { auto meta_bytes = index_meta->Serialize(pool_.get()); // Create GlobalIndexIOMeta - std::string file_path = file_writer_->ToPath(index_file_name_); - PAIMON_ASSIGN_OR_RAISE(int64_t file_size, file_writer_->GetFileSize(index_file_name_)); - GlobalIndexIOMeta io_meta(file_path, file_size, row_count_, meta_bytes); + PAIMON_ASSIGN_OR_RAISE(int64_t file_size, file_writer_->GetFileSize(index_file_path_)); + GlobalIndexIOMeta io_meta(index_file_path_, file_size, row_count_, meta_bytes); return std::vector{io_meta}; } diff --git a/src/paimon/common/global_index/btree/btree_global_index_writer.h b/src/paimon/common/global_index/btree/btree_global_index_writer.h index 3fd2d3db4..a69aa854c 100644 --- a/src/paimon/common/global_index/btree/btree_global_index_writer.h +++ b/src/paimon/common/global_index/btree/btree_global_index_writer.h @@ -79,7 +79,7 @@ class BTreeGlobalIndexWriter : public GlobalIndexWriter { const std::shared_ptr& arrow_type, const std::shared_ptr& key_serializer, const std::shared_ptr& file_writer, - const std::string& index_file_name, + const std::string& index_file_path, const std::shared_ptr& output_stream, std::unique_ptr&& sst_writer, const std::shared_ptr& pool); @@ -95,7 +95,7 @@ class BTreeGlobalIndexWriter : public GlobalIndexWriter { std::shared_ptr pool_; std::shared_ptr file_writer_; - std::string index_file_name_; + std::string index_file_path_; std::shared_ptr output_stream_; std::unique_ptr sst_writer_; diff --git a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader_test.cpp b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader_test.cpp index cfc8404ed..32819d312 100644 --- a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader_test.cpp +++ b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader_test.cpp @@ -45,25 +45,20 @@ class FakeLazyFileWriter : public GlobalIndexFileWriter { FakeLazyFileWriter(const std::shared_ptr& fs, const std::string& base_path) : fs_(fs), base_path_(base_path) {} - Result NewFileName(const std::string& prefix) const override { - return prefix + "_" + std::to_string(file_counter_++); + Result NewPath(const std::string& prefix) const override { + return base_path_ + "/" + prefix + "_" + std::to_string(file_counter_++); } Result> NewOutputStream( - const std::string& file_name) const override { - return fs_->Create(base_path_ + "/" + file_name, true); + const std::string& file_path) const override { + return fs_->Create(file_path, true); } - Result GetFileSize(const std::string& file_name) const override { - PAIMON_ASSIGN_OR_RAISE(FileStatus file_status, - fs_->GetFileStatus(base_path_ + "/" + file_name)); + Result GetFileSize(const std::string& file_path) const override { + PAIMON_ASSIGN_OR_RAISE(FileStatus file_status, fs_->GetFileStatus(file_path)); return file_status.GetLen(); } - std::string ToPath(const std::string& file_name) const override { - return base_path_ + "/" + file_name; - } - private: std::shared_ptr fs_; std::string base_path_; diff --git a/src/paimon/common/global_index/wrap/file_index_writer_wrapper.h b/src/paimon/common/global_index/wrap/file_index_writer_wrapper.h index fc2347d6f..2404e4449 100644 --- a/src/paimon/common/global_index/wrap/file_index_writer_wrapper.h +++ b/src/paimon/common/global_index/wrap/file_index_writer_wrapper.h @@ -56,9 +56,9 @@ class FileIndexWriterWrapper : public GlobalIndexWriter { if (count_ == 0) { return std::vector(); } - PAIMON_ASSIGN_OR_RAISE(std::string file_name, file_manager_->NewFileName(index_type_)); + PAIMON_ASSIGN_OR_RAISE(std::string file_path, file_manager_->NewPath(index_type_)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr out, - file_manager_->NewOutputStream(file_name)); + file_manager_->NewOutputStream(file_path)); PAIMON_ASSIGN_OR_RAISE(PAIMON_UNIQUE_PTR bytes, writer_->SerializedBytes()); PAIMON_ASSIGN_OR_RAISE(int64_t actual_size, out->Write(bytes->data(), bytes->size())); @@ -68,7 +68,7 @@ class FileIndexWriterWrapper : public GlobalIndexWriter { } PAIMON_RETURN_NOT_OK(out->Flush()); PAIMON_RETURN_NOT_OK(out->Close()); - GlobalIndexIOMeta meta(file_manager_->ToPath(file_name), /*file_size=*/bytes->size(), + GlobalIndexIOMeta meta(file_path, /*file_size=*/bytes->size(), /*row_count=*/count_, /*metadata=*/nullptr); return std::vector({meta}); } diff --git a/src/paimon/core/append/append_compact_coordinator.cpp b/src/paimon/core/append/append_compact_coordinator.cpp index a72458b31..b49369e2a 100644 --- a/src/paimon/core/append/append_compact_coordinator.cpp +++ b/src/paimon/core/append/append_compact_coordinator.cpp @@ -225,13 +225,16 @@ Result> BuildPathFactory( const std::shared_ptr& pool) { PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, core_options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - core_options.CreateGlobalIndexExternalPath()); + PAIMON_ASSIGN_OR_RAISE(std::vector external_path_weights, + core_options.GetExternalPathWeights()); + PAIMON_ASSIGN_OR_RAISE(std::vector global_index_external_paths, + core_options.CreateGlobalIndexExternalPaths()); return FileStorePathFactory::Create( table_path, arrow_schema, table_schema->PartitionKeys(), core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), external_paths, - global_index_external_path, core_options.IndexFileInDataFileDir(), pool); + global_index_external_paths, core_options.IndexFileInDataFileDir(), external_path_weights, + pool); } /// Scan the latest snapshot and collect small files grouped by partition. diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index 87c89de8c..f222fc02a 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -410,11 +410,11 @@ Result> FileSystemCatalog::GetSchemaExternalPaths( for (const auto& path : data_external_paths) { external_paths_set.insert(path); } - // collect external global index path - PAIMON_ASSIGN_OR_RAISE(std::optional index_external_path, - core_options.CreateGlobalIndexExternalPath()); - if (index_external_path != std::nullopt) { - external_paths_set.insert(index_external_path.value()); + // collect external global index paths + PAIMON_ASSIGN_OR_RAISE(std::vector index_external_paths, + core_options.CreateGlobalIndexExternalPaths()); + for (const auto& path : index_external_paths) { + external_paths_set.insert(path); } } return std::vector(external_paths_set.begin(), external_paths_set.end()); diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index f02af7be3..da45ad798 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -1231,7 +1231,7 @@ TEST(FileSystemCatalogTest, TestDropTableWithMultipleExternalPaths) { ArrowSchemaRelease(&schema); } -TEST(FileSystemCatalogTest, TestDropTableWithGlobalIndexExternalPathOnMainBranch) { +TEST(FileSystemCatalogTest, TestDropTableWithGlobalIndexExternalPathsOnMainBranch) { std::map options; options[Options::FILE_SYSTEM] = "local"; options[Options::FILE_FORMAT] = "orc"; @@ -1246,11 +1246,16 @@ TEST(FileSystemCatalogTest, TestDropTableWithGlobalIndexExternalPathOnMainBranch ASSERT_TRUE(global_index_dir); std::string global_index_path = global_index_dir->Str(); global_index_path = "FILE://" + global_index_path; + auto second_index_dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(second_index_dir); + std::string second_index_path = "FILE://" + second_index_dir->Str(); // Create a file in external path to simulate global index data ASSERT_OK_AND_ASSIGN(auto fs, FileSystemFactory::Get("local", dir->Str(), {})); std::string index_file = PathUtil::JoinPath(global_index_path, "index-file.bin"); ASSERT_OK(fs->WriteFile(index_file, "index data", /*overwrite=*/true)); + std::string second_index_file = PathUtil::JoinPath(second_index_path, "index-file.bin"); + ASSERT_OK(fs->WriteFile(second_index_file, "index data", /*overwrite=*/true)); // Verify index file exists ASSERT_OK_AND_ASSIGN(bool index_file_exists, fs->Exists(index_file)); @@ -1266,7 +1271,8 @@ TEST(FileSystemCatalogTest, TestDropTableWithGlobalIndexExternalPathOnMainBranch ASSERT_TRUE(arrow::ExportSchema(typed_schema, &schema).ok()); std::map table_options = options; - table_options[Options::GLOBAL_INDEX_EXTERNAL_PATH] = global_index_path; + table_options[Options::GLOBAL_INDEX_EXTERNAL_PATH] = + global_index_path + "," + second_index_path; ASSERT_OK(catalog.CreateTable(Identifier("test_db", "tbl_global_index"), &schema, {}, {}, table_options, false)); @@ -1288,11 +1294,13 @@ TEST(FileSystemCatalogTest, TestDropTableWithGlobalIndexExternalPathOnMainBranch // Verify global index external path is also cleaned up ASSERT_OK_AND_ASSIGN(bool global_index_exists, fs->Exists(global_index_path)); ASSERT_FALSE(global_index_exists); + ASSERT_OK_AND_ASSIGN(global_index_exists, fs->Exists(second_index_path)); + ASSERT_FALSE(global_index_exists); ArrowSchemaRelease(&schema); } -TEST(FileSystemCatalogTest, TestDropTableWithGlobalIndexExternalPathOnBranch) { +TEST(FileSystemCatalogTest, TestDropTableWithGlobalIndexExternalPathsOnBranch) { // Copy the real append_table_with_rt_branch.db test data, then patch the // branch-rt schema-1 to include a GLOBAL_INDEX_EXTERNAL_PATH option. // DropTable should discover and clean up that external path. diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp index 4ebe5cc55..4f4a76db0 100644 --- a/src/paimon/core/core_options.cpp +++ b/src/paimon/core/core_options.cpp @@ -222,7 +222,8 @@ class ConfigParser { return ParseEnum(Options::DATA_FILE_EXTERNAL_PATHS_STRATEGY, {{"none", ExternalPathStrategy::NONE}, {"specific-fs", ExternalPathStrategy::SPECIFIC_FS}, - {"round-robin", ExternalPathStrategy::ROUND_ROBIN}}, + {"round-robin", ExternalPathStrategy::ROUND_ROBIN}, + {"weight-robin", ExternalPathStrategy::WEIGHTED}}, "external path strategy", external_path_strategy); } @@ -403,8 +404,9 @@ struct CoreOptions::Impl { std::optional field_default_func; std::optional scan_fallback_branch; std::optional data_file_external_paths; + std::optional data_file_external_paths_weights; std::optional changelog_file_compression; - std::optional global_index_external_path; + std::optional global_index_external_paths; std::optional scan_tag_name; CompressOptions lookup_compress_options{"zstd", 1}; CompressOptions spill_compress_options{"zstd", 1}; @@ -566,6 +568,8 @@ struct CoreOptions::Impl { // Parse data-file.external-paths - external paths for data files, comma separated PAIMON_RETURN_NOT_OK( parser.Parse(Options::DATA_FILE_EXTERNAL_PATHS, &data_file_external_paths)); + PAIMON_RETURN_NOT_OK(parser.Parse(Options::DATA_FILE_EXTERNAL_PATHS_WEIGHTS, + &data_file_external_paths_weights)); // Parse data-file.external-paths.strategy - strategy for selecting external path PAIMON_RETURN_NOT_OK(parser.ParseExternalPathStrategy(&external_path_strategy)); // Parse data-file.prefix - file name prefix of data files, default "data-" @@ -936,9 +940,9 @@ struct CoreOptions::Impl { // Parse global-index.filter.refine-from-data, default false PAIMON_RETURN_NOT_OK(parser.Parse(Options::GLOBAL_INDEX_FILTER_REFINE_FROM_DATA, &global_index_filter_refine_from_data)); - // Parse global-index.external-path - global index root directory + // Parse global-index.external-path - global index root directories PAIMON_RETURN_NOT_OK( - parser.Parse(Options::GLOBAL_INDEX_EXTERNAL_PATH, &global_index_external_path)); + parser.Parse(Options::GLOBAL_INDEX_EXTERNAL_PATH, &global_index_external_paths)); return Status::OK(); } @@ -1862,6 +1866,27 @@ Result> CoreOptions::CreateExternalPaths() const { return external_paths; } +Result> CoreOptions::GetExternalPathWeights() const { + std::vector weights; + if (GetExternalPathStrategy() != ExternalPathStrategy::WEIGHTED || + !impl_->data_file_external_paths_weights) { + return weights; + } + for (const auto& w : StringUtils::Split(impl_->data_file_external_paths_weights.value(), ",")) { + std::string tmp_weight = w; + StringUtils::Trim(&tmp_weight); + if (!tmp_weight.empty() && tmp_weight.front() == '+') { + tmp_weight.erase(0, 1); + } + auto weight = StringUtils::StringToValue(tmp_weight); + if (!weight || weight.value() <= 0) { + return Status::Invalid("External path weight must be a positive integer: ", tmp_weight); + } + weights.push_back(weight.value()); + } + return weights; +} + std::string CoreOptions::DataFilePrefix() const { return impl_->data_file_prefix; } @@ -1936,22 +1961,28 @@ bool CoreOptions::GlobalIndexFilterRefineFromData() const { return impl_->global_index_filter_refine_from_data; } -std::optional CoreOptions::GetGlobalIndexExternalPath() const { - return impl_->global_index_external_path; +std::optional CoreOptions::GetGlobalIndexExternalPaths() const { + return impl_->global_index_external_paths; } -Result> CoreOptions::CreateGlobalIndexExternalPath() const { - std::optional global_index_external_path = GetGlobalIndexExternalPath(); - if (global_index_external_path == std::nullopt || global_index_external_path->empty()) { - return std::optional(); +Result> CoreOptions::CreateGlobalIndexExternalPaths() const { + std::vector external_paths; + std::optional global_index_external_paths = GetGlobalIndexExternalPaths(); + if (!global_index_external_paths || global_index_external_paths->empty()) { + return external_paths; } - std::string tmp_path = global_index_external_path.value(); - StringUtils::Trim(&tmp_path); - PAIMON_ASSIGN_OR_RAISE(Path path, PathUtil::ToPath(tmp_path)); - if (path.scheme.empty()) { - return Status::Invalid(fmt::format("scheme is null, path is {}", tmp_path)); + for (auto tmp_path : StringUtils::Split(global_index_external_paths.value(), ",")) { + StringUtils::Trim(&tmp_path); + PAIMON_ASSIGN_OR_RAISE(Path path, PathUtil::ToPath(tmp_path)); + if (path.scheme.empty()) { + return Status::Invalid(fmt::format("scheme is null, path is {}", tmp_path)); + } + external_paths.push_back(path.ToString()); } - return std::optional(path.ToString()); + if (external_paths.empty()) { + return Status::Invalid("external paths is empty"); + } + return external_paths; } std::optional CoreOptions::GetScanTagName() const { diff --git a/src/paimon/core/core_options.h b/src/paimon/core/core_options.h index bfc6ee251..4be486a9f 100644 --- a/src/paimon/core/core_options.h +++ b/src/paimon/core/core_options.h @@ -243,6 +243,7 @@ class PAIMON_EXPORT CoreOptions { ExternalPathStrategy GetExternalPathStrategy() const; Result> CreateExternalPaths() const; + Result> GetExternalPathWeights() const; bool EnableAdaptivePrefetchStrategy() const; std::string DataFilePrefix() const; @@ -275,7 +276,7 @@ class PAIMON_EXPORT CoreOptions { std::string FormatTableFileCompression() const; bool GlobalIndexEnabled() const; - Result> CreateGlobalIndexExternalPath() const; + Result> CreateGlobalIndexExternalPaths() const; std::optional GetScanTagName() const; @@ -322,7 +323,7 @@ class PAIMON_EXPORT CoreOptions { private: std::optional GetDataFileExternalPaths() const; - std::optional GetGlobalIndexExternalPath() const; + std::optional GetGlobalIndexExternalPaths() const; private: struct Impl; diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp index 1344d9101..eb2457277 100644 --- a/src/paimon/core/core_options_test.cpp +++ b/src/paimon/core/core_options_test.cpp @@ -183,7 +183,7 @@ TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_TRUE(core_options.LegacyPartitionNameEnabled()); ASSERT_TRUE(core_options.GlobalIndexEnabled()); ASSERT_FALSE(core_options.GlobalIndexFilterRefineFromData()); - ASSERT_EQ(std::nullopt, core_options.GetGlobalIndexExternalPath()); + ASSERT_EQ(std::nullopt, core_options.GetGlobalIndexExternalPaths()); ASSERT_EQ(std::nullopt, core_options.GetGlobalIndexThreadNum()); ASSERT_EQ(std::nullopt, core_options.GetScanTagName()); ASSERT_EQ(std::nullopt, core_options.GetOptimizedCompactionInterval()); @@ -491,8 +491,8 @@ TEST(CoreOptionsTest, TestFromMap) { ASSERT_FALSE(core_options.GlobalIndexEnabled()); ASSERT_TRUE(core_options.GlobalIndexFilterRefineFromData()); ASSERT_EQ(core_options.GetGlobalIndexThreadNum(), 4); - ASSERT_TRUE(core_options.GetGlobalIndexExternalPath()); - ASSERT_EQ(core_options.GetGlobalIndexExternalPath().value(), "FILE:///tmp/global_index/"); + ASSERT_TRUE(core_options.GetGlobalIndexExternalPaths()); + ASSERT_EQ(core_options.GetGlobalIndexExternalPaths().value(), "FILE:///tmp/global_index/"); ASSERT_EQ("test-tag", core_options.GetScanTagName().value()); ASSERT_EQ(StartupMode::FromSnapshotFull(), core_options.GetStartupMode()); ASSERT_EQ(375809637, core_options.GetCompactionFileSize(/*has_primary_key=*/true)); @@ -863,22 +863,95 @@ TEST(CoreOptionsTest, TestInvalidCreateExternalPath) { } } -TEST(CoreOptionsTest, TestCreateGlobalIndexExternalPath) { +TEST(CoreOptionsTest, TestCreateGlobalIndexExternalPaths) { std::map options = { {Options::GLOBAL_INDEX_EXTERNAL_PATH, " FILE:///tmp/index1"}, }; ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); - ASSERT_OK_AND_ASSIGN(std::optional external_path, - core_options.CreateGlobalIndexExternalPath()); - ASSERT_EQ("FILE:/tmp/index1", external_path.value()); + ASSERT_OK_AND_ASSIGN(std::vector external_paths, + core_options.CreateGlobalIndexExternalPaths()); + ASSERT_EQ(std::vector({"FILE:/tmp/index1"}), external_paths); +} + +TEST(CoreOptionsTest, GlobalIndexExternalPathsList) { + for (const std::string value : {"FILE:///a, FILE:///b", ",FILE:///a,,FILE:///b,"}) { + ASSERT_OK_AND_ASSIGN(CoreOptions options, + CoreOptions::FromMap({{Options::GLOBAL_INDEX_EXTERNAL_PATH, value}})); + ASSERT_OK_AND_ASSIGN(std::vector paths, + options.CreateGlobalIndexExternalPaths()); + ASSERT_EQ(paths, std::vector({"FILE:/a", "FILE:/b"})); + } + for (const std::string value : {",", " ", "FILE:///a,/b", "FILE:///a, ,FILE:///b"}) { + ASSERT_OK_AND_ASSIGN(CoreOptions options, + CoreOptions::FromMap({{Options::GLOBAL_INDEX_EXTERNAL_PATH, value}})); + ASSERT_NOK(options.CreateGlobalIndexExternalPaths()); + } + ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({})); + ASSERT_OK_AND_ASSIGN(std::vector paths, options.CreateGlobalIndexExternalPaths()); + ASSERT_TRUE(paths.empty()); + ASSERT_OK_AND_ASSIGN(options, + CoreOptions::FromMap({{Options::GLOBAL_INDEX_EXTERNAL_PATH, ""}})); + ASSERT_OK_AND_ASSIGN(paths, options.CreateGlobalIndexExternalPaths()); + ASSERT_TRUE(paths.empty()); +} + +TEST(CoreOptionsTest, ExternalPathWeights) { + for (const std::string value : {"1, 3", "1,3,", "+1,3,,", "1,,3", ",1,3", "1,,3,"}) { + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap({{Options::DATA_FILE_EXTERNAL_PATHS, "FILE:///a,FILE:///b"}, + {Options::DATA_FILE_EXTERNAL_PATHS_STRATEGY, "weight-robin"}, + {Options::DATA_FILE_EXTERNAL_PATHS_WEIGHTS, value}})); + ASSERT_EQ(options.GetExternalPathStrategy(), ExternalPathStrategy::WEIGHTED); + ASSERT_OK_AND_ASSIGN(std::vector weights, options.GetExternalPathWeights()); + ASSERT_EQ(weights, std::vector({1, 3})); + } + for (const std::string value : {"0", "-1", " ", "1, ,", "2147483648", "1.5", "abc"}) { + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap({{Options::DATA_FILE_EXTERNAL_PATHS, "FILE:///a,FILE:///b"}, + {Options::DATA_FILE_EXTERNAL_PATHS_STRATEGY, "weight-robin"}, + {Options::DATA_FILE_EXTERNAL_PATHS_WEIGHTS, value}})); + ASSERT_NOK_WITH_MSG(options.GetExternalPathWeights(), + "External path weight must be a positive integer:"); + } + for (const std::string value : {"", ",", ",,"}) { + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap({{Options::DATA_FILE_EXTERNAL_PATHS, "FILE:///a,FILE:///b"}, + {Options::DATA_FILE_EXTERNAL_PATHS_STRATEGY, "weight-robin"}, + {Options::DATA_FILE_EXTERNAL_PATHS_WEIGHTS, value}})); + ASSERT_OK_AND_ASSIGN(std::vector weights, options.GetExternalPathWeights()); + ASSERT_TRUE(weights.empty()); + } + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap({{Options::DATA_FILE_EXTERNAL_PATHS, "FILE:///a,FILE:///b"}, + {Options::DATA_FILE_EXTERNAL_PATHS_STRATEGY, "weight-robin"}})); + ASSERT_OK_AND_ASSIGN(std::vector weights, options.GetExternalPathWeights()); + ASSERT_TRUE(weights.empty()); +} + +TEST(CoreOptionsTest, ExternalPathWeightsIgnoredForOtherStrategies) { + for (const std::string strategy : {"none", "round-robin"}) { + for (const std::string value : {"1,99", "1", "1,2,3", "0", "abc"}) { + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap({{Options::DATA_FILE_EXTERNAL_PATHS, "FILE:///a,FILE:///b"}, + {Options::DATA_FILE_EXTERNAL_PATHS_STRATEGY, strategy}, + {Options::DATA_FILE_EXTERNAL_PATHS_WEIGHTS, value}})); + ASSERT_OK_AND_ASSIGN(std::vector weights, options.GetExternalPathWeights()); + ASSERT_TRUE(weights.empty()) << strategy << ": " << value; + } + } } -TEST(CoreOptionsTest, TestInvalidCreateGlobalIndexExternalPath) { +TEST(CoreOptionsTest, TestInvalidCreateGlobalIndexExternalPaths) { std::map options = { {Options::GLOBAL_INDEX_EXTERNAL_PATH, "/tmp/index1"}, }; ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); - ASSERT_NOK_WITH_MSG(core_options.CreateGlobalIndexExternalPath(), + ASSERT_NOK_WITH_MSG(core_options.CreateGlobalIndexExternalPaths(), "scheme is null, path is /tmp/index1"); } diff --git a/src/paimon/core/global_index/global_index_file_manager.h b/src/paimon/core/global_index/global_index_file_manager.h index 8ca5ebf70..6e53c7d45 100644 --- a/src/paimon/core/global_index/global_index_file_manager.h +++ b/src/paimon/core/global_index/global_index_file_manager.h @@ -56,16 +56,12 @@ class GlobalIndexFileManager : public GlobalIndexFileReader, return fs_->Open(file_path); } - Result NewFileName(const std::string& prefix) const override { + Result NewPath(const std::string& prefix) const override { std::string uuid; if (PAIMON_UNLIKELY(!UUID::Generate(&uuid))) { return Status::Invalid("fail to generate uuid for global index file manager"); } - return prefix + "-" + "global-index-" + uuid + ".index"; - } - - std::string ToPath(const std::string& file_name) const override { - return path_factory_->ToPath(file_name); + return path_factory_->NewPathFromName(prefix + "-global-index-" + uuid + ".index"); } std::string ToPath(const std::shared_ptr& file) const { @@ -73,12 +69,12 @@ class GlobalIndexFileManager : public GlobalIndexFileReader, } Result> NewOutputStream( - const std::string& file_name) const override { - return fs_->Create(ToPath(file_name), /*overwrite=*/false); + const std::string& file_path) const override { + return fs_->Create(file_path, /*overwrite=*/false); } - Result GetFileSize(const std::string& file_name) const override { - PAIMON_ASSIGN_OR_RAISE(FileStatus file_status, fs_->GetFileStatus(ToPath(file_name))); + Result GetFileSize(const std::string& file_path) const override { + PAIMON_ASSIGN_OR_RAISE(FileStatus file_status, fs_->GetFileStatus(file_path)); return file_status.GetLen(); } @@ -95,8 +91,9 @@ class GlobalIndexFileManager : public GlobalIndexFileReader, return Status::Invalid("global index checkpoint storage is not configured"); } PAIMON_ASSIGN_OR_RAISE(int64_t file_id, NextCheckpointFileId()); - PAIMON_RETURN_NOT_OK(fs_->Mkdirs(checkpoint_path_factory_->GetDirectoryPath())); - return fs_->Create(checkpoint_path_factory_->NewPath(file_id), /*overwrite=*/false); + std::string path = checkpoint_path_factory_->NewPath(file_id); + PAIMON_RETURN_NOT_OK(fs_->Mkdirs(PathUtil::GetParentDirPath(path))); + return fs_->Create(path, /*overwrite=*/false); } Result> OpenCheckpointInputStream() const override { @@ -154,21 +151,20 @@ class GlobalIndexFileManager : public GlobalIndexFileReader, } Result> ListCheckpointFiles() const { - std::vector file_statuses; - PAIMON_RETURN_NOT_OK( - fs_->ListDir(checkpoint_path_factory_->GetDirectoryPath(), &file_statuses)); std::vector checkpoint_files; - for (const BasicFileStatus& file_status : file_statuses) { - if (file_status.IsDir()) { - continue; - } - std::string file_name = PathUtil::GetName(file_status.GetPath()); - std::optional id = checkpoint_path_factory_->GetCheckpointId(file_name); - if (!id) { - continue; + for (const auto& directory : checkpoint_path_factory_->GetDirectoryPaths()) { + std::vector file_statuses; + PAIMON_RETURN_NOT_OK(fs_->ListDir(directory, &file_statuses)); + for (const BasicFileStatus& file_status : file_statuses) { + if (file_status.IsDir()) { + continue; + } + std::string file_name = PathUtil::GetName(file_status.GetPath()); + std::optional id = checkpoint_path_factory_->GetCheckpointId(file_name); + if (id) { + checkpoint_files.push_back(CheckpointFile{id.value(), file_status.GetPath()}); + } } - checkpoint_files.push_back( - CheckpointFile{id.value(), checkpoint_path_factory_->ToPath(file_name)}); } return checkpoint_files; } diff --git a/src/paimon/core/global_index/global_index_file_manager_test.cpp b/src/paimon/core/global_index/global_index_file_manager_test.cpp index f375a266e..e40028cbf 100644 --- a/src/paimon/core/global_index/global_index_file_manager_test.cpp +++ b/src/paimon/core/global_index/global_index_file_manager_test.cpp @@ -19,6 +19,7 @@ #include "paimon/core/global_index/global_index_file_manager.h" +#include #include #include #include @@ -79,16 +80,27 @@ class GlobalIndexFileManagerTest : public ::testing::Test { dir_->Str(), arrow::schema({}), /*partition_keys=*/{}, /*default_part_value=*/"", /*identifier=*/"mock", /*data_file_prefix=*/"data-", /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, - /*global_index_external_path=*/std::nullopt, - /*index_file_in_data_file_dir=*/false, GetDefaultPool())); + /*global_index_external_paths=*/{}, + /*index_file_in_data_file_dir=*/false, /*external_path_weights=*/{}, + GetDefaultPool())); } Result> CreateManager() const { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr checkpoint_path_factory, path_factory_->CreateGlobalIndexCheckpointPathFactory( "lumina", "vector", Range(10, 20), "task-1")); - return std::make_shared( - fs_, path_factory_->CreateGlobalIndexFileFactory(), std::move(checkpoint_path_factory)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr index_path_factory, + path_factory_->CreateGlobalIndexFileFactory()); + return std::make_shared(fs_, std::move(index_path_factory), + std::move(checkpoint_path_factory)); + } + + Status SetExternalPaths(const std::vector& paths) { + PAIMON_ASSIGN_OR_RAISE( + path_factory_, FileStorePathFactory::Create( + dir_->Str(), arrow::schema({}), {}, "", "mock", "data-", true, {}, + paths, false, /*external_path_weights=*/{}, GetDefaultPool())); + return Status::OK(); } Result CreateCheckpointFile( @@ -124,22 +136,23 @@ TEST_F(GlobalIndexFileManagerTest, TestIndexIOWithoutCheckpointAccess) { ASSERT_TRUE(std::dynamic_pointer_cast(writer)); ASSERT_TRUE(std::dynamic_pointer_cast(reader)); ASSERT_TRUE(manager->SupportsCheckpoint()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr index_path_factory, + path_factory_->CreateGlobalIndexFileFactory()); auto plain_manager = std::make_shared( - fs_, path_factory_->CreateGlobalIndexFileFactory(), /*checkpoint_path_factory=*/nullptr); + fs_, std::move(index_path_factory), /*checkpoint_path_factory=*/nullptr); ASSERT_TRUE(std::dynamic_pointer_cast(plain_manager)); ASSERT_FALSE(plain_manager->SupportsCheckpoint()); - ASSERT_OK_AND_ASSIGN(std::string name, writer->NewFileName("lumina")); - ASSERT_TRUE(StringUtils::EndsWith(name, ".index")); - ASSERT_EQ(writer->ToPath(name), PathUtil::JoinPath(dir_->Str(), "index/" + name)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr output, writer->NewOutputStream(name)); + ASSERT_OK_AND_ASSIGN(std::string path, writer->NewPath("lumina")); + ASSERT_TRUE(StringUtils::EndsWith(path, ".index")); + ASSERT_TRUE(StringUtils::StartsWith(path, PathUtil::JoinPath(dir_->Str(), "index/lumina-"))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr output, writer->NewOutputStream(path)); ASSERT_OK_AND_ASSIGN(int64_t written, output->Write("index", 5)); ASSERT_EQ(written, 5); ASSERT_OK(output->Close()); - ASSERT_OK_AND_ASSIGN(int64_t size, writer->GetFileSize(name)); + ASSERT_OK_AND_ASSIGN(int64_t size, writer->GetFileSize(path)); ASSERT_EQ(size, 5); - ASSERT_OK_AND_ASSIGN(std::unique_ptr input, - reader->GetInputStream(writer->ToPath(name))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr input, reader->GetInputStream(path)); char buffer[5]; ASSERT_OK_AND_ASSIGN(int64_t read, input->Read(buffer, sizeof(buffer))); ASSERT_EQ(read, 5); @@ -153,7 +166,9 @@ TEST_F(GlobalIndexFileManagerTest, TestIndexIOWithoutCheckpointAccess) { TEST_F(GlobalIndexFileManagerTest, TestCheckpointNotConfigured) { fs_->fail_list_ = true; - GlobalIndexFileManager manager(fs_, path_factory_->CreateGlobalIndexFileFactory(), + ASSERT_OK_AND_ASSIGN(std::unique_ptr index_path_factory, + path_factory_->CreateGlobalIndexFileFactory()); + GlobalIndexFileManager manager(fs_, std::move(index_path_factory), /*checkpoint_path_factory=*/nullptr); ASSERT_FALSE(manager.SupportsCheckpoint()); ASSERT_NOK_WITH_MSG(manager.CreateCheckpointOutputStream(), @@ -200,7 +215,7 @@ TEST_F(GlobalIndexFileManagerTest, TestCheckpointIOAndRestart) { std::string other_task_path = PathUtil::JoinPath( dir_->Str(), "index/checkpoint/task-2/lumina-global-index-vector-10-20-99.index.ckpt"); - std::string index_path = manager->ToPath("retained.index"); + ASSERT_OK_AND_ASSIGN(std::string index_path, manager->NewPath("retained")); ASSERT_OK(fs_->WriteFile(other_task_path, "other", /*overwrite=*/false)); ASSERT_OK(fs_->WriteFile(index_path, "index", /*overwrite=*/false)); ASSERT_OK(restarted->DeleteCheckpoint()); @@ -223,8 +238,8 @@ TEST_F(GlobalIndexFileManagerTest, TestExternalCheckpointIOAndRestart) { FileStorePathFactory::Create( dir_->Str(), arrow::schema({}), /*partition_keys=*/{}, /*default_part_value=*/"", /*identifier=*/"mock", /*data_file_prefix=*/"data-", - /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, external_path, - /*index_file_in_data_file_dir=*/false, GetDefaultPool())); + /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, {external_path}, + /*index_file_in_data_file_dir=*/false, /*external_path_weights=*/{}, GetDefaultPool())); ASSERT_OK_AND_ASSIGN(std::shared_ptr manager, CreateManager()); std::string directory = PathUtil::JoinPath(external_path, "checkpoint/task-1"); std::string file_name = "lumina-global-index-vector-10-20-0.index.ckpt"; @@ -254,10 +269,11 @@ TEST_F(GlobalIndexFileManagerTest, TestExternalCheckpointIOAndRestart) { ASSERT_OK_AND_ASSIGN(std::string next_name, CreateCheckpointFile(restarted)); ASSERT_EQ(next_name, "lumina-global-index-vector-10-20-1.index.ckpt"); + ASSERT_OK_AND_ASSIGN(std::string retained_index_path, manager->NewPath("retained")); std::vector retained_paths = { local_path, PathUtil::JoinPath(external_path, "checkpoint/task-2/" + file_name), PathUtil::JoinPath(directory, "lumina-global-index-other-10-20-0.index.ckpt"), - manager->ToPath("retained.index")}; + retained_index_path}; for (size_t i = 1; i < retained_paths.size(); ++i) { ASSERT_OK(fs_->WriteFile(retained_paths[i], "retained", /*overwrite=*/false)); } @@ -271,6 +287,80 @@ TEST_F(GlobalIndexFileManagerTest, TestExternalCheckpointIOAndRestart) { } } +TEST_F(GlobalIndexFileManagerTest, CheckpointFilesRoundRobin) { + std::vector roots = {PathUtil::JoinPath(dir_->Str(), "a"), + PathUtil::JoinPath(dir_->Str(), "b"), + PathUtil::JoinPath(dir_->Str(), "c")}; + ASSERT_OK(SetExternalPaths(roots)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr manager, CreateManager()); + std::vector directories; + for (const auto& root : roots) { + directories.push_back(PathUtil::JoinPath(root, "checkpoint/task-1")); + } + + size_t first_directory = 0; + for (size_t i = 0; i < directories.size() * 2; ++i) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr output, + manager->CreateCheckpointOutputStream()); + ASSERT_OK_AND_ASSIGN(std::string path, output->GetUri()); + ASSERT_OK(output->Close()); + if (i == 0) { + // The first directory is random; subsequent files follow the configured order. + auto iter = + std::find(directories.begin(), directories.end(), PathUtil::GetParentDirPath(path)); + ASSERT_NE(iter, directories.end()); + first_directory = iter - directories.begin(); + } + ASSERT_EQ(path, PathUtil::JoinPath( + directories[(first_directory + i) % directories.size()], + fmt::format("lumina-global-index-vector-10-20-{}.index.ckpt", i))); + } +} + +TEST_F(GlobalIndexFileManagerTest, LatestCheckpointIgnoresDuplicateOlderIds) { + std::string root_a = PathUtil::JoinPath(dir_->Str(), "a"); + std::string root_b = PathUtil::JoinPath(dir_->Str(), "b"); + ASSERT_OK(SetExternalPaths({root_a, root_b})); + for (const auto& root : {root_a, root_b}) { + ASSERT_OK(fs_->WriteFile( + PathUtil::JoinPath(root, + "checkpoint/task-1/lumina-global-index-vector-10-20-9.index.ckpt"), + "older", false)); + } + std::string latest_path = PathUtil::JoinPath( + root_b, "checkpoint/task-1/lumina-global-index-vector-10-20-10.index.ckpt"); + ASSERT_OK(fs_->WriteFile(latest_path, "latest", false)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr manager, CreateManager()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr input, manager->OpenCheckpointInputStream()); + ASSERT_OK_AND_ASSIGN(std::string recovered_path, input->GetUri()); + ASSERT_EQ(recovered_path, latest_path); + ASSERT_OK(input->Close()); + ASSERT_OK_AND_ASSIGN(std::string next, CreateCheckpointFile(manager)); + ASSERT_TRUE(StringUtils::EndsWith(next, "-11.index.ckpt")); + ASSERT_OK(manager->DeleteCheckpoint()); + ASSERT_OK_AND_ASSIGN(bool exists, manager->CheckpointExists()); + ASSERT_FALSE(exists); +} + +TEST_F(GlobalIndexFileManagerTest, TaskCleanupVisitsAllRootsAfterAnError) { + std::string root_a = "file:" + PathUtil::JoinPath(dir_->Str(), "a"); + std::string root_b = "file:" + PathUtil::JoinPath(dir_->Str(), "b"); + ASSERT_OK(CreateSchema({{Options::GLOBAL_INDEX_EXTERNAL_PATH, root_a + "," + root_b}})); + std::string directory_a = PathUtil::JoinPath(root_a, "checkpoint/task-1"); + std::string directory_b = PathUtil::JoinPath(root_b, "checkpoint/task-1"); + ASSERT_OK(fs_->WriteFile(PathUtil::JoinPath(directory_a, "unfinished"), "x", false)); + ASSERT_OK(fs_->WriteFile(PathUtil::JoinPath(directory_b, "unfinished"), "x", false)); + fs_->fail_delete_path_ = directory_a; + ASSERT_NOK(GlobalIndexWriteTask::DeleteCheckpoint(dir_->Str(), "task-1", {}, fs_)); + ASSERT_OK_AND_ASSIGN(bool exists, fs_->Exists(directory_b)); + ASSERT_FALSE(exists); + ASSERT_OK_AND_ASSIGN(exists, fs_->Exists(directory_a)); + ASSERT_TRUE(exists); + fs_->fail_delete_path_.clear(); + ASSERT_OK(GlobalIndexWriteTask::DeleteCheckpoint(dir_->Str(), "task-1", {}, fs_)); + ASSERT_OK(GlobalIndexWriteTask::DeleteCheckpoint(dir_->Str(), "task-1", {}, fs_)); +} + TEST_F(GlobalIndexFileManagerTest, TestInitializationFailureCanRetry) { ASSERT_OK_AND_ASSIGN(std::shared_ptr manager, CreateManager()); fs_->fail_list_ = true; @@ -377,6 +467,24 @@ TEST_F(GlobalIndexFileManagerTest, TestDeleteDoesNotResetFileId) { ASSERT_EQ(fs_->list_count_, 2); } +TEST_F(GlobalIndexFileManagerTest, CheckpointCleanupPreservesDeleteErrors) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr manager, CreateManager()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr output, + manager->CreateCheckpointOutputStream()); + ASSERT_OK_AND_ASSIGN(std::string path, output->GetUri()); + ASSERT_OK(output->Close()); + + fs_->fail_delete_path_ = path; + ASSERT_NOK_WITH_MSG(manager->DeleteCheckpoint(), "checkpoint delete failed"); + ASSERT_OK_AND_ASSIGN(bool exists, fs_->Exists(path)); + ASSERT_TRUE(exists); + + fs_->fail_delete_path_.clear(); + ASSERT_OK(manager->DeleteCheckpoint()); + ASSERT_OK_AND_ASSIGN(exists, manager->CheckpointExists()); + ASSERT_FALSE(exists); +} + TEST_F(GlobalIndexFileManagerTest, TestDeleteTaskCheckpoint) { ASSERT_OK(CreateSchema()); std::string directory = PathUtil::JoinPath(dir_->Str(), "index/checkpoint/task-1"); diff --git a/src/paimon/core/global_index/global_index_live_row_filter.cpp b/src/paimon/core/global_index/global_index_live_row_filter.cpp index 3d554fb19..95c22d721 100644 --- a/src/paimon/core/global_index/global_index_live_row_filter.cpp +++ b/src/paimon/core/global_index/global_index_live_row_filter.cpp @@ -67,15 +67,18 @@ Result> CreateSnapshotReader( std::string branch = BranchManager::NormalizeBranch(options.GetBranch()); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - options.CreateGlobalIndexExternalPath()); + PAIMON_ASSIGN_OR_RAISE(std::vector external_path_weights, + options.GetExternalPathWeights()); + PAIMON_ASSIGN_OR_RAISE(std::vector global_index_external_paths, + options.CreateGlobalIndexExternalPaths()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, FileStorePathFactory::Create( root_path, arrow_schema, table_schema->PartitionKeys(), options.GetPartitionDefaultName(), options.GetFileFormat()->Identifier(), options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), external_paths, - global_index_external_path, options.IndexFileInDataFileDir(), pool)); + global_index_external_paths, options.IndexFileInDataFileDir(), external_path_weights, + pool)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr manifest_list, ManifestList::Create(fs, options.GetManifestFormat(), options.GetManifestCompression(), diff --git a/src/paimon/core/global_index/global_index_scan_impl.cpp b/src/paimon/core/global_index/global_index_scan_impl.cpp index 70b269307..1286386d0 100644 --- a/src/paimon/core/global_index/global_index_scan_impl.cpp +++ b/src/paimon/core/global_index/global_index_scan_impl.cpp @@ -83,17 +83,20 @@ Result> GlobalIndexScanImpl::Create( GetOrCreateExecutor(executor, options)); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - options.CreateGlobalIndexExternalPath()); + PAIMON_ASSIGN_OR_RAISE(std::vector external_path_weights, + options.GetExternalPathWeights()); + PAIMON_ASSIGN_OR_RAISE(std::vector global_index_external_paths, + options.CreateGlobalIndexExternalPaths()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr file_store_path_factory, FileStorePathFactory::Create( root_path, arrow_schema, table_schema->PartitionKeys(), options.GetPartitionDefaultName(), options.GetFileFormat()->Identifier(), options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), external_paths, - global_index_external_path, options.IndexFileInDataFileDir(), pool)); - std::shared_ptr path_factory = - file_store_path_factory->CreateGlobalIndexFileFactory(); + global_index_external_paths, options.IndexFileInDataFileDir(), external_path_weights, + pool)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr path_factory, + file_store_path_factory->CreateGlobalIndexFileFactory()); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr index_manifest_file, diff --git a/src/paimon/core/global_index/global_index_write_task.cpp b/src/paimon/core/global_index/global_index_write_task.cpp index cb11fd920..7612779ec 100644 --- a/src/paimon/core/global_index/global_index_write_task.cpp +++ b/src/paimon/core/global_index/global_index_write_task.cpp @@ -87,13 +87,16 @@ Result> CreateFileStorePathFactory( auto all_arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, core_options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - core_options.CreateGlobalIndexExternalPath()); + PAIMON_ASSIGN_OR_RAISE(std::vector external_path_weights, + core_options.GetExternalPathWeights()); + PAIMON_ASSIGN_OR_RAISE(std::vector global_index_external_paths, + core_options.CreateGlobalIndexExternalPaths()); return FileStorePathFactory::Create( table_path, all_arrow_schema, table_schema->PartitionKeys(), core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), external_paths, - global_index_external_path, core_options.IndexFileInDataFileDir(), pool); + global_index_external_paths, core_options.IndexFileInDataFileDir(), external_path_weights, + pool); } Result> CreateGlobalIndexWriter( @@ -391,8 +394,10 @@ Result> GlobalIndexWriteTask::WriteIndex( path_factory->CreateGlobalIndexCheckpointPathFactory( index_type, field_name, range, task_id.value())); } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr index_path_factory, + path_factory->CreateGlobalIndexFileFactory()); auto index_file_manager = std::make_shared( - core_options.GetFileSystem(), path_factory->CreateGlobalIndexFileFactory(), + core_options.GetFileSystem(), std::move(index_path_factory), std::move(checkpoint_path_factory)); // create batch reader @@ -429,18 +434,24 @@ Status GlobalIndexWriteTask::DeleteCheckpoint(const std::string& table_path, PAIMON_ASSIGN_OR_RAISE(schema_and_options, LoadSchemaAndOptions(table_path, options, file_system)); const CoreOptions& core_options = schema_and_options.second; - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - core_options.CreateGlobalIndexExternalPath()); - PAIMON_ASSIGN_OR_RAISE(std::string task_directory, - IndexCheckpointPathFactory::GetTaskDirectory( - table_path, task_id, global_index_external_path)); + PAIMON_ASSIGN_OR_RAISE(std::vector global_index_external_paths, + core_options.CreateGlobalIndexExternalPaths()); + PAIMON_ASSIGN_OR_RAISE(std::vector task_directories, + IndexCheckpointPathFactory::GetTaskDirectories( + table_path, task_id, global_index_external_paths)); std::shared_ptr fs = core_options.GetFileSystem(); - PAIMON_ASSIGN_OR_RAISE(bool exists, fs->Exists(task_directory)); - if (!exists) { - return Status::OK(); + Status first_error = Status::OK(); + for (const auto& task_directory : task_directories) { + auto exists = fs->Exists(task_directory); + Status status = exists.status(); + if (exists.ok() && exists.value()) { + status = fs->Delete(task_directory, /*recursive=*/true); + } + if (!status.ok() && !status.IsNotExist() && first_error.ok()) { + first_error = status; + } } - Status status = fs->Delete(task_directory, /*recursive=*/true); - return status.IsNotExist() ? Status::OK() : status; + return first_error; } } // namespace paimon diff --git a/src/paimon/core/index/index_checkpoint_path_factory.h b/src/paimon/core/index/index_checkpoint_path_factory.h index cdc043bae..4366532c7 100644 --- a/src/paimon/core/index/index_checkpoint_path_factory.h +++ b/src/paimon/core/index/index_checkpoint_path_factory.h @@ -21,6 +21,7 @@ #include #include #include +#include #include "paimon/common/utils/path_util.h" #include "paimon/result.h" @@ -35,22 +36,28 @@ class IndexCheckpointPathFactory { /// Creates the path for the specified checkpoint id, without storage I/O. virtual std::string NewPath(int64_t checkpoint_id) const = 0; - virtual std::string ToPath(const std::string& file_name) const = 0; - /// Returns the directory containing the checkpoint files. - virtual const std::string& GetDirectoryPath() const = 0; + /// Returns the directories containing the checkpoint files. + virtual const std::vector& GetDirectoryPaths() const = 0; /// Returns the checkpoint id when the file name belongs to this factory. virtual std::optional GetCheckpointId(const std::string& file_name) const = 0; - /// Returns the task's checkpoint directory under the configured global index directory. - static Result GetTaskDirectory( + /// Returns the task's checkpoint directories under the configured global index directories. + static Result> GetTaskDirectories( const std::string& table_path, const std::string& task_id, - const std::optional& global_index_external_path) { + const std::vector& global_index_external_paths) { PAIMON_RETURN_NOT_OK(PathUtil::CheckSinglePathComponent("checkpoint task id", task_id)); - std::string index_directory = - global_index_external_path.value_or(PathUtil::JoinPath(table_path, "index")); - return PathUtil::JoinPath(PathUtil::JoinPath(index_directory, "checkpoint"), task_id); + std::vector roots = {PathUtil::JoinPath(table_path, "index")}; + if (!global_index_external_paths.empty()) { + roots = global_index_external_paths; + } + std::vector directories; + for (const auto& root : roots) { + directories.push_back( + PathUtil::JoinPath(PathUtil::JoinPath(root, "checkpoint"), task_id)); + } + return directories; } }; diff --git a/src/paimon/core/index/index_file_handler_test.cpp b/src/paimon/core/index/index_file_handler_test.cpp index a7eac7fb7..9ff497f6d 100644 --- a/src/paimon/core/index/index_file_handler_test.cpp +++ b/src/paimon/core/index/index_file_handler_test.cpp @@ -65,8 +65,8 @@ class IndexFileHandlerTest : public testing::Test { auto schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, core_options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - core_options.CreateGlobalIndexExternalPath()); + PAIMON_ASSIGN_OR_RAISE(std::vector global_index_external_paths, + core_options.CreateGlobalIndexExternalPaths()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, @@ -75,9 +75,9 @@ class IndexFileHandlerTest : public testing::Test { core_options.GetPartitionDefaultName(), /*identifier=*/"orc", core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), external_paths, - global_index_external_path, + global_index_external_paths, /*index_file_in_data_file_dir=*/core_options.IndexFileInDataFileDir(), - memory_pool_)); + /*external_path_weights=*/{}, memory_pool_)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr index_manifest_file, IndexManifestFile::Create( core_options.GetFileSystem(), core_options.GetManifestFormat(), diff --git a/src/paimon/core/index/index_in_data_file_dir_path_factory.h b/src/paimon/core/index/index_in_data_file_dir_path_factory.h index 7339ddf8c..2aac9fd37 100644 --- a/src/paimon/core/index/index_in_data_file_dir_path_factory.h +++ b/src/paimon/core/index/index_in_data_file_dir_path_factory.h @@ -43,7 +43,11 @@ class IndexInDataFileDirPathFactory : public IndexPathFactory { std::string NewPath() const override { std::string name = IndexPathFactory::INDEX_PREFIX + uuid_ + "-" + std::to_string(index_file_count_->fetch_add(1)); - return data_file_path_factory_->NewPathFromName(name); + return NewPathFromName(name); + } + + std::string NewPathFromName(const std::string& file_name) const override { + return data_file_path_factory_->NewPathFromName(file_name); } std::string ToPath(const std::shared_ptr& file) const override { @@ -53,10 +57,6 @@ class IndexInDataFileDirPathFactory : public IndexPathFactory { return PathUtil::JoinPath(data_file_path_factory_->Parent(), file->FileName()); } - std::string ToPath(const std::string& file_name) const override { - return data_file_path_factory_->NewPathFromName(file_name); - } - bool IsExternalPath() const override { return data_file_path_factory_->IsExternalPath(); } diff --git a/src/paimon/core/index/index_in_data_file_dir_path_factory_test.cpp b/src/paimon/core/index/index_in_data_file_dir_path_factory_test.cpp index fe49a46e4..ed12078d6 100644 --- a/src/paimon/core/index/index_in_data_file_dir_path_factory_test.cpp +++ b/src/paimon/core/index/index_in_data_file_dir_path_factory_test.cpp @@ -50,8 +50,7 @@ TEST(IndexInDataFileDirPathFactoryTest, TestSimple) { /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt); ASSERT_EQ(factory.ToPath(meta), "/tmp/p0=1/p1=0/bucket-0/deletion_file"); - // test ToPath with file_name - ASSERT_EQ(factory.ToPath("bitmap.index"), "/tmp/p0=1/p1=0/bucket-0/bitmap.index"); + ASSERT_EQ(factory.NewPathFromName("bitmap.index"), "/tmp/p0=1/p1=0/bucket-0/bitmap.index"); // test external path ASSERT_FALSE(factory.IsExternalPath()); } @@ -60,9 +59,9 @@ TEST(IndexInDataFileDirPathFactoryTest, TestWithExternalPath) { auto count = std::make_shared>(); count->store(1); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr external_path_provider, - ExternalPathProvider::Create({"/tmp/external_path/"}, "p0=1/p1=0/bucket-0")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr external_path_provider, + ExternalPathProvider::Create({"/tmp/external_path/"}, "p0=1/p1=0/bucket-0", + /*weights=*/{})); auto data_file_path_factory = std::make_shared(); ASSERT_OK( diff --git a/src/paimon/core/index/index_path_factory.h b/src/paimon/core/index/index_path_factory.h index 3cf7063d8..d9857145c 100644 --- a/src/paimon/core/index/index_path_factory.h +++ b/src/paimon/core/index/index_path_factory.h @@ -31,7 +31,7 @@ class IndexPathFactory { virtual ~IndexPathFactory() = default; virtual std::string NewPath() const = 0; - virtual std::string ToPath(const std::string& file_name) const = 0; + virtual std::string NewPathFromName(const std::string& file_name) const = 0; virtual std::string ToPath(const std::shared_ptr& file) const = 0; virtual bool IsExternalPath() const = 0; diff --git a/src/paimon/core/index/pksorted/pk_sorted_index_builder.cpp b/src/paimon/core/index/pksorted/pk_sorted_index_builder.cpp index 4d36aa1f3..616293283 100644 --- a/src/paimon/core/index/pksorted/pk_sorted_index_builder.cpp +++ b/src/paimon/core/index/pksorted/pk_sorted_index_builder.cpp @@ -59,34 +59,30 @@ class TrackingGlobalIndexFileWriter : public GlobalIndexFileWriter { explicit TrackingGlobalIndexFileWriter(const std::shared_ptr& delegate) : delegate_(delegate) {} - Result NewFileName(const std::string& prefix) const override { - PAIMON_ASSIGN_OR_RAISE(std::string file_name, delegate_->NewFileName(prefix)); - created_file_names_.push_back(file_name); - return file_name; + Result NewPath(const std::string& prefix) const override { + PAIMON_ASSIGN_OR_RAISE(std::string file_path, delegate_->NewPath(prefix)); + created_file_paths_.push_back(file_path); + return file_path; } Result> NewOutputStream( - const std::string& file_name) const override { - return delegate_->NewOutputStream(file_name); + const std::string& file_path) const override { + return delegate_->NewOutputStream(file_path); } - Result GetFileSize(const std::string& file_name) const override { - return delegate_->GetFileSize(file_name); - } - - std::string ToPath(const std::string& file_name) const override { - return delegate_->ToPath(file_name); + Result GetFileSize(const std::string& file_path) const override { + return delegate_->GetFileSize(file_path); } void Cleanup(const std::shared_ptr& fs) const { - for (const std::string& file_name : created_file_names_) { - [[maybe_unused]] Status status = fs->Delete(delegate_->ToPath(file_name)); + for (const std::string& file_path : created_file_paths_) { + [[maybe_unused]] Status status = fs->Delete(file_path); } } private: std::shared_ptr delegate_; - mutable std::vector created_file_names_; + mutable std::vector created_file_paths_; }; } // namespace diff --git a/src/paimon/core/io/data_file_path_factory_test.cpp b/src/paimon/core/io/data_file_path_factory_test.cpp index fe4824c94..380acf47a 100644 --- a/src/paimon/core/io/data_file_path_factory_test.cpp +++ b/src/paimon/core/io/data_file_path_factory_test.cpp @@ -68,9 +68,9 @@ TEST_F(DataFilePathFactoryTest, TestNewChangelogPath) { TEST_F(DataFilePathFactoryTest, TestNewPathWithDataFilePrefixAndExternalPath) { DataFilePathFactory factory; - ASSERT_OK_AND_ASSIGN( - std::unique_ptr external_path_provider, - ExternalPathProvider::Create({"/tmp/external_path/"}, "p0=1/p1=0/bucket-0")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr external_path_provider, + ExternalPathProvider::Create({"/tmp/external_path/"}, "p0=1/p1=0/bucket-0", + /*weights=*/{})); ASSERT_OK(factory_.Init(/*parent=*/"/tmp/p0=1/p1=0/bucket-0/", /*format_identifier=*/"txt", /*data_file_prefix=*/"test-data-", std::move(external_path_provider))); diff --git a/src/paimon/core/manifest/index_manifest_file_handler_test.cpp b/src/paimon/core/manifest/index_manifest_file_handler_test.cpp index 13045ed1c..6176900ab 100644 --- a/src/paimon/core/manifest/index_manifest_file_handler_test.cpp +++ b/src/paimon/core/manifest/index_manifest_file_handler_test.cpp @@ -65,8 +65,8 @@ class IndexManifestFileHandlerTest : public testing::Test { dir_->Str(), schema, /*partition_keys=*/{}, /*default_part_value=*/"", file_format->Identifier(), /*data_file_prefix=*/"data-", /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, - /*global_index_external_path=*/std::nullopt, - /*index_file_in_data_file_dir=*/false, pool_)); + /*global_index_external_paths=*/{}, + /*index_file_in_data_file_dir=*/false, /*external_path_weights=*/{}, pool_)); PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap({})); options.WithCache(cache); return IndexManifestFile::Create(dir_->GetFileSystem(), file_format, "zstd", path_factory, diff --git a/src/paimon/core/manifest/manifest_file_test.cpp b/src/paimon/core/manifest/manifest_file_test.cpp index b0ce4b9e4..551075b21 100644 --- a/src/paimon/core/manifest/manifest_file_test.cpp +++ b/src/paimon/core/manifest/manifest_file_test.cpp @@ -134,14 +134,15 @@ class ManifestFileTest : public testing::Test { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_format, FileFormatFactory::Get(file_format_str, {})); auto unused_schema = arrow::schema(arrow::FieldVector({arrow::field("f0", arrow::utf8())})); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr path_factory, - FileStorePathFactory::Create( - root_path, unused_schema, /*partition_keys=*/{}, - /*default_part_value=*/"", file_format->Identifier(), - /*data_file_prefix=*/"data-", - /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, - /*global_index_external_path=*/std::nullopt, - /*index_file_in_data_file_dir=*/false, pool)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr path_factory, + FileStorePathFactory::Create( + root_path, unused_schema, /*partition_keys=*/{}, + /*default_part_value=*/"", file_format->Identifier(), + /*data_file_prefix=*/"data-", + /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, + /*global_index_external_paths=*/{}, + /*index_file_in_data_file_dir=*/false, /*external_path_weights=*/{}, pool)); PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}})); if (cache_enabled) { @@ -288,8 +289,9 @@ TEST_F(ManifestFileTest, TestManifestCacheIsDisabledWithoutInjectedCache) { /*default_part_value=*/"", file_format->Identifier(), /*data_file_prefix=*/"data-", /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, - /*global_index_external_path=*/std::nullopt, - /*index_file_in_data_file_dir=*/false, pool)); + /*global_index_external_paths=*/{}, + /*index_file_in_data_file_dir=*/false, + /*external_path_weights=*/{}, pool)); ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}})); ASSERT_OK_AND_ASSIGN( @@ -329,8 +331,9 @@ TEST_F(ManifestFileTest, TestManifestCacheReusesDecodedBatches) { /*default_part_value=*/"", file_format->Identifier(), /*data_file_prefix=*/"data-", /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, - /*global_index_external_path=*/std::nullopt, - /*index_file_in_data_file_dir=*/false, pool)); + /*global_index_external_paths=*/{}, + /*index_file_in_data_file_dir=*/false, + /*external_path_weights=*/{}, pool)); ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}})); options.WithCache(manifest_cache); @@ -369,8 +372,9 @@ TEST_F(ManifestFileTest, TestReadBucketEntriesMaterializesOnlySelectedBucket) { /*default_part_value=*/"", file_format->Identifier(), /*data_file_prefix=*/"data-", /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, - /*global_index_external_path=*/std::nullopt, - /*index_file_in_data_file_dir=*/false, pool)); + /*global_index_external_paths=*/{}, + /*index_file_in_data_file_dir=*/false, + /*external_path_weights=*/{}, pool)); ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}})); options.WithCache(manifest_cache); @@ -422,8 +426,9 @@ TEST_F(ManifestFileTest, TestReadPassesKnownSizeToOpen) { /*default_part_value=*/"", file_format->Identifier(), /*data_file_prefix=*/"data-", /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, - /*global_index_external_path=*/std::nullopt, - /*index_file_in_data_file_dir=*/false, pool)); + /*global_index_external_paths=*/{}, + /*index_file_in_data_file_dir=*/false, + /*external_path_weights=*/{}, pool)); ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}})); ASSERT_OK_AND_ASSIGN( @@ -467,14 +472,15 @@ TEST_F(ManifestFileTest, TestInferredBucketProbeSkipsArrowMaterialization) { ASSERT_OK_AND_ASSIGN(std::shared_ptr file_format, FileFormatFactory::Get("avro", {})); auto unused_schema = arrow::schema(arrow::FieldVector({arrow::field("f0", arrow::utf8())})); - ASSERT_OK_AND_ASSIGN(std::shared_ptr path_factory, - FileStorePathFactory::Create( - test_dir->Str(), unused_schema, /*partition_keys=*/{}, - /*default_part_value=*/"", file_format->Identifier(), - /*data_file_prefix=*/"data-", - /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, - /*global_index_external_path=*/std::nullopt, - /*index_file_in_data_file_dir=*/false, pool)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr path_factory, + FileStorePathFactory::Create( + test_dir->Str(), unused_schema, /*partition_keys=*/{}, + /*default_part_value=*/"", file_format->Identifier(), + /*data_file_prefix=*/"data-", + /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, + /*global_index_external_paths=*/{}, + /*index_file_in_data_file_dir=*/false, /*external_path_weights=*/{}, pool)); ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}})); auto cache = std::make_shared( @@ -541,14 +547,15 @@ TEST_F(ManifestFileTest, TestReadBucketEntriesSkipsDeserializingOtherBuckets) { ASSERT_OK_AND_ASSIGN(std::shared_ptr file_format, FileFormatFactory::Get("avro", {})); auto unused_schema = arrow::schema(arrow::FieldVector({arrow::field("f0", arrow::utf8())})); - ASSERT_OK_AND_ASSIGN(std::shared_ptr path_factory, - FileStorePathFactory::Create( - test_dir->Str(), unused_schema, /*partition_keys=*/{}, - /*default_part_value=*/"", file_format->Identifier(), - /*data_file_prefix=*/"data-", - /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, - /*global_index_external_path=*/std::nullopt, - /*index_file_in_data_file_dir=*/false, pool)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr path_factory, + FileStorePathFactory::Create( + test_dir->Str(), unused_schema, /*partition_keys=*/{}, + /*default_part_value=*/"", file_format->Identifier(), + /*data_file_prefix=*/"data-", + /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, + /*global_index_external_paths=*/{}, + /*index_file_in_data_file_dir=*/false, /*external_path_weights=*/{}, pool)); ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}})); if (cache_mode != 0) { @@ -739,8 +746,9 @@ TEST_F(ManifestFileTest, TestLegacyManifestFormatIsReadOnly) { /*default_part_value=*/"", file_format->Identifier(), /*data_file_prefix=*/"data-", /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, - /*global_index_external_path=*/std::nullopt, - /*index_file_in_data_file_dir=*/false, pool)); + /*global_index_external_paths=*/{}, + /*index_file_in_data_file_dir=*/false, + /*external_path_weights=*/{}, pool)); ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({})); ASSERT_OK_AND_ASSIGN( std::unique_ptr manifest_file, diff --git a/src/paimon/core/manifest/manifest_list_test.cpp b/src/paimon/core/manifest/manifest_list_test.cpp index 8653eaf88..119edde5f 100644 --- a/src/paimon/core/manifest/manifest_list_test.cpp +++ b/src/paimon/core/manifest/manifest_list_test.cpp @@ -105,14 +105,15 @@ class ManifestListTest : public testing::Test { EXPECT_OK_AND_ASSIGN(std::shared_ptr file_format, FileFormatFactory::Get(file_format_str, {})); auto unused_schema = arrow::schema(arrow::FieldVector({arrow::field("f0", arrow::utf8())})); - EXPECT_OK_AND_ASSIGN(std::shared_ptr path_factory, - FileStorePathFactory::Create( - root_path, unused_schema, /*partition_keys=*/{}, - /*default_part_value=*/"", file_format->Identifier(), - /*data_file_prefix=*/"data-", - /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, - /*global_index_external_path=*/std::nullopt, - /*index_file_in_data_file_dir=*/false, pool)); + EXPECT_OK_AND_ASSIGN( + std::shared_ptr path_factory, + FileStorePathFactory::Create( + root_path, unused_schema, /*partition_keys=*/{}, + /*default_part_value=*/"", file_format->Identifier(), + /*data_file_prefix=*/"data-", + /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, + /*global_index_external_paths=*/{}, + /*index_file_in_data_file_dir=*/false, /*external_path_weights=*/{}, pool)); EXPECT_OK_AND_ASSIGN( auto manifest_list, ManifestList::Create(file_system, file_format, "zstd", path_factory, cache, pool)); diff --git a/src/paimon/core/manifest/manifest_row_range_test.cpp b/src/paimon/core/manifest/manifest_row_range_test.cpp index db2dd75cd..02089e1fb 100644 --- a/src/paimon/core/manifest/manifest_row_range_test.cpp +++ b/src/paimon/core/manifest/manifest_row_range_test.cpp @@ -61,9 +61,10 @@ class RowRangeManifestFileTest : public ::testing::Test { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format, FileFormatFactory::Get("avro", {})); auto schema = arrow::schema(arrow::FieldVector{}); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr path_factory, - FileStorePathFactory::Create(path, schema, {}, "", "avro", "data-", - true, {}, std::nullopt, false, pool)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr path_factory, + FileStorePathFactory::Create(path, schema, {}, "", "avro", "data-", true, {}, {}, false, + /*external_path_weights=*/{}, pool)); PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap({{Options::READ_BATCH_SIZE, "2"}})); if (cache) { @@ -283,8 +284,8 @@ TEST_F(RowRangeManifestFileTest, ScanPlanPreservesResultsAcrossLazyDecodeAndCach FileFormatFactory::Get("avro", {})); ASSERT_OK_AND_ASSIGN( std::shared_ptr paths, - FileStorePathFactory::Create(dir->Str(), schema, {}, "", "avro", "data-", true, {}, - std::nullopt, false, pool)); + FileStorePathFactory::Create(dir->Str(), schema, {}, "", "avro", "data-", true, {}, {}, + false, /*external_path_weights=*/{}, pool)); ASSERT_OK_AND_ASSIGN(std::shared_ptr lists, ManifestList::Create(fs, format, "null", paths, cache, pool)); ASSERT_OK_AND_ASSIGN(WrittenFile base_list, lists->Write({base_meta})); @@ -347,9 +348,10 @@ TEST_F(RowRangeManifestFileTest, OrcCachePreservesManifestMetadata) { ASSERT_OK_AND_ASSIGN(std::shared_ptr format, FileFormatFactory::Get("orc", {})); auto schema = arrow::schema(arrow::FieldVector{}); const std::string path = GetDataDir() + "/orc/append_09.db/append_09"; - ASSERT_OK_AND_ASSIGN(std::shared_ptr paths, - FileStorePathFactory::Create(path, schema, {}, "", "orc", "data-", true, - {}, std::nullopt, false, pool)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr paths, + FileStorePathFactory::Create(path, schema, {}, "", "orc", "data-", true, {}, {}, false, + /*external_path_weights=*/{}, pool)); ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{Options::READ_BATCH_SIZE, "2"}})); ASSERT_OK_AND_ASSIGN( diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp index bcaf72798..7fb611855 100644 --- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp +++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp @@ -373,15 +373,15 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam external_paths, options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - options.CreateGlobalIndexExternalPath()); + PAIMON_ASSIGN_OR_RAISE(std::vector global_index_external_paths, + options.CreateGlobalIndexExternalPaths()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, FileStorePathFactory::Create( table_path, arrow_schema_, /*partition_keys=*/{}, options.GetPartitionDefaultName(), options.GetFileFormat()->Identifier(), options.DataFilePrefix(), - options.LegacyPartitionNameEnabled(), external_paths, global_index_external_path, - options.IndexFileInDataFileDir(), pool_)); + options.LegacyPartitionNameEnabled(), external_paths, global_index_external_paths, + options.IndexFileInDataFileDir(), /*external_path_weights=*/{}, pool_)); return path_factory; } diff --git a/src/paimon/core/mergetree/lookup/remote_lookup_file_manager_test.cpp b/src/paimon/core/mergetree/lookup/remote_lookup_file_manager_test.cpp index 2d146086b..1554da980 100644 --- a/src/paimon/core/mergetree/lookup/remote_lookup_file_manager_test.cpp +++ b/src/paimon/core/mergetree/lookup/remote_lookup_file_manager_test.cpp @@ -126,15 +126,15 @@ class RemoteLookupFileManagerTest : public testing::Test { const std::string& table_path, const CoreOptions& options) const { PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - options.CreateGlobalIndexExternalPath()); + PAIMON_ASSIGN_OR_RAISE(std::vector global_index_external_paths, + options.CreateGlobalIndexExternalPaths()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, FileStorePathFactory::Create( table_path, arrow_schema_, /*partition_keys=*/{}, options.GetPartitionDefaultName(), options.GetFileFormat()->Identifier(), options.DataFilePrefix(), - options.LegacyPartitionNameEnabled(), external_paths, global_index_external_path, - options.IndexFileInDataFileDir(), pool_)); + options.LegacyPartitionNameEnabled(), external_paths, global_index_external_paths, + options.IndexFileInDataFileDir(), /*external_path_weights=*/{}, pool_)); return path_factory; } diff --git a/src/paimon/core/mergetree/lookup_levels_test.cpp b/src/paimon/core/mergetree/lookup_levels_test.cpp index 095656925..0d83d8278 100644 --- a/src/paimon/core/mergetree/lookup_levels_test.cpp +++ b/src/paimon/core/mergetree/lookup_levels_test.cpp @@ -132,15 +132,15 @@ class LookupLevelsTest : public testing::Test { const std::string& table_path, const CoreOptions& options) const { PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - options.CreateGlobalIndexExternalPath()); + PAIMON_ASSIGN_OR_RAISE(std::vector global_index_external_paths, + options.CreateGlobalIndexExternalPaths()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, FileStorePathFactory::Create( table_path, arrow_schema_, /*partition_keys=*/{}, options.GetPartitionDefaultName(), options.GetFileFormat()->Identifier(), options.DataFilePrefix(), - options.LegacyPartitionNameEnabled(), external_paths, global_index_external_path, - options.IndexFileInDataFileDir(), pool_)); + options.LegacyPartitionNameEnabled(), external_paths, global_index_external_paths, + options.IndexFileInDataFileDir(), /*external_path_weights=*/{}, pool_)); return path_factory; } diff --git a/src/paimon/core/migrate/file_meta_utils.cpp b/src/paimon/core/migrate/file_meta_utils.cpp index ee1f65992..679ecf3a0 100644 --- a/src/paimon/core/migrate/file_meta_utils.cpp +++ b/src/paimon/core/migrate/file_meta_utils.cpp @@ -150,13 +150,13 @@ Result> FileMetaUtils::GenerateCommitMessage( // generate bucket path PAIMON_ASSIGN_OR_RAISE( std::shared_ptr file_store_path_factory, - FileStorePathFactory::Create(dst_table_path, schema, table_schema->PartitionKeys(), - core_options.GetPartitionDefaultName(), format->Identifier(), - core_options.DataFilePrefix(), - core_options.LegacyPartitionNameEnabled(), - /*external_paths=*/std::vector(), - /*global_index_external_path=*/std::nullopt, - /*index_file_in_data_file_dir=*/false, memory_pool)); + FileStorePathFactory::Create( + dst_table_path, schema, table_schema->PartitionKeys(), + core_options.GetPartitionDefaultName(), format->Identifier(), + core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), + /*external_paths=*/std::vector(), + /*global_index_external_paths=*/{}, + /*index_file_in_data_file_dir=*/false, /*external_path_weights=*/{}, memory_pool)); PAIMON_ASSIGN_OR_RAISE(std::string bucket_path, file_store_path_factory->BucketPath(partition_row, /*bucket=*/0)); diff --git a/src/paimon/core/operation/append_only_file_store_scan_test.cpp b/src/paimon/core/operation/append_only_file_store_scan_test.cpp index b8dfbfced..62dc3a45d 100644 --- a/src/paimon/core/operation/append_only_file_store_scan_test.cpp +++ b/src/paimon/core/operation/append_only_file_store_scan_test.cpp @@ -237,7 +237,7 @@ TEST_F(AppendBucketPruningTest, ReadsSingleBucketManifestOnce) { ASSERT_OK_AND_ASSIGN( std::shared_ptr paths, FileStorePathFactory::Create(dir->Str(), initial->schema_, {}, "", "avro", "data-", true, - {}, std::nullopt, false, pool_)); + {}, {}, false, /*external_path_weights=*/{}, pool_)); ASSERT_OK(fs->Mkdirs(FileStorePathFactory::ManifestPath(dir->Str()))); ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({})); ASSERT_OK_AND_ASSIGN(std::shared_ptr manifest, @@ -304,13 +304,14 @@ TEST_F(AppendBucketPruningTest, SelectivelyDecodesInferredBucketCandidates) { ASSERT_TRUE(inferred_scan->bucket_selector_); const int32_t bucket = inferred_scan->bucket_selector_->Bucket(kNumBuckets); ASSERT_OK_AND_ASSIGN(std::shared_ptr format, FileFormatFactory::Get("avro", {})); - ASSERT_OK_AND_ASSIGN(std::shared_ptr paths, - FileStorePathFactory::Create( - dir->Str(), initial_scan->schema_, /*partition_keys=*/{}, - /*default_part_value=*/"", "avro", /*data_file_prefix=*/"data-", - /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, - /*global_index_external_path=*/std::nullopt, - /*index_file_in_data_file_dir=*/false, pool_)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr paths, + FileStorePathFactory::Create( + dir->Str(), initial_scan->schema_, /*partition_keys=*/{}, + /*default_part_value=*/"", "avro", /*data_file_prefix=*/"data-", + /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, + /*global_index_external_paths=*/{}, + /*index_file_in_data_file_dir=*/false, /*external_path_weights=*/{}, pool_)); ASSERT_OK(fs->Mkdirs(FileStorePathFactory::ManifestPath(dir->Str()))); SimpleStats stats = BinaryRowGenerator::GenerateStats( {std::string("a"), 0}, {std::string("z"), 100}, {0, 0}, pool_.get()); @@ -426,7 +427,7 @@ TEST_F(AppendBucketPruningTest, SelectivelyDecodesWithoutReadingHistoricalSchema ASSERT_OK_AND_ASSIGN( std::shared_ptr paths, FileStorePathFactory::Create(dir->Str(), initial->schema_, {}, "", "avro", "data-", true, - {}, std::nullopt, false, pool_)); + {}, {}, false, /*external_path_weights=*/{}, pool_)); ASSERT_OK(fs->Mkdirs(FileStorePathFactory::ManifestPath(dir->Str()))); for (bool cache_enabled : {false, true}) { SCOPED_TRACE(cache_enabled); diff --git a/src/paimon/core/operation/commit/commit_scanner_test.cpp b/src/paimon/core/operation/commit/commit_scanner_test.cpp index a3e2e2537..f90a9a036 100644 --- a/src/paimon/core/operation/commit/commit_scanner_test.cpp +++ b/src/paimon/core/operation/commit/commit_scanner_test.cpp @@ -127,8 +127,9 @@ class CommitScannerTest : public testing::Test { /*default_part_value=*/"__DEFAULT_PARTITION__", file_format->Identifier(), /*data_file_prefix=*/"data-", /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, - /*global_index_external_path=*/std::nullopt, - /*index_file_in_data_file_dir=*/false, GetDefaultPool())); + /*global_index_external_paths=*/{}, + /*index_file_in_data_file_dir=*/false, /*external_path_weights=*/{}, + GetDefaultPool())); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr index_manifest_file, IndexManifestFile::Create(dir_->GetFileSystem(), file_format, "zstd", path_factory, diff --git a/src/paimon/core/operation/commit/conflict_detection_test.cpp b/src/paimon/core/operation/commit/conflict_detection_test.cpp index c0122e776..47e57514e 100644 --- a/src/paimon/core/operation/commit/conflict_detection_test.cpp +++ b/src/paimon/core/operation/commit/conflict_detection_test.cpp @@ -141,8 +141,9 @@ class ConflictDetectionTest : public testing::Test { /*root=*/"/tmp/conflict_detection_test", arrow::schema(fields_), partition_keys, /*default_part_value=*/"__DEFAULT_PARTITION__", /*identifier=*/"orc", /*data_file_prefix=*/"data-", /*legacy_partition_name_enabled=*/false, - /*external_paths=*/{}, /*global_index_external_path=*/std::nullopt, - /*index_file_in_data_file_dir=*/false, GetDefaultPool())); + /*external_paths=*/{}, /*global_index_external_paths=*/{}, + /*index_file_in_data_file_dir=*/false, /*external_path_weights=*/{}, + GetDefaultPool())); return path_factory; } diff --git a/src/paimon/core/operation/commit/realtime_deduplicate_dv_materializer_test.cpp b/src/paimon/core/operation/commit/realtime_deduplicate_dv_materializer_test.cpp index 1a9cfabd5..67cc48d27 100644 --- a/src/paimon/core/operation/commit/realtime_deduplicate_dv_materializer_test.cpp +++ b/src/paimon/core/operation/commit/realtime_deduplicate_dv_materializer_test.cpp @@ -53,10 +53,11 @@ class RealtimeDeduplicateDvMaterializerTest : public testing::Test { const auto schema = arrow::schema({arrow::field("id", arrow::int64())}); ASSERT_OK_AND_ASSIGN(table_schema_, TableSchema::Create(0, schema, {}, {}, options)); ASSERT_OK_AND_ASSIGN( - paths_, FileStorePathFactory::Create( - dir_->Str(), schema, {}, "__DEFAULT_PARTITION__", "parquet", "data-", - /*legacy_partition_name_enabled=*/true, {}, std::nullopt, - /*index_file_in_data_file_dir=*/false, GetDefaultPool())); + paths_, FileStorePathFactory::Create(dir_->Str(), schema, {}, "__DEFAULT_PARTITION__", + "parquet", "data-", + /*legacy_partition_name_enabled=*/true, {}, {}, + /*index_file_in_data_file_dir=*/false, + /*external_path_weights=*/{}, GetDefaultPool())); materializer_ = std::make_unique(table_schema_, paths_, /*scanner=*/nullptr, options_, GetDefaultPool()); } diff --git a/src/paimon/core/operation/data_evolution_split_read_test.cpp b/src/paimon/core/operation/data_evolution_split_read_test.cpp index 4cdfce10b..69607ef25 100644 --- a/src/paimon/core/operation/data_evolution_split_read_test.cpp +++ b/src/paimon/core/operation/data_evolution_split_read_test.cpp @@ -456,8 +456,9 @@ TEST_F(DataEvolutionSplitReadTest, TestEmptyBlobReaders) { ASSERT_OK_AND_ASSIGN( std::shared_ptr path_factory, FileStorePathFactory::Create(dir->Str(), schema, {}, "default", "parquet", "data-", - /*legacy_partition_name_enabled=*/true, {}, std::nullopt, - /*index_file_in_data_file_dir=*/false, GetDefaultPool())); + /*legacy_partition_name_enabled=*/true, {}, {}, + /*index_file_in_data_file_dir=*/false, + /*external_path_weights=*/{}, GetDefaultPool())); DataEvolutionSplitRead split_read(path_factory, context, GetDefaultPool(), /*executor=*/nullptr); auto blob = CreateBlobFile("payload", 100, 10, 1, {{"b0"}}); @@ -1036,8 +1037,9 @@ TEST_F(DataEvolutionSplitReadTest, TestSequentialVectorReadReusesMainReader) { ASSERT_OK_AND_ASSIGN( std::shared_ptr path_factory, FileStorePathFactory::Create(dir->Str(), schema, {}, "default", "parquet", "data-", - /*legacy_partition_name_enabled=*/true, {}, std::nullopt, - /*index_file_in_data_file_dir=*/false, GetDefaultPool())); + /*legacy_partition_name_enabled=*/true, {}, {}, + /*index_file_in_data_file_dir=*/false, + /*external_path_weights=*/{}, GetDefaultPool())); ASSERT_OK_AND_ASSIGN(std::string bucket_path, path_factory->BucketPath(BinaryRow::EmptyRow(), 0)); ASSERT_OK(fs->Mkdirs(bucket_path)); diff --git a/src/paimon/core/operation/expire_snapshots_test.cpp b/src/paimon/core/operation/expire_snapshots_test.cpp index 43008f3f8..6ca70e19b 100644 --- a/src/paimon/core/operation/expire_snapshots_test.cpp +++ b/src/paimon/core/operation/expire_snapshots_test.cpp @@ -176,15 +176,15 @@ class ExpireSnapshotsTest : public testing::Test { EXPECT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(raw_options)); EXPECT_OK_AND_ASSIGN(std::vector external_paths, options.CreateExternalPaths()); - EXPECT_OK_AND_ASSIGN(std::optional global_index_external_path, - options.CreateGlobalIndexExternalPath()); + EXPECT_OK_AND_ASSIGN(std::vector global_index_external_paths, + options.CreateGlobalIndexExternalPaths()); EXPECT_OK_AND_ASSIGN( auto path_factory, FileStorePathFactory::Create( root, schema_, partition_keys_, options.GetPartitionDefaultName(), options.GetFileFormat()->Identifier(), options.DataFilePrefix(), - options.LegacyPartitionNameEnabled(), external_paths, global_index_external_path, - options.IndexFileInDataFileDir(), mem_pool_)); + options.LegacyPartitionNameEnabled(), external_paths, global_index_external_paths, + options.IndexFileInDataFileDir(), /*external_path_weights=*/{}, mem_pool_)); return path_factory; } diff --git a/src/paimon/core/operation/file_system_write_restore_test.cpp b/src/paimon/core/operation/file_system_write_restore_test.cpp index 27091f8c9..550fa6b26 100644 --- a/src/paimon/core/operation/file_system_write_restore_test.cpp +++ b/src/paimon/core/operation/file_system_write_restore_test.cpp @@ -57,15 +57,16 @@ Result> CreateRestore( std::shared_ptr arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - options.CreateGlobalIndexExternalPath()); + PAIMON_ASSIGN_OR_RAISE(std::vector global_index_external_paths, + options.CreateGlobalIndexExternalPaths()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, FileStorePathFactory::Create( table_path, arrow_schema, table_schema->PartitionKeys(), options.GetPartitionDefaultName(), options.GetFileFormat()->Identifier(), options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), external_paths, - global_index_external_path, options.IndexFileInDataFileDir(), pool)); + global_index_external_paths, options.IndexFileInDataFileDir(), + /*external_path_weights=*/{}, pool)); auto snapshot_manager = std::make_shared(fs, table_path); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr manifest_list, diff --git a/src/paimon/core/operation/key_value_file_store_scan_test.cpp b/src/paimon/core/operation/key_value_file_store_scan_test.cpp index 963d75274..6992f62d9 100644 --- a/src/paimon/core/operation/key_value_file_store_scan_test.cpp +++ b/src/paimon/core/operation/key_value_file_store_scan_test.cpp @@ -129,8 +129,8 @@ class KeyValueFileStoreScanTest : public testing::Test { auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, core_options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - core_options.CreateGlobalIndexExternalPath()); + PAIMON_ASSIGN_OR_RAISE(std::vector global_index_external_paths, + core_options.CreateGlobalIndexExternalPaths()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, @@ -138,8 +138,8 @@ class KeyValueFileStoreScanTest : public testing::Test { table_path, arrow_schema, table_schema->PartitionKeys(), core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), - pool_)); + external_paths, global_index_external_paths, core_options.IndexFileInDataFileDir(), + /*external_path_weights=*/{}, pool_)); auto manifest_file_format = core_options.GetManifestFormat(); auto snapshot_manager = std::make_shared(fs, table_path); diff --git a/src/paimon/core/operation/manifest_file_merger_test.cpp b/src/paimon/core/operation/manifest_file_merger_test.cpp index af0830aad..8f6b39eaf 100644 --- a/src/paimon/core/operation/manifest_file_merger_test.cpp +++ b/src/paimon/core/operation/manifest_file_merger_test.cpp @@ -135,16 +135,16 @@ class ManifestFileMergerTest : public testing::Test { ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({})); ASSERT_OK_AND_ASSIGN(std::vector external_paths, options.CreateExternalPaths()); - ASSERT_OK_AND_ASSIGN(std::optional global_index_external_path, - options.CreateGlobalIndexExternalPath()); + ASSERT_OK_AND_ASSIGN(std::vector global_index_external_paths, + options.CreateGlobalIndexExternalPaths()); ASSERT_OK_AND_ASSIGN( static std::shared_ptr path_factory, FileStorePathFactory::Create( path_str, schema, /*partition_keys=*/{"f0"}, options.GetPartitionDefaultName(), options.GetFileFormat()->Identifier(), options.DataFilePrefix(), - options.LegacyPartitionNameEnabled(), external_paths, global_index_external_path, - options.IndexFileInDataFileDir(), pool_)); + options.LegacyPartitionNameEnabled(), external_paths, global_index_external_paths, + options.IndexFileInDataFileDir(), /*external_path_weights=*/{}, pool_)); path_factory_ = path_factory; ASSERT_OK_AND_ASSIGN(std::shared_ptr partition_schema, FieldMapping::GetPartitionSchema(schema, {"f0"})); diff --git a/src/paimon/core/operation/merge_file_split_read_test.cpp b/src/paimon/core/operation/merge_file_split_read_test.cpp index 2096c2aa9..1e34704cb 100644 --- a/src/paimon/core/operation/merge_file_split_read_test.cpp +++ b/src/paimon/core/operation/merge_file_split_read_test.cpp @@ -337,8 +337,8 @@ class MergeFileSplitReadTest : public ::testing::Test, auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); EXPECT_OK_AND_ASSIGN(std::vector external_paths, core_options.CreateExternalPaths()); - EXPECT_OK_AND_ASSIGN(std::optional global_index_external_path, - core_options.CreateGlobalIndexExternalPath()); + EXPECT_OK_AND_ASSIGN(std::vector global_index_external_paths, + core_options.CreateGlobalIndexExternalPaths()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, @@ -346,8 +346,8 @@ class MergeFileSplitReadTest : public ::testing::Test, internal_context->GetPath(), arrow_schema, table_schema->PartitionKeys(), core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), - pool_)); + external_paths, global_index_external_paths, core_options.IndexFileInDataFileDir(), + /*external_path_weights=*/{}, pool_)); return MergeFileSplitRead::Create(path_factory, internal_context, pool_, executor_); } diff --git a/src/paimon/core/operation/orphan_files_cleaner.cpp b/src/paimon/core/operation/orphan_files_cleaner.cpp index 2f7052501..800d9f3e8 100644 --- a/src/paimon/core/operation/orphan_files_cleaner.cpp +++ b/src/paimon/core/operation/orphan_files_cleaner.cpp @@ -180,8 +180,10 @@ Result> OrphanFilesCleaner::Create( CoreOptions::FromMap(opts, ctx->GetSpecificFileSystem())); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - options.CreateGlobalIndexExternalPath()); + PAIMON_ASSIGN_OR_RAISE(std::vector external_path_weights, + options.GetExternalPathWeights()); + PAIMON_ASSIGN_OR_RAISE(std::vector global_index_external_paths, + options.CreateGlobalIndexExternalPaths()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, @@ -189,7 +191,8 @@ Result> OrphanFilesCleaner::Create( ctx->GetRootPath(), arrow_schema, schema->PartitionKeys(), options.GetPartitionDefaultName(), options.GetFileFormat()->Identifier(), options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), external_paths, - global_index_external_path, options.IndexFileInDataFileDir(), ctx->GetMemoryPool())); + global_index_external_paths, options.IndexFileInDataFileDir(), external_path_weights, + ctx->GetMemoryPool())); auto snapshot_manager = std::make_shared(options.GetFileSystem(), ctx->GetRootPath()); PAIMON_ASSIGN_OR_RAISE( diff --git a/src/paimon/core/operation/orphan_files_cleaner_test.cpp b/src/paimon/core/operation/orphan_files_cleaner_test.cpp index 1471d794b..58d9d0023 100644 --- a/src/paimon/core/operation/orphan_files_cleaner_test.cpp +++ b/src/paimon/core/operation/orphan_files_cleaner_test.cpp @@ -458,15 +458,16 @@ TEST(OrphanFilesCleanerTest, TestTableWithIndexManifest) { std::shared_ptr memory_pool = GetDefaultPool(); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); ASSERT_OK_AND_ASSIGN(std::vector external_paths, options.CreateExternalPaths()); - ASSERT_OK_AND_ASSIGN(std::optional global_index_external_path, - options.CreateGlobalIndexExternalPath()); + ASSERT_OK_AND_ASSIGN(std::vector global_index_external_paths, + options.CreateGlobalIndexExternalPaths()); ASSERT_OK_AND_ASSIGN( std::shared_ptr path_factory, FileStorePathFactory::Create( table_path, arrow_schema, table_schema->PartitionKeys(), options.GetPartitionDefaultName(), options.GetFileFormat()->Identifier(), options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), external_paths, - global_index_external_path, options.IndexFileInDataFileDir(), memory_pool)); + global_index_external_paths, options.IndexFileInDataFileDir(), + /*external_path_weights=*/{}, memory_pool)); ASSERT_OK_AND_ASSIGN( std::unique_ptr index_manifest_file, IndexManifestFile::Create(file_system, options.GetManifestFormat(), diff --git a/src/paimon/core/operation/raw_file_split_read_test.cpp b/src/paimon/core/operation/raw_file_split_read_test.cpp index f536696f4..89cdb88b7 100644 --- a/src/paimon/core/operation/raw_file_split_read_test.cpp +++ b/src/paimon/core/operation/raw_file_split_read_test.cpp @@ -166,15 +166,16 @@ class RawFileSplitReadTest : public ::testing::Test { DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_path, - options.CreateGlobalIndexExternalPath()); + PAIMON_ASSIGN_OR_RAISE(std::vector global_index_external_paths, + options.CreateGlobalIndexExternalPaths()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, FileStorePathFactory::Create( read_context->GetPath(), schema, table_schema->PartitionKeys(), options.GetPartitionDefaultName(), options.GetFileFormat()->Identifier(), options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), external_paths, - global_index_path, options.IndexFileInDataFileDir(), pool_)); + global_index_external_paths, options.IndexFileInDataFileDir(), + /*external_path_weights=*/{}, pool_)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr executor, CreateDefaultExecutor(/*thread_count=*/2)); std::shared_ptr shared_context(std::move(internal_context)); @@ -215,8 +216,8 @@ class RawFileSplitReadTest : public ::testing::Test { auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); ASSERT_OK_AND_ASSIGN(std::vector external_paths, core_options.CreateExternalPaths()); - ASSERT_OK_AND_ASSIGN(std::optional global_index_external_path, - core_options.CreateGlobalIndexExternalPath()); + ASSERT_OK_AND_ASSIGN(std::vector global_index_external_paths, + core_options.CreateGlobalIndexExternalPaths()); ASSERT_OK_AND_ASSIGN( std::shared_ptr path_factory, @@ -224,8 +225,8 @@ class RawFileSplitReadTest : public ::testing::Test { internal_context->GetPath(), arrow_schema, table_schema->PartitionKeys(), core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), - pool_)); + external_paths, global_index_external_paths, core_options.IndexFileInDataFileDir(), + /*external_path_weights=*/{}, pool_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr executor, CreateDefaultExecutor(/*thread_count=*/2)); auto split_read = std::make_unique( @@ -302,15 +303,15 @@ TEST_F(RawFileSplitReadTest, TestCreateReaderUsesExplicitPredicate) { const CoreOptions& options = context->GetCoreOptions(); const auto schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); ASSERT_OK_AND_ASSIGN(std::vector external_paths, options.CreateExternalPaths()); - ASSERT_OK_AND_ASSIGN(std::optional global_index_path, - options.CreateGlobalIndexExternalPath()); + ASSERT_OK_AND_ASSIGN(std::vector global_index_external_paths, + options.CreateGlobalIndexExternalPaths()); ASSERT_OK_AND_ASSIGN( std::shared_ptr paths, FileStorePathFactory::Create( path, schema, table_schema->PartitionKeys(), options.GetPartitionDefaultName(), options.GetFileFormat()->Identifier(), options.DataFilePrefix(), - options.LegacyPartitionNameEnabled(), external_paths, global_index_path, - options.IndexFileInDataFileDir(), pool_)); + options.LegacyPartitionNameEnabled(), external_paths, global_index_external_paths, + options.IndexFileInDataFileDir(), /*external_path_weights=*/{}, pool_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr executor, CreateDefaultExecutor(2)); RawFileSplitRead read(paths, context, pool_, executor); auto split = std::dynamic_pointer_cast(PrepareDataSplits().front()); @@ -567,16 +568,16 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchScoreProjection) { DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); ASSERT_OK_AND_ASSIGN(std::vector external_paths, core_options.CreateExternalPaths()); - ASSERT_OK_AND_ASSIGN(std::optional global_index_external_path, - core_options.CreateGlobalIndexExternalPath()); + ASSERT_OK_AND_ASSIGN(std::vector global_index_external_paths, + core_options.CreateGlobalIndexExternalPaths()); ASSERT_OK_AND_ASSIGN( std::shared_ptr path_factory, FileStorePathFactory::Create( path, data_schema, table_schema->PartitionKeys(), core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), - pool_)); + external_paths, global_index_external_paths, core_options.IndexFileInDataFileDir(), + /*external_path_weights=*/{}, pool_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr executor, CreateDefaultExecutor(/*thread_count=*/2)); auto split_read = std::make_unique(path_factory, std::move(internal_context), @@ -823,8 +824,8 @@ TEST_F(RawFileSplitReadTest, TestEmptyPlan) { auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); ASSERT_OK_AND_ASSIGN(std::vector external_paths, core_options.CreateExternalPaths()); - ASSERT_OK_AND_ASSIGN(std::optional global_index_external_path, - core_options.CreateGlobalIndexExternalPath()); + ASSERT_OK_AND_ASSIGN(std::vector global_index_external_paths, + core_options.CreateGlobalIndexExternalPaths()); ASSERT_OK_AND_ASSIGN( std::shared_ptr path_factory, @@ -832,8 +833,8 @@ TEST_F(RawFileSplitReadTest, TestEmptyPlan) { internal_context->GetPath(), arrow_schema, table_schema->PartitionKeys(), core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), - pool_)); + external_paths, global_index_external_paths, core_options.IndexFileInDataFileDir(), + /*external_path_weights=*/{}, pool_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr executor, CreateDefaultExecutor(/*thread_count=*/2)); diff --git a/src/paimon/core/options/external_path_strategy.h b/src/paimon/core/options/external_path_strategy.h index 6b6a0c9e7..fa5ebb2cf 100644 --- a/src/paimon/core/options/external_path_strategy.h +++ b/src/paimon/core/options/external_path_strategy.h @@ -27,6 +27,8 @@ enum class ExternalPathStrategy { // Select a specific file system as the external path. Currently supported are S3 and OSS. SPECIFIC_FS = 2, // When writing a new file, a path is chosen from data-file.external-paths in turn. - ROUND_ROBIN = 3 + ROUND_ROBIN = 3, + // Select a path randomly with probability proportional to its configured weight. + WEIGHTED = 4 }; } // namespace paimon diff --git a/src/paimon/core/postpone/postpone_bucket_file_store_write.h b/src/paimon/core/postpone/postpone_bucket_file_store_write.h index 96aa4bbc6..a6105219c 100644 --- a/src/paimon/core/postpone/postpone_bucket_file_store_write.h +++ b/src/paimon/core/postpone/postpone_bucket_file_store_write.h @@ -69,18 +69,20 @@ class PostponeBucketFileStoreWrite : public AbstractFileStoreWrite { // prepare FileStorePathFactory PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, new_options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - new_options.CreateGlobalIndexExternalPath()); + PAIMON_ASSIGN_OR_RAISE(std::vector external_path_weights, + new_options.GetExternalPathWeights()); + PAIMON_ASSIGN_OR_RAISE(std::vector global_index_external_paths, + new_options.CreateGlobalIndexExternalPaths()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr file_store_path_factory, - FileStorePathFactory::Create(root_path, schema, table_schema->PartitionKeys(), - new_options.GetPartitionDefaultName(), - new_options.GetWriteFileFormat(/*level=*/0)->Identifier(), - new_options.DataFilePrefix(), - new_options.LegacyPartitionNameEnabled(), external_paths, - global_index_external_path, - new_options.IndexFileInDataFileDir(), pool)); + FileStorePathFactory::Create( + root_path, schema, table_schema->PartitionKeys(), + new_options.GetPartitionDefaultName(), + new_options.GetWriteFileFormat(/*level=*/0)->Identifier(), + new_options.DataFilePrefix(), new_options.LegacyPartitionNameEnabled(), + external_paths, global_index_external_paths, new_options.IndexFileInDataFileDir(), + external_path_weights, pool)); // Ignoring previous files saves scanning time. // For postpone bucket tables, we only append new files to bucket = -2 directories. diff --git a/src/paimon/core/snapshot_file_scan.cpp b/src/paimon/core/snapshot_file_scan.cpp index 58dd539ce..607d84bc9 100644 --- a/src/paimon/core/snapshot_file_scan.cpp +++ b/src/paimon/core/snapshot_file_scan.cpp @@ -360,16 +360,18 @@ Result> SnapshotFileScan::ListFiles( DataField::ConvertDataFieldsToArrowSchema(latest_schema.value()->Fields()); PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, core_options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - core_options.CreateGlobalIndexExternalPath()); + PAIMON_ASSIGN_OR_RAISE(std::vector external_path_weights, + core_options.GetExternalPathWeights()); + PAIMON_ASSIGN_OR_RAISE(std::vector global_index_external_paths, + core_options.CreateGlobalIndexExternalPaths()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, FileStorePathFactory::Create( table_path, arrow_schema, latest_schema.value()->PartitionKeys(), core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), - pool)); + external_paths, global_index_external_paths, core_options.IndexFileInDataFileDir(), + external_path_weights, pool)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr manifest_list, diff --git a/src/paimon/core/table/file_store.cpp b/src/paimon/core/table/file_store.cpp index 8f1c6d7a8..15939cb1e 100644 --- a/src/paimon/core/table/file_store.cpp +++ b/src/paimon/core/table/file_store.cpp @@ -51,12 +51,15 @@ Result> FileStore::NewPathFactory( const std::vector& partition_keys, const std::string& file_format_identifier, const std::shared_ptr& memory_pool) const { PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options_.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - options_.CreateGlobalIndexExternalPath()); + PAIMON_ASSIGN_OR_RAISE(std::vector external_path_weights, + options_.GetExternalPathWeights()); + PAIMON_ASSIGN_OR_RAISE(std::vector global_index_external_paths, + options_.CreateGlobalIndexExternalPaths()); return FileStorePathFactory::Create( root_path_, arrow_schema, partition_keys, options_.GetPartitionDefaultName(), file_format_identifier, options_.DataFilePrefix(), options_.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, options_.IndexFileInDataFileDir(), memory_pool); + external_paths, global_index_external_paths, options_.IndexFileInDataFileDir(), + external_path_weights, memory_pool); } Result> FileStore::NewSnapshotManager( diff --git a/src/paimon/core/table/source/data_evolution_full_text_read.cpp b/src/paimon/core/table/source/data_evolution_full_text_read.cpp index 57dc8ec7f..465b2e37f 100644 --- a/src/paimon/core/table/source/data_evolution_full_text_read.cpp +++ b/src/paimon/core/table/source/data_evolution_full_text_read.cpp @@ -167,8 +167,8 @@ Result> DataEvolutionFullTextRead::Cr } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr final_executor, GlobalIndexScanImpl::GetOrCreateExecutor(executor, options)); - std::shared_ptr index_path_factory = - path_factory->CreateGlobalIndexFileFactory(); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr index_path_factory, + path_factory->CreateGlobalIndexFileFactory()); auto index_file_manager = std::make_shared( options.GetFileSystem(), index_path_factory, /*checkpoint_path_factory=*/nullptr); return std::unique_ptr(new DataEvolutionFullTextRead( diff --git a/src/paimon/core/table/source/filtered_row_id_reader.cpp b/src/paimon/core/table/source/filtered_row_id_reader.cpp index d3c092d7f..e30e79d33 100644 --- a/src/paimon/core/table/source/filtered_row_id_reader.cpp +++ b/src/paimon/core/table/source/filtered_row_id_reader.cpp @@ -114,16 +114,18 @@ Result> CreateReadWithSchema( auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, core_options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - core_options.CreateGlobalIndexExternalPath()); + PAIMON_ASSIGN_OR_RAISE(std::vector external_path_weights, + core_options.GetExternalPathWeights()); + PAIMON_ASSIGN_OR_RAISE(std::vector global_index_external_paths, + core_options.CreateGlobalIndexExternalPaths()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, FileStorePathFactory::Create( internal_context->GetPath(), arrow_schema, table_schema->PartitionKeys(), core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), - pool)); + external_paths, global_index_external_paths, core_options.IndexFileInDataFileDir(), + external_path_weights, pool)); return std::unique_ptr( std::make_unique(path_factory, internal_context, pool, executor)); } diff --git a/src/paimon/core/table/source/full_text_search_builder.cpp b/src/paimon/core/table/source/full_text_search_builder.cpp index c7d26a091..5f81c282f 100644 --- a/src/paimon/core/table/source/full_text_search_builder.cpp +++ b/src/paimon/core/table/source/full_text_search_builder.cpp @@ -249,15 +249,18 @@ Result FullTextSearchBuilder::Impl: Options::SCALAR_INDEX_SEARCH_MODE, Options::GLOBAL_INDEX_SEARCH_MODE)); } PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - options.CreateGlobalIndexExternalPath()); + PAIMON_ASSIGN_OR_RAISE(std::vector external_path_weights, + options.GetExternalPathWeights()); + PAIMON_ASSIGN_OR_RAISE(std::vector global_index_external_paths, + options.CreateGlobalIndexExternalPaths()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, FileStorePathFactory::Create( root_path, arrow_schema, table_schema->PartitionKeys(), options.GetPartitionDefaultName(), options.GetFileFormat()->Identifier(), options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), external_paths, - global_index_external_path, options.IndexFileInDataFileDir(), memory_pool_)); + global_index_external_paths, options.IndexFileInDataFileDir(), external_path_weights, + memory_pool_)); std::shared_ptr partition_filter; if (!partition_filters_.empty()) { PAIMON_ASSIGN_OR_RAISE(partition_filter, diff --git a/src/paimon/core/table/source/pk_count_reader_test.cpp b/src/paimon/core/table/source/pk_count_reader_test.cpp index 39c0b6b50..f9ce137ef 100644 --- a/src/paimon/core/table/source/pk_count_reader_test.cpp +++ b/src/paimon/core/table/source/pk_count_reader_test.cpp @@ -90,8 +90,8 @@ class PKCountReaderTest : public testing::Test { PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, core_options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - core_options.CreateGlobalIndexExternalPath()); + PAIMON_ASSIGN_OR_RAISE(std::vector global_index_external_paths, + core_options.CreateGlobalIndexExternalPaths()); PAIMON_ASSIGN_OR_RAISE( auto path_factory, @@ -100,8 +100,8 @@ class PKCountReaderTest : public testing::Test { internal_context->GetTableSchema()->PartitionKeys(), core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), - pool_)); + external_paths, global_index_external_paths, core_options.IndexFileInDataFileDir(), + /*external_path_weights=*/{}, pool_)); return std::shared_ptr(std::move(path_factory)); } diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp index 5c28048a8..29efa29f7 100644 --- a/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp +++ b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp @@ -61,25 +61,20 @@ class TestGlobalIndexFileWriter : public GlobalIndexFileWriter { TestGlobalIndexFileWriter(const std::shared_ptr& fs, const std::string& base_path) : fs_(fs), base_path_(base_path) {} - Result NewFileName(const std::string& prefix) const override { - return fmt::format("{}-index-{}", prefix, file_counter_++); + Result NewPath(const std::string& prefix) const override { + return base_path_ + "/" + fmt::format("{}-index-{}", prefix, file_counter_++); } Result> NewOutputStream( - const std::string& file_name) const override { - return fs_->Create(base_path_ + "/" + file_name, true); + const std::string& file_path) const override { + return fs_->Create(file_path, true); } - Result GetFileSize(const std::string& file_name) const override { - PAIMON_ASSIGN_OR_RAISE(FileStatus file_status, - fs_->GetFileStatus(base_path_ + "/" + file_name)); + Result GetFileSize(const std::string& file_path) const override { + PAIMON_ASSIGN_OR_RAISE(FileStatus file_status, fs_->GetFileStatus(file_path)); return file_status.GetLen(); } - std::string ToPath(const std::string& file_name) const override { - return base_path_ + "/" + file_name; - } - private: std::shared_ptr fs_; std::string base_path_; diff --git a/src/paimon/core/table/source/snapshot/snapshot_reader_test.cpp b/src/paimon/core/table/source/snapshot/snapshot_reader_test.cpp index 427d68fe2..c5c1b48b9 100644 --- a/src/paimon/core/table/source/snapshot/snapshot_reader_test.cpp +++ b/src/paimon/core/table/source/snapshot/snapshot_reader_test.cpp @@ -76,8 +76,8 @@ class SnapshotReaderTest : public testing::Test { FileStorePathFactory::Create( dir_->Str(), schema, /*partition_keys=*/{}, /*default_part_value=*/"", "orc", /*data_file_prefix=*/"data-", /*legacy_partition_name_enabled=*/true, - /*external_paths=*/{}, /*global_index_external_path=*/std::nullopt, - /*index_file_in_data_file_dir=*/false, pool_)); + /*external_paths=*/{}, /*global_index_external_paths=*/{}, + /*index_file_in_data_file_dir=*/false, /*external_path_weights=*/{}, pool_)); auto path_factories = std::make_shared(path_factory); return std::make_unique(std::make_shared(), std::unique_ptr(), diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index 647da76c3..0d8a00747 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -202,16 +202,18 @@ Result AggregateFileStats(const std::shared_ptr arrow::ImportSchema(c_arrow_schema.get())); PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, core_options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - core_options.CreateGlobalIndexExternalPath()); + PAIMON_ASSIGN_OR_RAISE(std::vector external_path_weights, + core_options.GetExternalPathWeights()); + PAIMON_ASSIGN_OR_RAISE(std::vector global_index_external_paths, + core_options.CreateGlobalIndexExternalPaths()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, FileStorePathFactory::Create( table_path, arrow_schema, table_schema.PartitionKeys(), core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), - pool)); + external_paths, global_index_external_paths, core_options.IndexFileInDataFileDir(), + external_path_weights, pool)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr manifest_list, ManifestList::Create(fs, core_options.GetManifestFormat(), core_options.GetManifestCompression(), path_factory, diff --git a/src/paimon/core/table/system/metadata_system_tables.cpp b/src/paimon/core/table/system/metadata_system_tables.cpp index 8db1c41cf..da25162d1 100644 --- a/src/paimon/core/table/system/metadata_system_tables.cpp +++ b/src/paimon/core/table/system/metadata_system_tables.cpp @@ -206,16 +206,18 @@ Result> CreatePathFactory( DataField::ConvertDataFieldsToArrowSchema(context.table_schema->Fields()); PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, core_options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - core_options.CreateGlobalIndexExternalPath()); + PAIMON_ASSIGN_OR_RAISE(std::vector external_path_weights, + core_options.GetExternalPathWeights()); + PAIMON_ASSIGN_OR_RAISE(std::vector global_index_external_paths, + core_options.CreateGlobalIndexExternalPaths()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, FileStorePathFactory::Create( context.table_path, arrow_schema, context.table_schema->PartitionKeys(), core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), - pool)); + external_paths, global_index_external_paths, core_options.IndexFileInDataFileDir(), + external_path_weights, pool)); return path_factory; } diff --git a/src/paimon/core/utils/file_store_path_factory.cpp b/src/paimon/core/utils/file_store_path_factory.cpp index cbb767fcd..ab8edc377 100644 --- a/src/paimon/core/utils/file_store_path_factory.cpp +++ b/src/paimon/core/utils/file_store_path_factory.cpp @@ -73,14 +73,16 @@ FileStorePathFactory::FileStorePathFactory( const std::string& data_file_prefix, const std::string& uuid, std::unique_ptr partition_computer, const std::vector& external_paths, - const std::optional& global_index_external_path, bool index_file_in_data_file_dir) + const std::vector& global_index_external_paths, bool index_file_in_data_file_dir, + const std::vector& external_path_weights) : root_(root), format_identifier_(format_identifier), data_file_prefix_(data_file_prefix), uuid_(uuid), partition_computer_(std::move(partition_computer)), external_paths_(external_paths), - global_index_external_path_(global_index_external_path), + external_path_weights_(external_path_weights), + global_index_external_paths_(global_index_external_paths), index_file_in_data_file_dir_(index_file_in_data_file_dir) {} Result> FileStorePathFactory::Create( @@ -88,7 +90,8 @@ Result> FileStorePathFactory::Create( const std::vector& partition_keys, const std::string& default_part_value, const std::string& identifier, const std::string& data_file_prefix, bool legacy_partition_name_enabled, const std::vector& external_paths, - const std::optional& global_index_external_path, bool index_file_in_data_file_dir, + const std::vector& global_index_external_paths, bool index_file_in_data_file_dir, + const std::vector& external_path_weights, const std::shared_ptr& memory_pool) { if (memory_pool == nullptr) { return Status::Invalid("memory pool is null pointer"); @@ -103,7 +106,7 @@ Result> FileStorePathFactory::Create( legacy_partition_name_enabled, memory_pool)); return std::shared_ptr(new FileStorePathFactory( root, identifier, data_file_prefix, uuid, std::move(partition_computer), external_paths, - global_index_external_path, index_file_in_data_file_dir)); + global_index_external_paths, index_file_in_data_file_dir, external_path_weights)); } std::unique_ptr FileStorePathFactory::CreateManifestFileFactory() { @@ -178,15 +181,17 @@ Result> FileStorePathFactory::CreateIndexFileF return CreateGlobalIndexFileFactory(); } -std::unique_ptr FileStorePathFactory::CreateGlobalIndexFileFactory() { +Result> FileStorePathFactory::CreateGlobalIndexFileFactory() { class IndexPathFactoryImpl : public IndexPathFactory { public: - explicit IndexPathFactoryImpl(const std::shared_ptr& factory) - : factory_(factory) { + IndexPathFactoryImpl(const std::shared_ptr& factory, + std::unique_ptr provider) + : factory_(factory), provider_(std::move(provider)) { assert(factory_); } std::string NewPath() const override { - return factory_->NewIndexFile(); + return NewPathFromName(fmt::format("{}{}-{}", INDEX_PREFIX, factory_->UUID(), + factory_->index_file_count_->fetch_add(1))); } std::string ToPath(const std::shared_ptr& file) const override { const auto& external_path = file->ExternalPath(); @@ -195,21 +200,33 @@ std::unique_ptr FileStorePathFactory::CreateGlobalIndexFileFac } return PathUtil::JoinPath(factory_->IndexPath(factory_->RootPath()), file->FileName()); } - std::string ToPath(const std::string& file_name) const override { - const auto& external_path = factory_->GetGlobalIndexExternalPath(); - if (external_path) { - return PathUtil::JoinPath(external_path.value(), file_name); + std::string NewPathFromName(const std::string& file_name) const override { + if (provider_) { + return provider_->GetNextExternalDataPath(file_name); } - return PathUtil::JoinPath(factory_->IndexPath(factory_->RootPath()), file_name); + return factory_->ToIndexFilePath(file_name); } bool IsExternalPath() const override { - return factory_->GetGlobalIndexExternalPath() != std::nullopt; + return provider_ != nullptr; } private: std::shared_ptr factory_; + std::unique_ptr provider_; }; - return std::make_unique(shared_from_this()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr provider, + CreateExternalIndexPathProvider()); + return std::unique_ptr( + std::make_unique(shared_from_this(), std::move(provider))); +} + +Result> +FileStorePathFactory::CreateExternalIndexPathProvider() const { + if (global_index_external_paths_.empty()) { + return std::unique_ptr(); + } + return ExternalPathProvider::Create(global_index_external_paths_, /*relative_bucket_path=*/"", + /*weights=*/{}); } Result> @@ -219,23 +236,22 @@ FileStorePathFactory::CreateGlobalIndexCheckpointPathFactory(const std::string& const std::string& task_id) { class IndexCheckpointPathFactoryImpl : public IndexCheckpointPathFactory { public: - IndexCheckpointPathFactoryImpl(const std::string& directory, - const std::string& file_name_prefix) - : directory_(directory), file_name_prefix_(file_name_prefix) {} + IndexCheckpointPathFactoryImpl(const std::vector& directories, + const std::string& file_name_prefix, + std::unique_ptr provider) + : directories_(directories), + file_name_prefix_(file_name_prefix), + provider_(std::move(provider)) {} std::string NewPath(int64_t checkpoint_id) const override { assert(checkpoint_id >= 0); std::string file_name = fmt::format("{}{}{}", file_name_prefix_, checkpoint_id, kIndexCheckpointFileSuffix); - return ToPath(file_name); - } - - std::string ToPath(const std::string& file_name) const override { - return PathUtil::JoinPath(directory_, file_name); + return provider_->GetNextExternalDataPath(file_name); } - const std::string& GetDirectoryPath() const override { - return directory_; + const std::vector& GetDirectoryPaths() const override { + return directories_; } std::optional GetCheckpointId(const std::string& file_name) const override { @@ -243,17 +259,23 @@ FileStorePathFactory::CreateGlobalIndexCheckpointPathFactory(const std::string& } private: - std::string directory_; + std::vector directories_; std::string file_name_prefix_; + std::unique_ptr provider_; }; PAIMON_RETURN_NOT_OK(PathUtil::CheckSinglePathComponent("checkpoint index type", index_type)); PAIMON_RETURN_NOT_OK(PathUtil::CheckSinglePathComponent("checkpoint field", field_name)); - PAIMON_ASSIGN_OR_RAISE(std::string directory, IndexCheckpointPathFactory::GetTaskDirectory( - root_, task_id, global_index_external_path_)); + PAIMON_ASSIGN_OR_RAISE(std::vector directories, + IndexCheckpointPathFactory::GetTaskDirectories( + root_, task_id, global_index_external_paths_)); std::string file_name_prefix = fmt::format("{}-global-index-{}-{}-{}-", index_type, field_name, range.from, range.to); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr provider, + ExternalPathProvider::Create(directories, /*relative_bucket_path=*/"", + /*weights=*/{})); return std::unique_ptr( - std::make_unique(directory, file_name_prefix)); + std::make_unique(directories, file_name_prefix, + std::move(provider))); } Result> FileStorePathFactory::CreateDataFilePathFactory( @@ -273,7 +295,7 @@ Result> FileStorePathFactory::CreateExtern return std::unique_ptr(); } PAIMON_ASSIGN_OR_RAISE(std::string bucket_path, RelativeBucketPath(partition, bucket)); - return ExternalPathProvider::Create(external_paths_, bucket_path); + return ExternalPathProvider::Create(external_paths_, bucket_path, external_path_weights_); } Result FileStorePathFactory::RelativeBucketPath(const BinaryRow& partition, diff --git a/src/paimon/core/utils/file_store_path_factory.h b/src/paimon/core/utils/file_store_path_factory.h index 030b6b78c..50119ead8 100644 --- a/src/paimon/core/utils/file_store_path_factory.h +++ b/src/paimon/core/utils/file_store_path_factory.h @@ -22,7 +22,6 @@ #include #include #include -#include #include #include #include @@ -59,8 +58,9 @@ class FileStorePathFactory : public std::enable_shared_from_this& partition_keys, const std::string& default_part_value, const std::string& identifier, const std::string& data_file_prefix, bool legacy_partition_name_enabled, const std::vector& external_paths, - const std::optional& global_index_external_path, - bool index_file_in_data_file_dir, const std::shared_ptr& memory_pool); + const std::vector& global_index_external_paths, + bool index_file_in_data_file_dir, const std::vector& external_path_weights, + const std::shared_ptr& memory_pool); static std::string ManifestPath(const std::string& root) { return PathUtil::JoinPath(root, "/manifest"); @@ -77,7 +77,7 @@ class FileStorePathFactory : public std::enable_shared_from_this CreateIndexManifestFileFactory(); Result> CreateIndexFileFactory(const BinaryRow& partition, int32_t bucket); - std::unique_ptr CreateGlobalIndexFileFactory(); + Result> CreateGlobalIndexFileFactory(); Result> CreateGlobalIndexCheckpointPathFactory( const std::string& index_type, const std::string& field_name, const Range& range, const std::string& task_id); @@ -102,8 +102,8 @@ class FileStorePathFactory : public std::enable_shared_from_this& GetGlobalIndexExternalPath() const { - return global_index_external_path_; + const std::vector& GetGlobalIndexExternalPaths() const { + return global_index_external_paths_; } /// Thread-safe for concurrent calls. Cache accesses are protected by internal locks. @@ -123,11 +123,6 @@ class FileStorePathFactory : public std::enable_shared_from_thisfetch_add(1))); - } std::string NewStatsFile() const { return PathUtil::JoinPath( StatisticsPath(root_), @@ -151,11 +146,13 @@ class FileStorePathFactory : public std::enable_shared_from_this partition_computer, const std::vector& external_paths, - const std::optional& global_index_external_path, - bool index_file_in_data_file_dir); + const std::vector& global_index_external_paths, + bool index_file_in_data_file_dir, + const std::vector& external_path_weights); Result> CreateExternalPathProvider( const BinaryRow& partition, int32_t bucket) const; + Result> CreateExternalIndexPathProvider() const; private: std::string root_; @@ -164,7 +161,8 @@ class FileStorePathFactory : public std::enable_shared_from_this partition_computer_; std::vector external_paths_; - std::optional global_index_external_path_; + std::vector external_path_weights_; + std::vector global_index_external_paths_; bool index_file_in_data_file_dir_; mutable std::atomic manifest_file_count_ = 0; diff --git a/src/paimon/core/utils/file_store_path_factory_cache.h b/src/paimon/core/utils/file_store_path_factory_cache.h index 196619899..3193b9d2c 100644 --- a/src/paimon/core/utils/file_store_path_factory_cache.h +++ b/src/paimon/core/utils/file_store_path_factory_cache.h @@ -44,16 +44,18 @@ class FileStorePathFactoryCache { auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema_->Fields()); PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options_.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - options_.CreateGlobalIndexExternalPath()); + PAIMON_ASSIGN_OR_RAISE(std::vector external_path_weights, + options_.GetExternalPathWeights()); + PAIMON_ASSIGN_OR_RAISE(std::vector global_index_external_paths, + options_.CreateGlobalIndexExternalPaths()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, FileStorePathFactory::Create( root_, arrow_schema, table_schema_->PartitionKeys(), options_.GetPartitionDefaultName(), format, options_.DataFilePrefix(), - options_.LegacyPartitionNameEnabled(), external_paths, global_index_external_path, - options_.IndexFileInDataFileDir(), pool_)); + options_.LegacyPartitionNameEnabled(), external_paths, global_index_external_paths, + options_.IndexFileInDataFileDir(), external_path_weights, pool_)); format_to_path_factory_[format] = path_factory; return path_factory; } diff --git a/src/paimon/core/utils/file_store_path_factory_test.cpp b/src/paimon/core/utils/file_store_path_factory_test.cpp index 1d38035e3..06ba065d7 100644 --- a/src/paimon/core/utils/file_store_path_factory_test.cpp +++ b/src/paimon/core/utils/file_store_path_factory_test.cpp @@ -18,6 +18,7 @@ #include "paimon/core/utils/file_store_path_factory.h" +#include #include #include #include @@ -55,7 +56,7 @@ class FileStorePathFactoryTest : public ::testing::Test { std::shared_ptr CreateFactory( const std::string& root, - const std::optional& global_index_external_path = std::nullopt) const { + const std::vector& global_index_external_paths = {}) const { arrow::FieldVector fields = {arrow::field("f0", arrow::boolean()), arrow::field("f1", arrow::int8()), arrow::field("f2", arrow::int8()), @@ -83,8 +84,8 @@ class FileStorePathFactoryTest : public ::testing::Test { FileStorePathFactory::Create( root, schema, {"f0", "f3"}, options.GetPartitionDefaultName(), options.GetFileFormat()->Identifier(), options.DataFilePrefix(), - options.LegacyPartitionNameEnabled(), external_paths, global_index_external_path, - options.IndexFileInDataFileDir(), mem_pool_)); + options.LegacyPartitionNameEnabled(), external_paths, global_index_external_paths, + options.IndexFileInDataFileDir(), /*external_path_weights=*/{}, mem_pool_)); return path_factory; } @@ -160,13 +161,14 @@ TEST_F(FileStorePathFactoryTest, TestCreateFactoryWithNoPartition) { arrow::field("f10", arrow::float64()), arrow::field("f11", arrow::utf8()), arrow::field("f12", arrow::binary()), arrow::field("non-partition-field", arrow::int32())}; auto schema = arrow::schema(fields); - ASSERT_OK_AND_ASSIGN(auto path_factory, - FileStorePathFactory::Create( - dir->Str(), schema, {}, "default", /*identifier=*/"mock_format", - /*data_file_prefix=*/"data-", /*legacy_partition_name_enabled=*/true, - /*external_paths=*/std::vector(), - /*global_index_external_path=*/std::nullopt, - /*index_file_in_data_file_dir=*/false, mem_pool_)); + ASSERT_OK_AND_ASSIGN( + auto path_factory, + FileStorePathFactory::Create( + dir->Str(), schema, {}, "default", /*identifier=*/"mock_format", + /*data_file_prefix=*/"data-", /*legacy_partition_name_enabled=*/true, + /*external_paths=*/std::vector(), + /*global_index_external_paths=*/{}, + /*index_file_in_data_file_dir=*/false, /*external_path_weights=*/{}, mem_pool_)); ASSERT_OK_AND_ASSIGN(auto data_file_path_factory, path_factory->CreateDataFilePathFactory(BinaryRow::EmptyRow(), 123)); ASSERT_EQ(data_file_path_factory->ToPath("my-data-file-name"), @@ -193,8 +195,10 @@ TEST_F(FileStorePathFactoryTest, TestIndexManifestAndStatsPaths) { ASSERT_EQ(stats_file_path, dir->Str() + "/statistics/stats-" + uuid + "-" + std::to_string(i)); } + ASSERT_OK_AND_ASSIGN(std::unique_ptr index_factory, + path_factory->CreateGlobalIndexFileFactory()); for (int32_t i = 0; i < 20; i++) { - std::string index_file_path = path_factory->NewIndexFile(); + std::string index_file_path = index_factory->NewPath(); ASSERT_EQ(index_file_path, dir->Str() + "/index/index-" + uuid + "-" + std::to_string(i)); } std::string index_file_path = path_factory->ToIndexFilePath("my-index-file-name"); @@ -217,14 +221,15 @@ TEST_F(FileStorePathFactoryTest, TestCreateDataFilePathFactoryWithPartition) { arrow::field("f12", arrow::binary()), arrow::field("non-partition-field", arrow::int32())}; auto schema = arrow::schema(fields); - ASSERT_OK_AND_ASSIGN(auto path_factory, FileStorePathFactory::Create( - dir->Str(), schema, {"dt", "hr"}, "default", - /*identifier=*/"mock_format", - /*data_file_prefix=*/"data-", - /*legacy_partition_name_enabled=*/true, - /*external_paths=*/std::vector(), - /*global_index_external_path=*/std::nullopt, - /*index_file_in_data_file_dir=*/false, mem_pool_)); + ASSERT_OK_AND_ASSIGN(auto path_factory, + FileStorePathFactory::Create(dir->Str(), schema, {"dt", "hr"}, "default", + /*identifier=*/"mock_format", + /*data_file_prefix=*/"data-", + /*legacy_partition_name_enabled=*/true, + /*external_paths=*/std::vector(), + /*global_index_external_paths=*/{}, + /*index_file_in_data_file_dir=*/false, + /*external_path_weights=*/{}, mem_pool_)); CheckPartition(20211224, 18, *path_factory, "/dt=20211224/hr=18"); CheckPartition(20211224, std::nullopt, *path_factory, "/dt=20211224/hr=default"); CheckPartition(std::nullopt, 16, *path_factory, "/dt=default/hr=16"); @@ -251,8 +256,9 @@ TEST_F(FileStorePathFactoryTest, TestGetHierarchicalPartitionPath) { /*identifier=*/"mock_format", /*data_file_prefix=*/"data-", /*legacy_partition_name_enabled=*/true, /*external_paths=*/std::vector(), - /*global_index_external_path=*/std::nullopt, - /*index_file_in_data_file_dir=*/false, mem_pool_)); + /*global_index_external_paths=*/{}, + /*index_file_in_data_file_dir=*/false, + /*external_path_weights=*/{}, mem_pool_)); { BinaryRow row = BinaryRowGenerator::GenerateRow({10l, 5}, mem_pool_.get()); @@ -283,14 +289,15 @@ TEST_F(FileStorePathFactoryTest, TestToBinaryRowAndToPartitionString) { arrow::field("f10", arrow::float64()), arrow::field("f11", arrow::utf8()), arrow::field("f12", arrow::binary()), arrow::field("non-partition-field", arrow::int32())}; auto schema = arrow::schema(fields); - ASSERT_OK_AND_ASSIGN(auto path_factory, FileStorePathFactory::Create( - dir->Str(), schema, {"dt", "hr"}, "default", - /*identifier=*/"mock_format", - /*data_file_prefix=*/"data-", - /*legacy_partition_name_enabled=*/true, - /*external_paths=*/std::vector(), - /*global_index_external_path=*/std::nullopt, - /*index_file_in_data_file_dir=*/false, mem_pool_)); + ASSERT_OK_AND_ASSIGN(auto path_factory, + FileStorePathFactory::Create(dir->Str(), schema, {"dt", "hr"}, "default", + /*identifier=*/"mock_format", + /*data_file_prefix=*/"data-", + /*legacy_partition_name_enabled=*/true, + /*external_paths=*/std::vector(), + /*global_index_external_paths=*/{}, + /*index_file_in_data_file_dir=*/false, + /*external_path_weights=*/{}, mem_pool_)); { std::map partition_map; partition_map["dt"] = "20211224"; @@ -391,8 +398,9 @@ TEST_F(FileStorePathFactoryTest, TestConcurrentToBinaryRowAndGetPartitionString) /*identifier=*/"mock_format", /*data_file_prefix=*/"data-", /*legacy_partition_name_enabled=*/true, /*external_paths=*/std::vector(), - /*global_index_external_path=*/std::nullopt, - /*index_file_in_data_file_dir=*/false, mem_pool_)); + /*global_index_external_paths=*/{}, + /*index_file_in_data_file_dir=*/false, + /*external_path_weights=*/{}, mem_pool_)); const std::vector, std::string>> test_cases = { {{{"dt", "20211224"}, {"hr", "23"}}, "dt=20211224/hr=23/"}, @@ -473,15 +481,16 @@ TEST_F(FileStorePathFactoryTest, TestCreateIndexFileFactory) { /*data_file_prefix=*/"data-", /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, - /*global_index_external_path=*/std::nullopt, - /*index_file_in_data_file_dir=*/false, mem_pool_)); + /*global_index_external_paths=*/{}, + /*index_file_in_data_file_dir=*/false, + /*external_path_weights=*/{}, mem_pool_)); auto partition = BinaryRowGenerator::GenerateRow({true, 10}, mem_pool_.get()); ASSERT_OK_AND_ASSIGN( auto index_path_factory, file_store_path_factory->CreateIndexFileFactory(partition, /*bucket=*/2)); ASSERT_EQ(index_path_factory->NewPath(), dir->Str() + "/index/index-" + file_store_path_factory->uuid_ + "-0"); - ASSERT_EQ(index_path_factory->ToPath("global_bitmap.index"), + ASSERT_EQ(index_path_factory->NewPathFromName("global_bitmap.index"), dir->Str() + "/index/global_bitmap.index"); } { @@ -499,15 +508,16 @@ TEST_F(FileStorePathFactoryTest, TestCreateIndexFileFactory) { /*data_file_prefix=*/"data-", /*legacy_partition_name_enabled=*/true, /*external_paths=*/{"/tmp/external-path"}, - /*global_index_external_path=*/std::nullopt, - /*index_file_in_data_file_dir=*/false, mem_pool_)); + /*global_index_external_paths=*/{}, + /*index_file_in_data_file_dir=*/false, + /*external_path_weights=*/{}, mem_pool_)); auto partition = BinaryRowGenerator::GenerateRow({true, 10}, mem_pool_.get()); ASSERT_OK_AND_ASSIGN( auto index_path_factory, file_store_path_factory->CreateIndexFileFactory(partition, /*bucket=*/2)); ASSERT_EQ(index_path_factory->NewPath(), dir->Str() + "/index/index-" + file_store_path_factory->uuid_ + "-0"); - ASSERT_EQ(index_path_factory->ToPath("global_bitmap.index"), + ASSERT_EQ(index_path_factory->NewPathFromName("global_bitmap.index"), dir->Str() + "/index/global_bitmap.index"); } { @@ -525,8 +535,9 @@ TEST_F(FileStorePathFactoryTest, TestCreateIndexFileFactory) { /*data_file_prefix=*/"data-", /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, - /*global_index_external_path=*/std::nullopt, - /*index_file_in_data_file_dir=*/true, mem_pool_)); + /*global_index_external_paths=*/{}, + /*index_file_in_data_file_dir=*/true, + /*external_path_weights=*/{}, mem_pool_)); auto partition = BinaryRowGenerator::GenerateRow({true, 10}, mem_pool_.get()); ASSERT_OK_AND_ASSIGN( auto index_path_factory, @@ -549,8 +560,9 @@ TEST_F(FileStorePathFactoryTest, TestCreateIndexFileFactory) { /*data_file_prefix=*/"data-", /*legacy_partition_name_enabled=*/true, /*external_paths=*/{"/tmp/external-path"}, - /*global_index_external_path=*/std::nullopt, - /*index_file_in_data_file_dir=*/true, mem_pool_)); + /*global_index_external_paths=*/{}, + /*index_file_in_data_file_dir=*/true, + /*external_path_weights=*/{}, mem_pool_)); auto partition = BinaryRowGenerator::GenerateRow({true, 10}, mem_pool_.get()); ASSERT_OK_AND_ASSIGN( auto index_path_factory, @@ -558,6 +570,12 @@ TEST_F(FileStorePathFactoryTest, TestCreateIndexFileFactory) { ASSERT_EQ(index_path_factory->NewPath(), "/tmp/external-path/f0=true/f1=10/bucket-2/index-" + file_store_path_factory->uuid_ + "-0"); + ASSERT_EQ(index_path_factory->NewPathFromName("named.index"), + "/tmp/external-path/f0=true/f1=10/bucket-2/named.index"); + auto local_meta = std::make_shared("bitmap", "named.index", 0, 0, + std::nullopt, std::nullopt); + ASSERT_EQ(index_path_factory->ToPath(local_meta), + dir->Str() + "/f0=true/f1=10/bucket-2/named.index"); } { // test with global index external path @@ -574,13 +592,15 @@ TEST_F(FileStorePathFactoryTest, TestCreateIndexFileFactory) { /*data_file_prefix=*/"data-", /*legacy_partition_name_enabled=*/true, /*external_paths=*/std::vector(), - /*global_index_external_path=*/{"/tmp/external-path"}, - /*index_file_in_data_file_dir=*/false, mem_pool_)); + /*global_index_external_paths=*/{"/tmp/external-path"}, + /*index_file_in_data_file_dir=*/false, + /*external_path_weights=*/{}, mem_pool_)); auto partition = BinaryRowGenerator::GenerateRow({true, 10}, mem_pool_.get()); ASSERT_OK_AND_ASSIGN( auto index_path_factory, file_store_path_factory->CreateIndexFileFactory(partition, /*bucket=*/2)); - ASSERT_EQ(index_path_factory->ToPath("bitmap.index"), "/tmp/external-path/bitmap.index"); + ASSERT_EQ(index_path_factory->NewPathFromName("bitmap.index"), + "/tmp/external-path/bitmap.index"); ASSERT_TRUE(index_path_factory->IsExternalPath()); auto index_file_meta = std::make_shared( @@ -589,6 +609,12 @@ TEST_F(FileStorePathFactoryTest, TestCreateIndexFileFactory) { /*external_path=*/"/tmp/external-path/bitmap.index", /*global_index_meta=*/std::nullopt); ASSERT_EQ(index_path_factory->ToPath(index_file_meta), "/tmp/external-path/bitmap.index"); + auto local_meta = std::make_shared("bitmap", "old.index", 0, 0, std::nullopt, + std::nullopt); + ASSERT_EQ(index_path_factory->ToPath(local_meta), dir->Str() + "/index/old.index"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr local_factory, + CreateFactory(dir->Str())->CreateGlobalIndexFileFactory()); + ASSERT_EQ(local_factory->ToPath(index_file_meta), "/tmp/external-path/bitmap.index"); } } @@ -603,16 +629,15 @@ TEST_F(FileStorePathFactoryTest, TestCreateGlobalIndexCheckpointPathFactory) { file_store_path_factory->CreateGlobalIndexCheckpointPathFactory( "lumina", "vector", Range(10, 20), "task-1")); std::string checkpoint_dir = PathUtil::JoinPath(dir->Str(), "index/checkpoint/task-1"); - ASSERT_OK_AND_ASSIGN(std::string task_directory, IndexCheckpointPathFactory::GetTaskDirectory( - dir->Str(), "task-1", std::nullopt)); - ASSERT_EQ(task_directory, checkpoint_dir); - ASSERT_EQ(checkpoint_path_factory->GetDirectoryPath(), checkpoint_dir); + ASSERT_OK_AND_ASSIGN(std::vector task_directories, + IndexCheckpointPathFactory::GetTaskDirectories(dir->Str(), "task-1", {})); + ASSERT_EQ(task_directories, std::vector{checkpoint_dir}); + ASSERT_EQ(checkpoint_path_factory->GetDirectoryPaths(), + std::vector{checkpoint_dir}); std::string first_path = checkpoint_path_factory->NewPath(0); ASSERT_EQ(first_path, PathUtil::JoinPath(checkpoint_dir, prefix + "0.index.ckpt")); std::string second_path = checkpoint_path_factory->NewPath(1); ASSERT_EQ(second_path, PathUtil::JoinPath(checkpoint_dir, prefix + "1.index.ckpt")); - ASSERT_EQ(checkpoint_path_factory->ToPath("checkpoint"), - PathUtil::JoinPath(checkpoint_dir, "checkpoint")); ASSERT_EQ(checkpoint_path_factory->GetCheckpointId(prefix + "9.index.ckpt"), 9); ASSERT_EQ( checkpoint_path_factory->GetCheckpointId("lumina-global-index-other-10-20-9.index.ckpt"), @@ -641,19 +666,19 @@ TEST_F(FileStorePathFactoryTest, TestCreateGlobalIndexCheckpointPathFactoryWithE ASSERT_TRUE(dir); for (const std::string& external_path : {"file://" + dir->Str() + "/external-index/", std::string("s3://bucket/external-index")}) { - auto factory = CreateFactory(dir->Str(), external_path); + auto factory = CreateFactory(dir->Str(), {external_path}); ASSERT_OK_AND_ASSIGN(std::unique_ptr checkpoint_factory, factory->CreateGlobalIndexCheckpointPathFactory( "lumina", "vector", Range(10, 20), "task-1")); std::string checkpoint_dir = PathUtil::JoinPath(external_path, "checkpoint/task-1"); ASSERT_OK_AND_ASSIGN( - std::string task_directory, - IndexCheckpointPathFactory::GetTaskDirectory(dir->Str(), "task-1", external_path)); - ASSERT_EQ(task_directory, checkpoint_dir); - ASSERT_EQ(checkpoint_factory->GetDirectoryPath(), checkpoint_dir); + std::vector task_directories, + IndexCheckpointPathFactory::GetTaskDirectories(dir->Str(), "task-1", {external_path})); + ASSERT_EQ(task_directories, std::vector{checkpoint_dir}); + ASSERT_EQ(checkpoint_factory->GetDirectoryPaths(), + std::vector{checkpoint_dir}); std::string file_name = "lumina-global-index-vector-10-20-9.index.ckpt"; ASSERT_EQ(checkpoint_factory->NewPath(9), PathUtil::JoinPath(checkpoint_dir, file_name)); - ASSERT_EQ(checkpoint_factory->ToPath(file_name), checkpoint_factory->NewPath(9)); } ASSERT_OK_AND_ASSIGN(bool exists, dir->GetFileSystem()->Exists( PathUtil::JoinPath(dir->Str(), "external-index"))); @@ -671,9 +696,10 @@ TEST_F(FileStorePathFactoryTest, TestCheckpointFileIdPath) { "lumina", "vector", Range(10, 20), "task-1")); std::string path = checkpoint_factory->NewPath(file_id); ASSERT_EQ(checkpoint_factory->GetCheckpointId(PathUtil::GetName(path)), file_id); - ASSERT_OK_AND_ASSIGN(bool exists, - dir->GetFileSystem()->Exists(checkpoint_factory->GetDirectoryPath())); - ASSERT_FALSE(exists); + for (const auto& directory : checkpoint_factory->GetDirectoryPaths()) { + ASSERT_OK_AND_ASSIGN(bool exists, dir->GetFileSystem()->Exists(directory)); + ASSERT_FALSE(exists); + } } } @@ -688,9 +714,13 @@ TEST_F(FileStorePathFactoryTest, TestCheckpointTaskIsolation) { ASSERT_OK_AND_ASSIGN(std::unique_ptr other_factory, factory->CreateGlobalIndexCheckpointPathFactory("lumina", "vector", Range(10, 20), "task-2")); - ASSERT_NE(own_factory->GetDirectoryPath(), other_factory->GetDirectoryPath()); - ASSERT_OK(fs->Mkdirs(own_factory->GetDirectoryPath())); - ASSERT_OK(fs->Mkdirs(other_factory->GetDirectoryPath())); + ASSERT_NE(own_factory->GetDirectoryPaths(), other_factory->GetDirectoryPaths()); + for (const auto& directory : own_factory->GetDirectoryPaths()) { + ASSERT_OK(fs->Mkdirs(directory)); + } + for (const auto& directory : other_factory->GetDirectoryPaths()) { + ASSERT_OK(fs->Mkdirs(directory)); + } std::string own_path = own_factory->NewPath(0); ASSERT_OK(fs->WriteFile(own_path, "checkpoint", /*overwrite=*/false)); std::string same_name_other_path = other_factory->NewPath(0); @@ -701,8 +731,9 @@ TEST_F(FileStorePathFactoryTest, TestCheckpointTaskIsolation) { ASSERT_EQ(own_factory->GetCheckpointId(PathUtil::GetName(own_path)), 0); ASSERT_EQ(own_factory->GetCheckpointId(PathUtil::GetName(other_path)), 100); - GlobalIndexFileManager manager(fs, factory->CreateGlobalIndexFileFactory(), - std::move(own_factory)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr index_path_factory, + factory->CreateGlobalIndexFileFactory()); + GlobalIndexFileManager manager(fs, std::move(index_path_factory), std::move(own_factory)); ASSERT_OK_AND_ASSIGN(bool exists, manager.CheckpointExists()); ASSERT_TRUE(exists); ASSERT_OK_AND_ASSIGN(std::unique_ptr input, manager.OpenCheckpointInputStream()); @@ -733,7 +764,7 @@ TEST_F(FileStorePathFactoryTest, TestCheckpointRejectsInvalidPathComponents) { auto factory = CreateFactory(dir->Str()); for (const std::string& invalid : std::vector{"", ".", "..", " ", "a/b", "a\\b", "a\nb", std::string("a\0b", 3)}) { - ASSERT_NOK(IndexCheckpointPathFactory::GetTaskDirectory(dir->Str(), invalid, std::nullopt)); + ASSERT_NOK(IndexCheckpointPathFactory::GetTaskDirectories(dir->Str(), invalid, {})); ASSERT_NOK(factory->CreateGlobalIndexCheckpointPathFactory(invalid, "vector", Range(10, 20), "task")); ASSERT_NOK(factory->CreateGlobalIndexCheckpointPathFactory("lumina", invalid, Range(10, 20), diff --git a/src/paimon/core/utils/index_file_path_factories_test.cpp b/src/paimon/core/utils/index_file_path_factories_test.cpp index e0419e6d7..22f70e793 100644 --- a/src/paimon/core/utils/index_file_path_factories_test.cpp +++ b/src/paimon/core/utils/index_file_path_factories_test.cpp @@ -43,8 +43,9 @@ TEST(IndexFilePathFactoriesTest, TestSimple) { /*data_file_prefix=*/"data-", /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, - /*global_index_external_path=*/std::nullopt, - /*index_file_in_data_file_dir=*/false, pool)); + /*global_index_external_paths=*/{}, + /*index_file_in_data_file_dir=*/false, + /*external_path_weights=*/{}, pool)); auto uuid = file_store_path_factory->uuid_; auto factories = std::make_shared(file_store_path_factory); ASSERT_EQ(factories->cache_.Size(), 0); @@ -91,8 +92,9 @@ TEST(IndexFilePathFactoriesTest, TestWithExternalPath) { /*data_file_prefix=*/"data-", /*legacy_partition_name_enabled=*/true, /*external_paths=*/{"/tmp/external-path"}, - /*global_index_external_path=*/std::nullopt, - /*index_file_in_data_file_dir=*/true, pool)); + /*global_index_external_paths=*/{}, + /*index_file_in_data_file_dir=*/true, + /*external_path_weights=*/{}, pool)); auto uuid = file_store_path_factory->uuid_; auto factories = std::make_shared(file_store_path_factory); ASSERT_EQ(factories->cache_.Size(), 0); diff --git a/src/paimon/global_index/full_text/full_text_global_index_test.cpp b/src/paimon/global_index/full_text/full_text_global_index_test.cpp index d361e62bd..c373ccfa3 100644 --- a/src/paimon/global_index/full_text/full_text_global_index_test.cpp +++ b/src/paimon/global_index/full_text/full_text_global_index_test.cpp @@ -128,24 +128,20 @@ class FlushFailingFileWriter : public GlobalIndexFileWriter { int32_t fail_from) : delegate_(delegate), fail_from_(fail_from) {} - Result NewFileName(const std::string& prefix) const override { - return delegate_->NewFileName(prefix); + Result NewPath(const std::string& prefix) const override { + return delegate_->NewPath(prefix); } Result> NewOutputStream( - const std::string& file_name) const override { + const std::string& file_path) const override { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr stream, - delegate_->NewOutputStream(file_name)); + delegate_->NewOutputStream(file_path)); return std::make_unique(std::move(stream), fail_from_, close_count_); } - Result GetFileSize(const std::string& file_name) const override { - return delegate_->GetFileSize(file_name); - } - - std::string ToPath(const std::string& file_name) const override { - return delegate_->ToPath(file_name); + Result GetFileSize(const std::string& file_path) const override { + return delegate_->GetFileSize(file_path); } int32_t CloseCount() const { diff --git a/src/paimon/global_index/full_text/full_text_global_index_writer.cpp b/src/paimon/global_index/full_text/full_text_global_index_writer.cpp index 375fc6f8d..0df69ac95 100644 --- a/src/paimon/global_index/full_text/full_text_global_index_writer.cpp +++ b/src/paimon/global_index/full_text/full_text_global_index_writer.cpp @@ -194,15 +194,15 @@ Result> FullTextGlobalIndexWriter::Finish() { Result FullTextGlobalIndexWriter::WriteIndex( PaimonFtindexWriterHandle* writer) const { - PAIMON_ASSIGN_OR_RAISE(std::string file_name, file_writer_->NewFileName(kFileNamePrefix)); + PAIMON_ASSIGN_OR_RAISE(std::string file_path, file_writer_->NewPath(kFileNamePrefix)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr out, - file_writer_->NewOutputStream(file_name)); + file_writer_->NewOutputStream(file_path)); // Closes the stream on every error path, keeping the original error. ScopeGuard close_guard([&]() { Status status = out->Close(); if (!status.ok()) { PAIMON_LOG_WARN(logger_, "failed to close full-text index file %s: %s", - file_name.c_str(), status.ToString().c_str()); + file_path.c_str(), status.ToString().c_str()); } }); OutputContext ctx; @@ -210,17 +210,17 @@ Result FullTextGlobalIndexWriter::WriteIndex( PaimonFtindexOutputFile output{static_cast(&ctx), &WriteToOutputStream, &FlushOutputStream}; if (paimon_ftindex_writer_write_index(writer, output) != 0) { - return LastFtindexError(fmt::format("write full-text index {}", file_name), ctx.status); + return LastFtindexError(fmt::format("write full-text index {}", file_path), ctx.status); } PAIMON_RETURN_NOT_OK(out->Flush()); close_guard.Release(); PAIMON_RETURN_NOT_OK(out->Close()); - PAIMON_ASSIGN_OR_RAISE(int64_t file_size, file_writer_->GetFileSize(file_name)); + PAIMON_ASSIGN_OR_RAISE(int64_t file_size, file_writer_->GetFileSize(file_path)); std::string options_json; PAIMON_RETURN_NOT_OK(RapidJsonUtil::ToJsonString(options_, &options_json)); auto metadata = std::make_shared(options_json, pool_.get()); - return GlobalIndexIOMeta(file_writer_->ToPath(file_name), file_size, row_count_, metadata); + return GlobalIndexIOMeta(file_path, file_size, row_count_, metadata); } } // namespace paimon::full_text diff --git a/src/paimon/global_index/lucene/lucene_global_index_test.cpp b/src/paimon/global_index/lucene/lucene_global_index_test.cpp index 273bd19e6..415d02836 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_test.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index_test.cpp @@ -56,7 +56,7 @@ class LuceneGlobalIndexTest : public ::testing::Test, assert(false); return ""; } - std::string ToPath(const std::string& file_name) const override { + std::string NewPathFromName(const std::string& file_name) const override { return PathUtil::JoinPath(index_path_, file_name); } bool IsExternalPath() const override { diff --git a/src/paimon/global_index/lucene/lucene_global_index_writer.cpp b/src/paimon/global_index/lucene/lucene_global_index_writer.cpp index 8560746a0..23e495979 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_writer.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index_writer.cpp @@ -180,10 +180,10 @@ Result LuceneGlobalIndexWriter::FlushIndexToFinal() { // list tmp dir auto tmp_file_names = write_context_.lucene_dir->listAll(); - PAIMON_ASSIGN_OR_RAISE(std::string index_file_name, file_writer_->NewFileName(kIdentifier)); + PAIMON_ASSIGN_OR_RAISE(std::string index_file_path, file_writer_->NewPath(kIdentifier)); // prepare output from file_writer PAIMON_ASSIGN_OR_RAISE(std::shared_ptr out, - file_writer_->NewOutputStream(index_file_name)); + file_writer_->NewOutputStream(index_file_path)); DataOutputStream data_output_stream(out); PAIMON_RETURN_NOT_OK(data_output_stream.WriteValue(kVersion)); PAIMON_RETURN_NOT_OK( @@ -220,7 +220,7 @@ Result LuceneGlobalIndexWriter::FlushIndexToFinal() { PAIMON_RETURN_NOT_OK(out->Flush()); PAIMON_RETURN_NOT_OK(out->Close()); write_context_.lucene_dir->close(); - return index_file_name; + return index_file_path; } catch (const std::exception& e) { return Status::Invalid( fmt::format("finish for lucene global index writer failed, with {} error.", e.what())); @@ -231,13 +231,13 @@ Result LuceneGlobalIndexWriter::FlushIndexToFinal() { } Result> LuceneGlobalIndexWriter::Finish() { - PAIMON_ASSIGN_OR_RAISE(std::string index_file_name, FlushIndexToFinal()); + PAIMON_ASSIGN_OR_RAISE(std::string index_file_path, FlushIndexToFinal()); // prepare global index meta - PAIMON_ASSIGN_OR_RAISE(int64_t file_size, file_writer_->GetFileSize(index_file_name)); + PAIMON_ASSIGN_OR_RAISE(int64_t file_size, file_writer_->GetFileSize(index_file_path)); std::string options_json; PAIMON_RETURN_NOT_OK(RapidJsonUtil::ToJsonString(options_, &options_json)); auto meta_bytes = std::make_shared(options_json, pool_.get()); - GlobalIndexIOMeta meta(file_writer_->ToPath(index_file_name), file_size, + GlobalIndexIOMeta meta(index_file_path, file_size, /*row_count=*/row_id_, /*metadata=*/meta_bytes); return std::vector({meta}); diff --git a/src/paimon/indexer/lumina/lumina_global_index.cpp b/src/paimon/indexer/lumina/lumina_global_index.cpp index beac36d68..f544e871e 100644 --- a/src/paimon/indexer/lumina/lumina_global_index.cpp +++ b/src/paimon/indexer/lumina/lumina_global_index.cpp @@ -225,19 +225,19 @@ Result> LuminaIndexWriter::Finish() { std::unique_ptr build_context = std::move(build_result).value(); // dump index - PAIMON_ASSIGN_OR_RAISE(std::string index_file_name, - file_manager_->NewFileName(LuminaDefines::kIdentifier)); + PAIMON_ASSIGN_OR_RAISE(std::string index_file_path, + file_manager_->NewPath(LuminaDefines::kIdentifier)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr out, - file_manager_->NewOutputStream(index_file_name)); + file_manager_->NewOutputStream(index_file_path)); auto file_writer = std::make_unique(out); PAIMON_RETURN_NOT_OK_FROM_LUMINA( build_context->builder.Dump(std::move(file_writer), io_options_)); // prepare GlobalIndexIOMeta - PAIMON_ASSIGN_OR_RAISE(int64_t file_size, file_manager_->GetFileSize(index_file_name)); + PAIMON_ASSIGN_OR_RAISE(int64_t file_size, file_manager_->GetFileSize(index_file_path)); std::string options_json; PAIMON_RETURN_NOT_OK(RapidJsonUtil::ToJsonString(lumina_options_, &options_json)); auto meta_bytes = std::make_shared(options_json, pool_->GetPaimonPool().get()); - GlobalIndexIOMeta meta(file_manager_->ToPath(index_file_name), file_size, + GlobalIndexIOMeta meta(index_file_path, file_size, /*row_count=*/count_, /*metadata=*/meta_bytes); if (checkpoint_file_manager_) { diff --git a/src/paimon/indexer/lumina/lumina_global_index_test.cpp b/src/paimon/indexer/lumina/lumina_global_index_test.cpp index 99a7c8e10..9e2020d6f 100644 --- a/src/paimon/indexer/lumina/lumina_global_index_test.cpp +++ b/src/paimon/indexer/lumina/lumina_global_index_test.cpp @@ -60,7 +60,7 @@ class LuminaGlobalIndexTest : public ::testing::Test { assert(false); return ""; } - std::string ToPath(const std::string& file_name) const override { + std::string NewPathFromName(const std::string& file_name) const override { return PathUtil::JoinPath(index_path_, file_name); } bool IsExternalPath() const override { @@ -141,8 +141,8 @@ class LuminaGlobalIndexTest : public ::testing::Test { table_path, arrow::schema({}), /*partition_keys=*/{}, /*default_part_value=*/"", /*identifier=*/"mock", /*data_file_prefix=*/"data-", /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, - /*global_index_external_path=*/std::nullopt, - /*index_file_in_data_file_dir=*/false, pool_); + /*global_index_external_paths=*/{}, + /*index_file_in_data_file_dir=*/false, /*external_path_weights=*/{}, pool_); } Result> CreateCheckpointPathFactory( @@ -159,8 +159,10 @@ class LuminaGlobalIndexTest : public ::testing::Test { PAIMON_ASSIGN_OR_RAISE( std::unique_ptr checkpoint_path_factory, path_factory->CreateGlobalIndexCheckpointPathFactory("lumina", "f0", range, "task-1")); - return std::make_shared( - fs_, path_factory->CreateGlobalIndexFileFactory(), std::move(checkpoint_path_factory)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr index_path_factory, + path_factory->CreateGlobalIndexFileFactory()); + return std::make_shared(fs_, std::move(index_path_factory), + std::move(checkpoint_path_factory)); } Result WriteGlobalIndex(const std::string& index_root, @@ -174,8 +176,10 @@ class LuminaGlobalIndexTest : public ::testing::Test { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr checkpoint_path_factory, path_factory->CreateGlobalIndexCheckpointPathFactory( "lumina", "f0", expected_range, "task-1")); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr index_path_factory, + path_factory->CreateGlobalIndexFileFactory()); auto file_writer = std::make_shared( - fs_, path_factory->CreateGlobalIndexFileFactory(), std::move(checkpoint_path_factory)); + fs_, std::move(index_path_factory), std::move(checkpoint_path_factory)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr global_writer, global_index->CreateWriter("f0", CreateArrowSchema(data_type).get(), @@ -380,8 +384,10 @@ TEST_F(LuminaGlobalIndexTest, TestCheckpointCapabilityOnlyRequiredWhenEnabled) { ASSERT_TRUE(dir); ASSERT_OK_AND_ASSIGN(std::shared_ptr path_factory, CreateFileStorePathFactory(dir->Str())); + ASSERT_OK_AND_ASSIGN(std::unique_ptr index_path_factory, + path_factory->CreateGlobalIndexFileFactory()); auto plain_manager = - std::make_shared(fs_, path_factory->CreateGlobalIndexFileFactory(), + std::make_shared(fs_, std::move(index_path_factory), /*checkpoint_path_factory=*/nullptr); ASSERT_OK_AND_ASSIGN(std::shared_ptr checkpoint_manager, CreateCountingCheckpointFileManager(path_factory, Range(0, 3))); @@ -456,7 +462,9 @@ TEST_F(LuminaGlobalIndexTest, TestCheckpointFileManagement) { CreateCheckpointPathFactory(test_root, "lumina", "f0", Range(10, 20))); ASSERT_OK_AND_ASSIGN(std::shared_ptr path_factory, CreateFileStorePathFactory(test_root)); - GlobalIndexFileManager file_manager(fs_, path_factory->CreateGlobalIndexFileFactory(), + ASSERT_OK_AND_ASSIGN(std::unique_ptr index_path_factory, + path_factory->CreateGlobalIndexFileFactory()); + GlobalIndexFileManager file_manager(fs_, std::move(index_path_factory), std::move(checkpoint_path_factory)); ASSERT_OK_AND_ASSIGN(bool checkpoint_exists, file_manager.CheckpointExists()); @@ -493,7 +501,9 @@ TEST_F(LuminaGlobalIndexTest, TestCheckpointFileManagement) { ASSERT_OK_AND_ASSIGN(checkpoint_path_factory, CreateCheckpointPathFactory(test_root, "lumina", "f0", Range(10, 20))); - GlobalIndexFileManager reopened_manager(fs_, path_factory->CreateGlobalIndexFileFactory(), + ASSERT_OK_AND_ASSIGN(std::unique_ptr restarted_index_path_factory, + path_factory->CreateGlobalIndexFileFactory()); + GlobalIndexFileManager reopened_manager(fs_, std::move(restarted_index_path_factory), std::move(checkpoint_path_factory)); ASSERT_OK_AND_ASSIGN(std::unique_ptr latest_input, @@ -650,8 +660,10 @@ TEST_F(LuminaGlobalIndexTest, TestBuildFailureDiscardsCheckpointAndRetries) { ASSERT_OK_AND_ASSIGN(std::unique_ptr checkpoint_path_factory, path_factory->CreateGlobalIndexCheckpointPathFactory( "lumina", "f0", Range(0, 3), "task-1")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr index_path_factory, + path_factory->CreateGlobalIndexFileFactory()); auto file_manager = std::make_shared( - fs_, path_factory->CreateGlobalIndexFileFactory(), std::move(checkpoint_path_factory)); + fs_, std::move(index_path_factory), std::move(checkpoint_path_factory)); ::lumina::api::BuilderOptions builder_options; builder_options.Set(::lumina::core::kIndexType, ::lumina::core::kIndexTypeBruteforce) diff --git a/src/paimon/testing/mock/mock_index_path_factory.h b/src/paimon/testing/mock/mock_index_path_factory.h index 101690019..f32322f48 100644 --- a/src/paimon/testing/mock/mock_index_path_factory.h +++ b/src/paimon/testing/mock/mock_index_path_factory.h @@ -35,7 +35,7 @@ class MockIndexPathFactory : public IndexPathFactory { return PathUtil::JoinPath(index_path_, std::string(IndexPathFactory::INDEX_PREFIX) + std::to_string(index_file_count_->fetch_add(1))); } - std::string ToPath(const std::string& file_name) const override { + std::string NewPathFromName(const std::string& file_name) const override { return PathUtil::JoinPath(index_path_, file_name); } std::string ToPath(const std::shared_ptr& file) const override { diff --git a/src/paimon/testing/utils/data_generator.cpp b/src/paimon/testing/utils/data_generator.cpp index ec1aa5c86..f5536f566 100644 --- a/src/paimon/testing/utils/data_generator.cpp +++ b/src/paimon/testing/utils/data_generator.cpp @@ -244,8 +244,8 @@ Result>> DataGenerator::SplitArrayByPar /*default_part_value=*/"__DEFAULT_PARTITION__", /*identifier=*/"orc", /*data_file_prefix=*/"data-", /*legacy_partition_name_enabled=*/true, /*external_paths=*/std::vector(), - /*global_index_external_path=*/std::nullopt, - /*index_file_in_data_file_dir=*/false, memory_pool_)); + /*global_index_external_paths=*/{}, + /*index_file_in_data_file_dir=*/false, /*external_path_weights=*/{}, memory_pool_)); for (const auto& binary_row : binary_rows) { PAIMON_ASSIGN_OR_RAISE(BinaryRow partition_row, diff --git a/src/paimon/testing/utils/deletion_vector_test_helper.h b/src/paimon/testing/utils/deletion_vector_test_helper.h index 35adf7f55..3939e6982 100644 --- a/src/paimon/testing/utils/deletion_vector_test_helper.h +++ b/src/paimon/testing/utils/deletion_vector_test_helper.h @@ -97,16 +97,16 @@ class DeletionVectorTestHelper { auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, core_options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - core_options.CreateGlobalIndexExternalPath()); + PAIMON_ASSIGN_OR_RAISE(std::vector global_index_external_paths, + core_options.CreateGlobalIndexExternalPaths()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, - FileStorePathFactory::Create(table_path, arrow_schema, table_schema->PartitionKeys(), - core_options.GetPartitionDefaultName(), - file_format_identifier, core_options.DataFilePrefix(), - core_options.LegacyPartitionNameEnabled(), external_paths, - global_index_external_path, - core_options.IndexFileInDataFileDir(), pool)); + FileStorePathFactory::Create( + table_path, arrow_schema, table_schema->PartitionKeys(), + core_options.GetPartitionDefaultName(), file_format_identifier, + core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), + external_paths, global_index_external_paths, core_options.IndexFileInDataFileDir(), + /*external_path_weights=*/{}, pool)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr index_path_factory, path_factory->CreateIndexFileFactory(base_msg_impl->Partition(), base_msg_impl->Bucket()));