Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions docs/source/user_guide/write.rst
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,21 @@ Parquet, and Avro, thereby improving write throughput.
the files it ignored. See :doc:`compaction` for the ``full-compaction`` and
``lookup`` producers.

External Data Paths
-------------------

Set ``data-file.external-paths`` to comma-separated roots and
``data-file.external-paths.strategy`` to ``round-robin`` to distribute new files
across them. For weighted random selection, set the strategy to ``weight-robin`` and
``data-file.external-paths.weights`` to positive integers in the same order as
the paths. For example, ``10,5,15`` selects three roots with probabilities
``1/3``, ``1/6``, and ``1/2``. These are probabilities per file, not byte quotas.

Without weights, ``weight-robin`` falls back to round robin. A single external path
always uses that path. With multiple paths and configured weights, the number
of weights must equal the number of paths. The default strategy is ``none``,
which writes to the table directory.

Bucketing Modes
---------------

Expand Down
13 changes: 11 additions & 2 deletions include/paimon/defs.h
Original file line number Diff line number Diff line change
Expand Up @@ -495,8 +495,16 @@ struct PAIMON_EXPORT Options {
/// written, multiple elements separated by commas.
static const char DATA_FILE_EXTERNAL_PATHS[];
/// "data-file.external-paths.strategy" - The strategy of selecting an external path when
/// writing data. Values can be: "none", "specific-fs", "round-robin". Default value is "none".
/// writing data. Values: "none", "specific-fs", "round-robin", "weight-robin". Default is
/// "none".
static const char DATA_FILE_EXTERNAL_PATHS_STRATEGY[];
Comment thread
lxy-9602 marked this conversation as resolved.
Resolved
/// "data-file.external-paths.weights" - The weights for external paths when
/// data-file.external-paths.strategy is set to weight-robin.
/// Format: 'weight1,weight2,...' with weights corresponding to paths in
/// data-file.external-paths by order.
/// Example: '10,5,15' means first path has weight 10, second 5, third 15.
/// Weights must be positive integers.
static const char DATA_FILE_EXTERNAL_PATHS_WEIGHTS[];
/// "data-file.prefix" - Specify the file name prefix of data files. Default value is "data-".
static const char DATA_FILE_PREFIX[];
/// "index-file-in-data-file-dir" - Whether index file in data file directory. Default value is
Expand Down Expand Up @@ -611,7 +619,8 @@ struct PAIMON_EXPORT Options {
/// "global-index.thread-num" - The maximum number of concurrent scanner for global index. No
/// default value. By default is the number of processors available to the machine.
static const char GLOBAL_INDEX_THREAD_NUM[];
/// "global-index.external-path" - Global index root directory, if not set, the global index
/// "global-index.external-path" - Comma-separated global index root directories. New files
/// and checkpoints use independent round robin. If not set, the global index
/// files will be stored under the index directory.
static const char GLOBAL_INDEX_EXTERNAL_PATH[];
/// "global-index.search-mode" - Fallback search mode for global index queries, used when the
Expand Down
15 changes: 6 additions & 9 deletions include/paimon/global_index/io/global_index_file_writer.h
Original file line number Diff line number Diff line change
Expand Up @@ -29,19 +29,16 @@ class PAIMON_EXPORT GlobalIndexFileWriter {
public:
virtual ~GlobalIndexFileWriter() = default;

/// Generates a unique file name for a new index file using the given prefix.
/// Generates a unique full path for a new index file using the given prefix.
/// @note This function may be called multiple times if the index consists of multiple files.
virtual Result<std::string> NewFileName(const std::string& prefix) const = 0;
virtual Result<std::string> NewPath(const std::string& prefix) const = 0;

/// Opens a new output stream for writing index data to the specified file.
/// Opens a new output stream at the supplied full path, without selecting a directory.
virtual Result<std::unique_ptr<OutputStream>> NewOutputStream(
const std::string& file_name) const = 0;
const std::string& file_path) const = 0;

/// Get the file size of input file name.
virtual Result<int64_t> GetFileSize(const std::string& file_name) const = 0;

/// Get the index file path of input file name.
virtual std::string ToPath(const std::string& file_name) const = 0;
/// Gets the size of the file at the supplied full path.
virtual Result<int64_t> GetFileSize(const std::string& file_path) const = 0;
};

} // namespace paimon
1 change: 1 addition & 0 deletions src/paimon/common/defs.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ const char Options::FILE_INDEX_FORMAT_VERSION[] = "file-index.format.version";
const char Options::FILE_INDEX_IN_MANIFEST_THRESHOLD[] = "file-index.in-manifest-threshold";
const char Options::DATA_FILE_EXTERNAL_PATHS[] = "data-file.external-paths";
const char Options::DATA_FILE_EXTERNAL_PATHS_STRATEGY[] = "data-file.external-paths.strategy";
const char Options::DATA_FILE_EXTERNAL_PATHS_WEIGHTS[] = "data-file.external-paths.weights";
const char Options::DATA_FILE_PREFIX[] = "data-file.prefix";
const char Options::INDEX_FILE_IN_DATA_FILE_DIR[] = "index-file-in-data-file-dir";
const char Options::ROW_TRACKING_ENABLED[] = "row-tracking.enabled";
Expand Down
42 changes: 34 additions & 8 deletions src/paimon/common/fs/external_path_provider.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,10 @@

#pragma once

#include <algorithm>
#include <atomic>
#include <cstddef>
#include <cstdint>
#include <memory>
#include <random>
#include <string>
Expand All @@ -32,32 +34,55 @@
#include "paimon/status.h"

namespace paimon {
/// Provider for external data paths.
/// Provider for external file paths.
class ExternalPathProvider {
public:
static Result<std::unique_ptr<ExternalPathProvider>> Create(
const std::vector<std::string>& external_table_paths,
const std::string& relative_bucket_path) {
const std::string& relative_bucket_path, const std::vector<int32_t>& weights) {
if (external_table_paths.empty()) {
return Status::Invalid("external table paths cannot be empty");
}
return std::unique_ptr<ExternalPathProvider>(
new ExternalPathProvider(external_table_paths, relative_bucket_path));
std::vector<uint64_t> cumulative_weights;
if (external_table_paths.size() > 1 && !weights.empty()) {
if (weights.size() != external_table_paths.size()) {
return Status::Invalid("external paths and weights must have the same size");
}
cumulative_weights.reserve(weights.size());
uint64_t total = 0;
for (int32_t weight : weights) {
total += weight;
cumulative_weights.push_back(total);
}
}
return std::unique_ptr<ExternalPathProvider>(new ExternalPathProvider(
external_table_paths, relative_bucket_path, cumulative_weights));
}

/// Get the next external data path.
/// Get the next external file path.
///
/// @return the next external data path
/// @return the next external file path
std::string GetNextExternalDataPath(const std::string& file_name) {
size_t position = (++position_) % external_table_paths_.size();
if (!cumulative_weights_.empty()) {
thread_local std::mt19937_64 generator(
std::random_device{}()); // NOLINT(whitespace/braces)
std::uniform_int_distribution<uint64_t> distribution(0, cumulative_weights_.back() - 1);
position = std::upper_bound(cumulative_weights_.begin(), cumulative_weights_.end(),
distribution(generator)) -
cumulative_weights_.begin();
}
return PathUtil::JoinPath(
PathUtil::JoinPath(external_table_paths_[position], relative_bucket_path_), file_name);
}

private:
ExternalPathProvider(const std::vector<std::string>& external_table_paths,
const std::string& relative_bucket_path)
: external_table_paths_(external_table_paths), relative_bucket_path_(relative_bucket_path) {
const std::string& relative_bucket_path,
const std::vector<uint64_t>& cumulative_weights)
: external_table_paths_(external_table_paths),
relative_bucket_path_(relative_bucket_path),
cumulative_weights_(cumulative_weights) {
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_int_distribution<> dist(0, external_table_paths_.size() - 1);
Expand All @@ -68,5 +93,6 @@ class ExternalPathProvider {
std::vector<std::string> external_table_paths_;
std::string relative_bucket_path_;
std::atomic<size_t> position_;
std::vector<uint64_t> cumulative_weights_;
};
} // namespace paimon
34 changes: 30 additions & 4 deletions src/paimon/common/fs/external_path_provider_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@ TEST(ExternalPathProviderTest, TestInvalidExternalDataPath) {
std::vector<std::string> external_table_paths;
std::string relative_bucket_path = "p0=1/p1=0/bucket-0";

auto provider = ExternalPathProvider::Create(external_table_paths, relative_bucket_path);
auto provider =
ExternalPathProvider::Create(external_table_paths, relative_bucket_path, /*weights=*/{});
ASSERT_NOK_WITH_MSG(provider.status(), "external table paths cannot be empty");
}

Expand All @@ -43,7 +44,8 @@ TEST(ExternalPathProviderTest, TestGetNextExternalDataPath) {
std::string relative_bucket_path = "p0=1/p1=0/bucket-0";

ASSERT_OK_AND_ASSIGN(std::unique_ptr<ExternalPathProvider> provider,
ExternalPathProvider::Create(external_table_paths, relative_bucket_path));
ExternalPathProvider::Create(external_table_paths, relative_bucket_path,
/*weights=*/{}));
ASSERT_EQ(provider->GetNextExternalDataPath("file.orc"),
"/tmp/external_path/p0=1/p1=0/bucket-0/file.orc");
}
Expand All @@ -56,7 +58,8 @@ TEST(ExternalPathProviderTest, TestGetNextExternalDataPath2) {
std::string relative_bucket_path = "p0=1/p1=0/bucket-0";

ASSERT_OK_AND_ASSIGN(std::unique_ptr<ExternalPathProvider> provider,
ExternalPathProvider::Create(external_table_paths, relative_bucket_path));
ExternalPathProvider::Create(external_table_paths, relative_bucket_path,
/*weights=*/{}));

std::set<std::string> result_data_paths;
result_data_paths.insert(provider->GetNextExternalDataPath("file.orc"));
Expand All @@ -70,6 +73,28 @@ TEST(ExternalPathProviderTest, TestGetNextExternalDataPath2) {
}));
}

TEST(ExternalPathProviderTest, TestGetNextExternalDataPathWithWeights) {
std::vector<std::string> external_table_paths = {"/tmp/external_path_a/",
"/tmp/external_path_b/"};
std::string relative_bucket_path = "p0=1/p1=0/bucket-0";
ASSERT_OK_AND_ASSIGN(std::unique_ptr<ExternalPathProvider> provider,
ExternalPathProvider::Create(external_table_paths, relative_bucket_path,
/*weights=*/{9, 1}));

int32_t first_path_count = 0;
int32_t second_path_count = 0;
for (int32_t i = 0; i < 1000; ++i) {
std::string path = provider->GetNextExternalDataPath("file.orc");
if (path == "/tmp/external_path_a/p0=1/p1=0/bucket-0/file.orc") {
++first_path_count;
} else {
ASSERT_EQ(path, "/tmp/external_path_b/p0=1/p1=0/bucket-0/file.orc");
++second_path_count;
}
}
ASSERT_GT(first_path_count, second_path_count);
}

TEST(ExternalPathProviderTest, TestGetNextExternalDataPathConcurrently) {
std::vector<std::string> external_table_paths;
external_table_paths.emplace_back("/tmp/external_path_a/");
Expand All @@ -78,7 +103,8 @@ TEST(ExternalPathProviderTest, TestGetNextExternalDataPathConcurrently) {
std::string relative_bucket_path = "p0=1/p1=0/bucket-0";

ASSERT_OK_AND_ASSIGN(std::unique_ptr<ExternalPathProvider> provider,
ExternalPathProvider::Create(external_table_paths, relative_bucket_path));
ExternalPathProvider::Create(external_table_paths, relative_bucket_path,
/*weights=*/{}));

const std::set<std::string> expected_data_paths = {
Comment thread
lxy-9602 marked this conversation as resolved.
Resolved
"/tmp/external_path_a/p0=1/p1=0/bucket-0/file.orc",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -142,9 +142,9 @@ Result<std::vector<GlobalIndexIOMeta>> BitmapGlobalIndexWriter::Finish() {
}
SortedIndexFileMeta index_meta(first_key_bytes, last_key_bytes, !null_rows_.IsEmpty());
std::shared_ptr<Bytes> metadata = index_meta.Serialize(pool_.get());
PAIMON_ASSIGN_OR_RAISE(int64_t file_size, file_writer_->GetFileSize(file_name_));
return std::vector<GlobalIndexIOMeta>{GlobalIndexIOMeta(
file_writer_->ToPath(file_name_), file_size, row_count_, std::move(metadata))};
PAIMON_ASSIGN_OR_RAISE(int64_t file_size, file_writer_->GetFileSize(file_path_));
return std::vector<GlobalIndexIOMeta>{
GlobalIndexIOMeta(file_path_, file_size, row_count_, std::move(metadata))};
}

Status BitmapGlobalIndexWriter::FlushCurrentBitmap() {
Expand All @@ -169,9 +169,9 @@ BitmapGlobalIndexWriter::GetOrCreateStreamingWriter() {
if (streaming_writer_ != nullptr) {
return streaming_writer_.get();
}
PAIMON_ASSIGN_OR_RAISE(file_name_, file_writer_->NewFileName(kBitmapIdentifier));
PAIMON_ASSIGN_OR_RAISE(file_path_, file_writer_->NewPath(kBitmapIdentifier));
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<OutputStream> output_stream,
file_writer_->NewOutputStream(file_name_));
file_writer_->NewOutputStream(file_path_));
output_stream_ = std::move(output_stream);
PAIMON_ASSIGN_OR_RAISE(streaming_writer_, BitmapGlobalIndexFormat::StreamingWriter::Create(
output_stream_, dictionary_block_size_,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ class BitmapGlobalIndexWriter : public GlobalIndexWriter {
std::shared_ptr<BlockCompressionFactory> compression_factory_;
std::shared_ptr<MemoryPool> pool_;

std::string file_name_;
std::string file_path_;
std::shared_ptr<OutputStream> output_stream_;
std::unique_ptr<BitmapGlobalIndexFormat::StreamingWriter> streaming_writer_;
int64_t row_count_ = 0;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,25 +50,20 @@ class FakeGlobalIndexFileWriter : public GlobalIndexFileWriter {
FakeGlobalIndexFileWriter(const std::shared_ptr<FileSystem>& fs, const std::string& base_path)
: fs_(fs), base_path_(base_path) {}

Result<std::string> NewFileName(const std::string& prefix) const override {
return prefix + "_" + std::to_string(file_counter_++);
Result<std::string> NewPath(const std::string& prefix) const override {
return base_path_ + "/" + prefix + "_" + std::to_string(file_counter_++);
}

Result<std::unique_ptr<OutputStream>> NewOutputStream(
const std::string& file_name) const override {
return fs_->Create(base_path_ + "/" + file_name, true);
const std::string& file_path) const override {
return fs_->Create(file_path, true);
}

Result<int64_t> GetFileSize(const std::string& file_name) const override {
PAIMON_ASSIGN_OR_RAISE(FileStatus file_status,
fs_->GetFileStatus(base_path_ + "/" + file_name));
Result<int64_t> GetFileSize(const std::string& file_path) const override {
PAIMON_ASSIGN_OR_RAISE(FileStatus file_status, fs_->GetFileStatus(file_path));
return file_status.GetLen();
}

std::string ToPath(const std::string& file_name) const override {
return base_path_ + "/" + file_name;
}

private:
std::shared_ptr<FileSystem> fs_;
std::string base_path_;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,29 +61,29 @@ Result<std::shared_ptr<BTreeGlobalIndexWriter>> BTreeGlobalIndexWriter::Create(
PAIMON_ASSIGN_OR_RAISE(bloom_filter_builder,
BloomFilter::DynamicBuilder(kBloomFilterFpp, pool));
}
PAIMON_ASSIGN_OR_RAISE(std::string index_file_name,
file_writer->NewFileName(BtreeDefs::kIdentifier));
PAIMON_ASSIGN_OR_RAISE(std::string index_file_path,
file_writer->NewPath(BtreeDefs::kIdentifier));
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<OutputStream> output_stream,
file_writer->NewOutputStream(index_file_name));
file_writer->NewOutputStream(index_file_path));
auto sst_file_writer = std::make_unique<SstFileWriter>(output_stream, bloom_filter_builder,
block_size, compression_factory, pool);
return std::shared_ptr<BTreeGlobalIndexWriter>(new BTreeGlobalIndexWriter(
field_name, arrow_type, key_serializer, file_writer, index_file_name, output_stream,
field_name, arrow_type, key_serializer, file_writer, index_file_path, output_stream,
std::move(sst_file_writer), pool));
}

BTreeGlobalIndexWriter::BTreeGlobalIndexWriter(
const std::string& field_name, const std::shared_ptr<arrow::DataType>& arrow_type,
const std::shared_ptr<KeySerializer>& key_serializer,
const std::shared_ptr<GlobalIndexFileWriter>& file_writer, const std::string& index_file_name,
const std::shared_ptr<GlobalIndexFileWriter>& file_writer, const std::string& index_file_path,
const std::shared_ptr<OutputStream>& output_stream, std::unique_ptr<SstFileWriter>&& sst_writer,
const std::shared_ptr<MemoryPool>& pool)
: field_name_(field_name),
arrow_type_(arrow_type),
key_serializer_(key_serializer),
pool_(pool),
file_writer_(file_writer),
index_file_name_(index_file_name),
index_file_path_(index_file_path),
output_stream_(output_stream),
sst_writer_(std::move(sst_writer)) {}

Expand Down Expand Up @@ -219,9 +219,8 @@ Result<std::vector<GlobalIndexIOMeta>> BTreeGlobalIndexWriter::Finish() {
auto meta_bytes = index_meta->Serialize(pool_.get());

// Create GlobalIndexIOMeta
std::string file_path = file_writer_->ToPath(index_file_name_);
PAIMON_ASSIGN_OR_RAISE(int64_t file_size, file_writer_->GetFileSize(index_file_name_));
GlobalIndexIOMeta io_meta(file_path, file_size, row_count_, meta_bytes);
PAIMON_ASSIGN_OR_RAISE(int64_t file_size, file_writer_->GetFileSize(index_file_path_));
GlobalIndexIOMeta io_meta(index_file_path_, file_size, row_count_, meta_bytes);
return std::vector<GlobalIndexIOMeta>{io_meta};
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ class BTreeGlobalIndexWriter : public GlobalIndexWriter {
const std::shared_ptr<arrow::DataType>& arrow_type,
const std::shared_ptr<KeySerializer>& key_serializer,
const std::shared_ptr<GlobalIndexFileWriter>& file_writer,
const std::string& index_file_name,
const std::string& index_file_path,
const std::shared_ptr<OutputStream>& output_stream,
std::unique_ptr<SstFileWriter>&& sst_writer,
const std::shared_ptr<MemoryPool>& pool);
Expand All @@ -95,7 +95,7 @@ class BTreeGlobalIndexWriter : public GlobalIndexWriter {
std::shared_ptr<MemoryPool> pool_;

std::shared_ptr<GlobalIndexFileWriter> file_writer_;
std::string index_file_name_;
std::string index_file_path_;
std::shared_ptr<OutputStream> output_stream_;
std::unique_ptr<SstFileWriter> sst_writer_;

Expand Down
Loading
Loading