Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 45 additions & 2 deletions benchmark/parquet_format_benchmark.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -574,6 +574,8 @@ Result<int64_t> WriteParquetFile(const std::shared_ptr<FileSystem>& fs, const st

struct ReadStats {
int64_t rows = 0;
double first_batch_ns = 0;
int64_t peak_memory_bytes = 0;
// Reader-side counterpart of file size: makes pruning and skipping visible apart from CPU.
uint64_t storage_bytes = 0;
// Without these a filtered case shows only that it got faster, not that pruning is why.
Expand All @@ -598,15 +600,18 @@ Result<ReadStats> ReadParquetFile(const std::shared_ptr<FileSystem>& fs, const s
const std::map<std::string, std::string>& options,
int32_t batch_size,
const std::shared_ptr<arrow::MemoryPool>& pool) {
const auto started = std::chrono::steady_clock::now();
auto read_pool = std::make_shared<arrow::ProxyMemoryPool>(pool.get());
PAIMON_ASSIGN_OR_RAISE(FileStatus file_status, fs->GetFileStatus(path));
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InputStream> input, fs->Open(path));
auto in_stream = std::make_shared<ArrowInputStreamAdapter>(input, file_status.GetLen(), pool);
auto in_stream =
std::make_shared<ArrowInputStreamAdapter>(input, file_status.GetLen(), read_pool);
// Held separately so the counter outlives the adapter the reader takes ownership of.
std::shared_ptr<std::atomic<uint64_t>> storage_read_bytes = in_stream->StorageReadBytes();
PAIMON_ASSIGN_OR_RAISE(
std::unique_ptr<ParquetFileBatchReader> reader,
ParquetFileBatchReader::Create(std::move(in_stream), options, batch_size,
/*file_metadata=*/nullptr, storage_read_bytes, pool,
/*file_metadata=*/nullptr, storage_read_bytes, read_pool,
/*hints=*/std::nullopt));

// SetReadSchema imports the C schema and takes ownership of it.
Expand All @@ -620,6 +625,12 @@ Result<ReadStats> ReadParquetFile(const std::shared_ptr<FileSystem>& fs, const s
if (BatchReader::IsEofBatch(batch)) {
break;
}
if (stats.batches == 0) {
stats.first_batch_ns =
std::chrono::duration<double, std::nano>(std::chrono::steady_clock::now() - started)
.count();
}
++stats.batches;
stats.rows += batch.first->length;
ArrowArrayRelease(batch.first.get());
ArrowSchemaRelease(batch.second.get());
Expand All @@ -636,6 +647,7 @@ Result<ReadStats> ReadParquetFile(const std::shared_ptr<FileSystem>& fs, const s
ReadCounter(metrics, paimon::parquet::ParquetMetrics::READ_BATCH_COUNT));
reader->Close();
stats.storage_bytes = storage_read_bytes->load();
stats.peak_memory_bytes = read_pool->max_memory();
return stats;
}

Expand Down Expand Up @@ -1201,6 +1213,8 @@ void RunReadBenchmark(::benchmark::State& state, const ReadFixture& fixture,
}

ReadStats stats;
double first_batch_ns = 0;
int64_t peak_memory_bytes = 0;
Timer timer;
for (auto _ : state) {
Result<ReadStats> result =
Expand All @@ -1210,6 +1224,8 @@ void RunReadBenchmark(::benchmark::State& state, const ReadFixture& fixture,
return;
}
stats = result.value();
first_batch_ns += stats.first_batch_ns;
peak_memory_bytes = std::max(peak_memory_bytes, stats.peak_memory_bytes);
}
if (stats.rows < expected.min || stats.rows > expected.max) {
FailBenchmark(state, Status::Invalid("read " + std::to_string(stats.rows) +
Expand Down Expand Up @@ -1239,6 +1255,10 @@ void RunReadBenchmark(::benchmark::State& state, const ReadFixture& fixture,
state.counters["row_groups_after_filter"] =
::benchmark::Counter(static_cast<double>(stats.row_groups_after_filter));
state.counters["batches"] = ::benchmark::Counter(static_cast<double>(stats.batches));
state.counters["first_batch_ns"] =
::benchmark::Counter(first_batch_ns / static_cast<double>(state.iterations()));
state.counters["peak_memory_bytes"] =
::benchmark::Counter(static_cast<double>(peak_memory_bytes));
}

// Sparse lookup into a large row group of 1 KiB values. Page size controls how much
Expand Down Expand Up @@ -1344,6 +1364,19 @@ void BM_ParquetRead_Filtered(::benchmark::State& state) {
RowExpectation{threshold, row_group_bound});
}

void BM_ParquetRead_FilteredBatchSize(::benchmark::State& state) {
const int64_t threshold = kRowGroupLength / 2;
const bool enable_page_index = state.range(1) != 0;
std::shared_ptr<Predicate> predicate =
PredicateBuilder::LessThan(0, "id", FieldType::BIGINT, Literal(threshold));
RunReadBenchmark(
state, PlainFlatFixture(), FlatSchema(), predicate, std::nullopt,
{{paimon::parquet::PARQUET_READ_ENABLE_PAGE_INDEX_FILTER,
enable_page_index ? "true" : "false"}},
static_cast<int32_t>(state.range(0)),
RowExpectation{threshold, enable_page_index ? kRowGroupLength - 1 : kRowGroupLength});
}

// arg: distance between selected rows. The strides straddle the coalesce hole limit (32 rows by
// default), which splits this into two regimes: at or below it neighbouring single-row ranges
// merge into long spans, so rows_read runs far ahead of selected_rows and nothing is skipped;
Expand Down Expand Up @@ -1603,6 +1636,16 @@ BENCHMARK(BM_ParquetRead_Filtered)
->Args({50, 0})
->Unit(benchmark::kMillisecond)
->UseRealTime();
BENCHMARK(BM_ParquetRead_FilteredBatchSize)
->ArgNames({"batch_rows", "page_index"})
->Args({64, 1})
->Args({64, 0})
->Args({1024, 1})
->Args({1024, 0})
->Args({4096, 1})
->Args({4096, 0})
->Unit(benchmark::kMillisecond)
->UseRealTime();
BENCHMARK(BM_ParquetRead_SkipHeavy)
->ArgName("stride")
->Arg(8)
Expand Down
18 changes: 18 additions & 0 deletions docs/source/examples/benchmark.rst
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,14 @@ visible in both directions. Read cases additionally report ``rows_read``,
``batches``, and ``row_groups`` / ``row_groups_after_filter`` from the reader's
own metrics.

Read cases also report ``first_batch_ns``, the mean time from opening the file
through returning the first batch, including metadata, schema setup and
pre-buffering. ``peak_memory_bytes`` is the maximum tracked Arrow allocation
across iterations, including pre-buffered data and decoder state. It excludes
fixture generation and allocations outside the reader's Arrow pool, so it is
not a process-wide memory measurement. Returned batches are released as they
are consumed.

Compare filtered cases on ``ns_per_input_row`` and ``bytes_per_input_row``, not
``ns_per_row`` and ``bytes_per_row``. The latter pair divides by the rows a case
actually materialized, so pruning shrinks numerator and denominator together and
Expand Down Expand Up @@ -208,6 +216,16 @@ Read batch size sweep, repeated for a stable comparison::
--benchmark_repetitions=5 \
--benchmark_report_aggregates_only=true

Page-filtered decoding with 64, 1024 and 4096 rows per batch, paired with
page-index filtering disabled. The predicate partially selects the first row
group; compare ``first_batch_ns`` and ``peak_memory_bytes`` across batch sizes,
and check ``read_bytes`` to distinguish decoding changes from page pruning::

paimon-parquet-format-benchmark \
--benchmark_filter=BM_ParquetRead_FilteredBatchSize \
--benchmark_repetitions=5 \
--benchmark_report_aggregates_only=true

Null density on both sides, to see what definition levels cost::

paimon-parquet-format-benchmark --benchmark_filter='Parquet(Write|Read)_Nulls'
Expand Down
2 changes: 2 additions & 0 deletions src/paimon/format/parquet/file_reader_wrapper.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -517,6 +517,8 @@ void FileReaderWrapper::DispatchPreBuffer(std::vector<::arrow::io::ReadRange> ra
Status FileReaderWrapper::PrepareForReading(const std::vector<TargetRowGroup>& target_row_groups,
const std::vector<int32_t>& column_indices) {
try {
current_page_filtered_reader_.reset();
filtered_global_offset_ = 0;
target_row_groups_ = target_row_groups;
target_column_indices_ = column_indices;

Expand Down
4 changes: 0 additions & 4 deletions src/paimon/format/parquet/file_reader_wrapper.h
Original file line number Diff line number Diff line change
Expand Up @@ -122,10 +122,6 @@ class FileReaderWrapper {

/// Prepare for immediate reading of the specified row groups and columns.
/// Initializes the reader and starts pre-buffering I/O.
///
/// Note: when the read schema has nested sub-field projection,
/// page-level filtering is disabled temporarily due to known offset
/// calculation issues for nested pages.
Status PrepareForReading(const std::vector<TargetRowGroup>& target_row_groups,
const std::vector<int32_t>& column_indices);

Expand Down
Loading
Loading