Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 61 additions & 4 deletions benchmark/parquet_format_benchmark.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,6 @@
// attributed without the catalog lookup, split planning, merge/sort and commit that the
// table-level read_write_benchmark includes. Each case comment says what that case answers.
//
// Every axis - type, cardinality, null density, batch size, encoding, selectivity - is swept on
// its own rather than as a combined matrix, because the point is attributing one change rather
// than describing a workload.
//
// Data is generated outside the timed region and on Arrow's default pool, because the writer
// cuts a new row group once its own pool crosses parquet.writer.max.memory.use.
//
Expand All @@ -50,6 +46,7 @@
#include "arrow/c/bridge.h"
#include "arrow/c/helpers.h"
#include "arrow/util/bit_util.h"
#include "arrow/util/thread_pool.h"
#include "benchmark/benchmark.h"
#include "fmt/format.h"
#include "paimon/common/utils/arrow/arrow_input_stream_adapter.h"
Expand Down Expand Up @@ -1311,6 +1308,54 @@ void BM_ParquetRead_FullScan(::benchmark::State& state) {
/*selection_bitmap=*/std::nullopt, /*options=*/{}, kReadBatchSize);
}

// Policy -1 uses defaults; -2 mixes legacy counts 0/1/3/6 across readers.
void BM_ParquetRead_Concurrent(::benchmark::State& state) {
const auto& fixture = FlatFixture();
if (FailBenchmark(state, fixture.status())) {
return;
}
std::map<std::string, std::string> options;
int64_t legacy_thread_count = state.range(0);
if (legacy_thread_count == -2) {
constexpr int32_t kLegacyThreadCounts[] = {0, 1, 3, 6};
legacy_thread_count = kLegacyThreadCounts[state.thread_index() % 4];
}
if (legacy_thread_count >= 0) {
options[paimon::parquet::PARQUET_READ_EXECUTOR_THREAD_COUNT] =
fmt::format("{}", legacy_thread_count);
}
const auto schema = FlatSchema();
double reader_elapsed_ns = 0;
Timer workload_timer;
for (auto _ : state) {
Timer timer;
Result<ReadStats> result = ReadParquetFile(
fixture.fs(), fixture.path(), schema, /*predicate=*/nullptr,
/*selection_bitmap=*/std::nullopt, options, kReadBatchSize, fixture.arrow_pool());
reader_elapsed_ns += timer.ElapsedNanos();
if (FailBenchmark(state, result.status())) {
return;
}
if (result.value().rows != kRowsPerFile) {
FailBenchmark(state,
Status::Invalid("concurrent reader returned an incorrect row count"));
return;
}
}
// Counter values are summed across reader threads. Include the loop's completion barrier
// so a faster reader in the mixed case does not inflate aggregate throughput.
state.counters["rows_per_second"] =
::benchmark::Counter(static_cast<double>(state.iterations() * kRowsPerFile) * 1e9 /
workload_timer.ElapsedNanos());
if (state.iterations() > 0) {
state.counters["latency_ms"] =
::benchmark::Counter(reader_elapsed_ns / static_cast<double>(state.iterations()) / 1e6,
::benchmark::Counter::kAvgThreads);
}
state.counters["cpu_capacity"] =
::benchmark::Counter(arrow::GetCpuThreadPoolCapacity(), ::benchmark::Counter::kAvgThreads);
}

// One column at a time, separating per-column transfer from materializing the whole row.
void BM_ParquetRead_Projection(::benchmark::State& state, const char* column) {
std::shared_ptr<arrow::Field> field = FlatSchema()->GetFieldByName(column);
Expand Down Expand Up @@ -1584,6 +1629,18 @@ BENCHMARK(BM_ParquetRead_OffsetIndexCache)
->Unit(benchmark::kMillisecond)
->UseRealTime();
BENCHMARK(BM_ParquetRead_FullScan)->Unit(benchmark::kMillisecond)->UseRealTime();
BENCHMARK(BM_ParquetRead_Concurrent)
->ArgName("policy")
->Arg(-1)
->Arg(0)
->Arg(3)
->Arg(6)
->Arg(-2)
->Threads(1)
->Threads(4)
->MeasureProcessCPUTime()
->UseRealTime()
->Unit(benchmark::kMillisecond);
BENCHMARK_CAPTURE(BM_ParquetRead_Projection, id, "id")
->Unit(benchmark::kMillisecond)
->UseRealTime();
Expand Down
46 changes: 38 additions & 8 deletions docs/source/examples/benchmark.rst
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,8 @@ source file or table.

Two things shape how the results should be read:

- Every axis is swept on its own rather than as a combined matrix, so each
case answers one question and a change can be attributed to it.
- Most cases sweep a single axis. Concurrent reads vary both reader policy
and reader count to expose interactions through the shared CPU pool.
- Writes go through the local FileSystem into a temporary directory, so
absolute numbers carry the cost of that path. Comparisons are meaningful
only under the same environment and methodology - the same machine, build
Expand Down Expand Up @@ -157,13 +157,14 @@ Reader cases (``BM_ParquetRead_*``) cover full scan, single-column projection,
predicate-filtered reads at varying selectivity with page-index filtering on and
off, skip-heavy reads driven by a strided selection bitmap, null density,
``DECIMAL`` at precision 9 / 18 / 38, ``DOUBLE``, dictionary-encoded against
plain-encoded files, rows per ``NextBatch`` call, and nested column reads.
plain-encoded files, rows per ``NextBatch`` call, nested column reads, and
concurrent readers with different threading policies.

Every case reports ``ns_per_row`` next to ``bytes_per_row`` - ``file_bytes`` for
writes, ``read_bytes`` for reads - so a change that trades CPU for size is
visible in both directions. Read cases additionally report ``rows_read``,
``batches``, and ``row_groups`` / ``row_groups_after_filter`` from the reader's
own metrics.
Writer and single-reader cases report ``ns_per_row`` next to ``bytes_per_row`` -
``file_bytes`` for writes, ``read_bytes`` for reads - so a change that trades CPU
for size is visible in both directions. Single-reader cases additionally report
``rows_read``, ``batches``, and ``row_groups`` / ``row_groups_after_filter`` from
the reader's own metrics.

Compare filtered cases on ``ns_per_input_row`` and ``bytes_per_input_row``, not
``ns_per_row`` and ``bytes_per_row``. The latter pair divides by the rows a case
Expand Down Expand Up @@ -211,3 +212,32 @@ Read batch size sweep, repeated for a stable comparison::
Null density on both sides, to see what definition levels cost::

paimon-parquet-format-benchmark --benchmark_filter='Parquet(Write|Read)_Nulls'

Concurrent reader CPU policy
----------------------------

``BM_ParquetRead_Concurrent`` runs one or four readers over the same cached local
file. Policy ``-1`` uses default reader options; ``0``, ``3`` and ``6`` set the
legacy ``parquet.read.executor.thread-count`` option; ``-2`` assigns legacy
values ``0``, ``1``, ``3`` and ``6`` by reader index. This allows the same workload
to compare the old global-capacity mutation with the application-owned policy
described in :doc:`../user_guide/read`.

The counters report aggregate ``rows_per_second``, mean reader open/scan/close
``latency_ms``, and the resulting Arrow ``cpu_capacity``. Google Benchmark's CPU
time includes all process threads, including Arrow column workers. These cases
exercise format readers directly; they do not include Paimon's outer prefetch
executor.

For before/after comparisons, run each policy and reader count in a fresh process
with the same application CPU budget. Older readers change the global pool, so
running several policies in one process can affect later cases. Check the
reported capacity alongside throughput and latency: a difference in CPU budget
is part of the policy change, not a fixed-budget speedup.

Four readers with mixed legacy settings and an application CPU budget of four::

OMP_NUM_THREADS=4 paimon-parquet-format-benchmark \
--benchmark_filter='BM_ParquetRead_Concurrent/policy:-2/.*threads:4$' \
--benchmark_min_time=1s \
--benchmark_repetitions=3
55 changes: 55 additions & 0 deletions docs/source/user_guide/read.rst
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,61 @@ cross-language task scheduling and interaction (e.g., Java and C++), substantial
across the two language ecosystems.


Parquet CPU parallelism
-----------------------

``parquet.read.use-threads`` controls parallel column decoding within a Parquet
reader and defaults to ``true``. ``false`` decodes columns on the calling thread.
Creating, reading and closing readers never changes Arrow's global CPU pool
capacity. Threaded readers share the existing pool, including with Arrow IPC
spill operations; they do not reserve a separate number of threads per reader.

The application owns the CPU capacity policy. Configure Arrow's CPU pool once
at application startup, for example with ``OMP_NUM_THREADS`` before the process
starts, or with ``arrow::SetCpuThreadPoolCapacity(n)`` before starting concurrent
Arrow work. The API must address the same Arrow library instance used by Paimon.
Without application configuration, Arrow chooses its own default capacity.
``WriteContextBuilder::SetWriteBufferSpillThreadNumber(n)`` remains an explicit
process-wide capacity setter when ``n > 0``: ``Finish()`` applies it to the same
pool, and subsequent Parquet readers preserve that setting. Coordinate these
spill settings with the application's budget; different positive spill settings
still resize the shared pool.

Migration from ``parquet.read.executor.thread-count``
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

The numeric option is deprecated and accepts an unsigned 32-bit integer. When
``parquet.read.use-threads`` is absent, ``0`` still disables column threading
and any positive value enables it. The legacy default of ``3`` therefore still
enables threaded reads, but **no longer sets a three-thread global capacity**.
Different positive values have the same effect and coexist without changing
each other's scheduler. When both options are supplied,
``parquet.read.use-threads`` takes precedence; a malformed legacy value is still
rejected.

Replace a legacy zero with ``parquet.read.use-threads=false``. Replace a positive
value with ``parquet.read.use-threads=true`` and move the capacity setting to
application startup if a fixed budget is required. For example, a former value
of ``6`` becomes ``use-threads=true`` with an application CPU capacity of six.

Interaction with prefetch
~~~~~~~~~~~~~~~~~~~~~~~~~

Paimon's prefetch executor and Arrow's CPU pool are separate resources.
``ReadContextBuilder::WithExecutor`` and ``SetPrefetchMaxParallelNum`` control
outer file/range parallelism; ``parquet.read.use-threads`` controls inner column
parallelism. Parallel construction of prefetch sub-readers does not create or
resize CPU pools. With many concurrent readers, measure both settings together;
disabling inner column threading can reduce scheduling overhead.

The supported Arrow 17 synchronous ``GetRecordBatchReader`` API has no scoped
CPU executor parameter. Its asynchronous generator does, but using it would
change the batch/row-group read path. This configuration change keeps the
synchronous path and its decoding behavior. Do not run synchronous threaded
Parquet reads on the same Arrow CPU pool they submit column tasks to: a worker
waiting on its own saturated pool can deadlock. The Paimon prefetch executor is
independent of that pool.

Planning with a custom memory pool
----------------------------------

Expand Down
4 changes: 3 additions & 1 deletion include/paimon/write_context.h
Original file line number Diff line number Diff line change
Expand Up @@ -302,7 +302,9 @@ class PAIMON_EXPORT WriteContextBuilder {

/// Set the thread number for write buffer spill operations. (default is 0)
/// If <= 0, threading is disabled for spill IPC read/write.
/// If > 0, sets arrow CPU thread pool capacity for spill operations.
/// If > 0, Finish() sets the process-wide Arrow CPU pool capacity and enables threaded
/// spill IPC read/write. Parquet readers and other Arrow CPU consumers share this pool.
/// Coordinate positive values with application CPU configuration before concurrent work.
/// @param thread_number The thread number to use for spill operations.
/// @return Reference to this builder for method chaining.
WriteContextBuilder& SetWriteBufferSpillThreadNumber(int32_t thread_number);
Expand Down
2 changes: 1 addition & 1 deletion src/paimon/format/parquet/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ if(PAIMON_BUILD_TESTS)
column_index_filter_test.cpp
variant_parquet_test.cpp
STATIC_LINK_LIBS
paimon_shared
paimon_static
test_utils_static
${PAIMON_PARQUET_FILE_FORMAT_STATIC_LINK_LIBS}
${PAIMON_AVRO_FILE_FORMAT_STATIC_LINK_LIBS}
Expand Down
15 changes: 7 additions & 8 deletions src/paimon/format/parquet/parquet_file_batch_reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@
#include "arrow/record_batch.h"
#include "arrow/type.h"
#include "arrow/util/range.h"
#include "arrow/util/thread_pool.h"
#include "fmt/format.h"
#include "paimon/common/metrics/metrics_impl.h"
#include "paimon/common/utils/arrow/arrow_utils.h"
Expand Down Expand Up @@ -823,21 +822,21 @@ Result<::parquet::ArrowReaderProperties> ParquetFileBatchReader::CreateArrowRead
const std::map<std::string, std::string>& options, int32_t batch_size,
const std::optional<ReadHints>& hints) {
PAIMON_ASSIGN_OR_RAISE(
uint32_t executor_thread_count,
uint32_t legacy_thread_count,
OptionsUtils::GetValueFromMap<uint32_t>(options, PARQUET_READ_EXECUTOR_THREAD_COUNT,
DEFAULT_PARQUET_READ_EXECUTOR_THREAD_COUNT));
PAIMON_ASSIGN_OR_RAISE(bool use_threads,
OptionsUtils::GetValueFromMap<bool>(options, PARQUET_READ_USE_THREADS,
legacy_thread_count != 0));

::parquet::ArrowReaderProperties arrow_reader_props;
// TODO(jinli.zjw): set more ArrowReaderProperties (compare with java)
PAIMON_ASSIGN_OR_RAISE(bool enable_pre_buffer, ResolvePreBufferEnabled(options, hints));
arrow_reader_props.set_pre_buffer(enable_pre_buffer);
arrow_reader_props.set_batch_size(static_cast<int64_t>(batch_size));
if (executor_thread_count != 0) {
PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::SetCpuThreadPoolCapacity(executor_thread_count));
arrow_reader_props.set_use_threads(true);
} else {
arrow_reader_props.set_use_threads(false);
}
// Arrow 17's synchronous reader uses the application-owned global CPU pool.
// Reader properties must not change its capacity.
arrow_reader_props.set_use_threads(use_threads);
PAIMON_ASSIGN_OR_RAISE(bool cache_lazy, OptionsUtils::GetValueFromMap<bool>(
options, PARQUET_READ_CACHE_OPTION_LAZY, false));
PAIMON_ASSIGN_OR_RAISE(
Expand Down
Loading
Loading