diff --git a/benchmark/parquet_format_benchmark.cpp b/benchmark/parquet_format_benchmark.cpp index 40094d31d..047b8b2f7 100644 --- a/benchmark/parquet_format_benchmark.cpp +++ b/benchmark/parquet_format_benchmark.cpp @@ -22,10 +22,6 @@ // attributed without the catalog lookup, split planning, merge/sort and commit that the // table-level read_write_benchmark includes. Each case comment says what that case answers. // -// Every axis - type, cardinality, null density, batch size, encoding, selectivity - is swept on -// its own rather than as a combined matrix, because the point is attributing one change rather -// than describing a workload. -// // Data is generated outside the timed region and on Arrow's default pool, because the writer // cuts a new row group once its own pool crosses parquet.writer.max.memory.use. // @@ -50,6 +46,7 @@ #include "arrow/c/bridge.h" #include "arrow/c/helpers.h" #include "arrow/util/bit_util.h" +#include "arrow/util/thread_pool.h" #include "benchmark/benchmark.h" #include "fmt/format.h" #include "paimon/common/utils/arrow/arrow_input_stream_adapter.h" @@ -1311,6 +1308,54 @@ void BM_ParquetRead_FullScan(::benchmark::State& state) { /*selection_bitmap=*/std::nullopt, /*options=*/{}, kReadBatchSize); } +// Policy -1 uses defaults; -2 mixes legacy counts 0/1/3/6 across readers. +void BM_ParquetRead_Concurrent(::benchmark::State& state) { + const auto& fixture = FlatFixture(); + if (FailBenchmark(state, fixture.status())) { + return; + } + std::map options; + int64_t legacy_thread_count = state.range(0); + if (legacy_thread_count == -2) { + constexpr int32_t kLegacyThreadCounts[] = {0, 1, 3, 6}; + legacy_thread_count = kLegacyThreadCounts[state.thread_index() % 4]; + } + if (legacy_thread_count >= 0) { + options[paimon::parquet::PARQUET_READ_EXECUTOR_THREAD_COUNT] = + fmt::format("{}", legacy_thread_count); + } + const auto schema = FlatSchema(); + double reader_elapsed_ns = 0; + Timer workload_timer; + for (auto _ : state) { + Timer timer; + Result result = ReadParquetFile( + fixture.fs(), fixture.path(), schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt, options, kReadBatchSize, fixture.arrow_pool()); + reader_elapsed_ns += timer.ElapsedNanos(); + if (FailBenchmark(state, result.status())) { + return; + } + if (result.value().rows != kRowsPerFile) { + FailBenchmark(state, + Status::Invalid("concurrent reader returned an incorrect row count")); + return; + } + } + // Counter values are summed across reader threads. Include the loop's completion barrier + // so a faster reader in the mixed case does not inflate aggregate throughput. + state.counters["rows_per_second"] = + ::benchmark::Counter(static_cast(state.iterations() * kRowsPerFile) * 1e9 / + workload_timer.ElapsedNanos()); + if (state.iterations() > 0) { + state.counters["latency_ms"] = + ::benchmark::Counter(reader_elapsed_ns / static_cast(state.iterations()) / 1e6, + ::benchmark::Counter::kAvgThreads); + } + state.counters["cpu_capacity"] = + ::benchmark::Counter(arrow::GetCpuThreadPoolCapacity(), ::benchmark::Counter::kAvgThreads); +} + // One column at a time, separating per-column transfer from materializing the whole row. void BM_ParquetRead_Projection(::benchmark::State& state, const char* column) { std::shared_ptr field = FlatSchema()->GetFieldByName(column); @@ -1584,6 +1629,18 @@ BENCHMARK(BM_ParquetRead_OffsetIndexCache) ->Unit(benchmark::kMillisecond) ->UseRealTime(); BENCHMARK(BM_ParquetRead_FullScan)->Unit(benchmark::kMillisecond)->UseRealTime(); +BENCHMARK(BM_ParquetRead_Concurrent) + ->ArgName("policy") + ->Arg(-1) + ->Arg(0) + ->Arg(3) + ->Arg(6) + ->Arg(-2) + ->Threads(1) + ->Threads(4) + ->MeasureProcessCPUTime() + ->UseRealTime() + ->Unit(benchmark::kMillisecond); BENCHMARK_CAPTURE(BM_ParquetRead_Projection, id, "id") ->Unit(benchmark::kMillisecond) ->UseRealTime(); diff --git a/docs/source/examples/benchmark.rst b/docs/source/examples/benchmark.rst index 01d49cfb9..c7805c6e2 100644 --- a/docs/source/examples/benchmark.rst +++ b/docs/source/examples/benchmark.rst @@ -108,8 +108,8 @@ source file or table. Two things shape how the results should be read: -- Every axis is swept on its own rather than as a combined matrix, so each - case answers one question and a change can be attributed to it. +- Most cases sweep a single axis. Concurrent reads vary both reader policy + and reader count to expose interactions through the shared CPU pool. - Writes go through the local FileSystem into a temporary directory, so absolute numbers carry the cost of that path. Comparisons are meaningful only under the same environment and methodology - the same machine, build @@ -157,13 +157,14 @@ Reader cases (``BM_ParquetRead_*``) cover full scan, single-column projection, predicate-filtered reads at varying selectivity with page-index filtering on and off, skip-heavy reads driven by a strided selection bitmap, null density, ``DECIMAL`` at precision 9 / 18 / 38, ``DOUBLE``, dictionary-encoded against -plain-encoded files, rows per ``NextBatch`` call, and nested column reads. +plain-encoded files, rows per ``NextBatch`` call, nested column reads, and +concurrent readers with different threading policies. -Every case reports ``ns_per_row`` next to ``bytes_per_row`` - ``file_bytes`` for -writes, ``read_bytes`` for reads - so a change that trades CPU for size is -visible in both directions. Read cases additionally report ``rows_read``, -``batches``, and ``row_groups`` / ``row_groups_after_filter`` from the reader's -own metrics. +Writer and single-reader cases report ``ns_per_row`` next to ``bytes_per_row`` - +``file_bytes`` for writes, ``read_bytes`` for reads - so a change that trades CPU +for size is visible in both directions. Single-reader cases additionally report +``rows_read``, ``batches``, and ``row_groups`` / ``row_groups_after_filter`` from +the reader's own metrics. Compare filtered cases on ``ns_per_input_row`` and ``bytes_per_input_row``, not ``ns_per_row`` and ``bytes_per_row``. The latter pair divides by the rows a case @@ -211,3 +212,32 @@ Read batch size sweep, repeated for a stable comparison:: Null density on both sides, to see what definition levels cost:: paimon-parquet-format-benchmark --benchmark_filter='Parquet(Write|Read)_Nulls' + +Concurrent reader CPU policy +---------------------------- + +``BM_ParquetRead_Concurrent`` runs one or four readers over the same cached local +file. Policy ``-1`` uses default reader options; ``0``, ``3`` and ``6`` set the +legacy ``parquet.read.executor.thread-count`` option; ``-2`` assigns legacy +values ``0``, ``1``, ``3`` and ``6`` by reader index. This allows the same workload +to compare the old global-capacity mutation with the application-owned policy +described in :doc:`../user_guide/read`. + +The counters report aggregate ``rows_per_second``, mean reader open/scan/close +``latency_ms``, and the resulting Arrow ``cpu_capacity``. Google Benchmark's CPU +time includes all process threads, including Arrow column workers. These cases +exercise format readers directly; they do not include Paimon's outer prefetch +executor. + +For before/after comparisons, run each policy and reader count in a fresh process +with the same application CPU budget. Older readers change the global pool, so +running several policies in one process can affect later cases. Check the +reported capacity alongside throughput and latency: a difference in CPU budget +is part of the policy change, not a fixed-budget speedup. + +Four readers with mixed legacy settings and an application CPU budget of four:: + + OMP_NUM_THREADS=4 paimon-parquet-format-benchmark \ + --benchmark_filter='BM_ParquetRead_Concurrent/policy:-2/.*threads:4$' \ + --benchmark_min_time=1s \ + --benchmark_repetitions=3 diff --git a/docs/source/user_guide/read.rst b/docs/source/user_guide/read.rst index d55a141c1..6360aa4c1 100644 --- a/docs/source/user_guide/read.rst +++ b/docs/source/user_guide/read.rst @@ -41,6 +41,61 @@ cross-language task scheduling and interaction (e.g., Java and C++), substantial across the two language ecosystems. +Parquet CPU parallelism +----------------------- + +``parquet.read.use-threads`` controls parallel column decoding within a Parquet +reader and defaults to ``true``. ``false`` decodes columns on the calling thread. +Creating, reading and closing readers never changes Arrow's global CPU pool +capacity. Threaded readers share the existing pool, including with Arrow IPC +spill operations; they do not reserve a separate number of threads per reader. + +The application owns the CPU capacity policy. Configure Arrow's CPU pool once +at application startup, for example with ``OMP_NUM_THREADS`` before the process +starts, or with ``arrow::SetCpuThreadPoolCapacity(n)`` before starting concurrent +Arrow work. The API must address the same Arrow library instance used by Paimon. +Without application configuration, Arrow chooses its own default capacity. +``WriteContextBuilder::SetWriteBufferSpillThreadNumber(n)`` remains an explicit +process-wide capacity setter when ``n > 0``: ``Finish()`` applies it to the same +pool, and subsequent Parquet readers preserve that setting. Coordinate these +spill settings with the application's budget; different positive spill settings +still resize the shared pool. + +Migration from ``parquet.read.executor.thread-count`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The numeric option is deprecated and accepts an unsigned 32-bit integer. When +``parquet.read.use-threads`` is absent, ``0`` still disables column threading +and any positive value enables it. The legacy default of ``3`` therefore still +enables threaded reads, but **no longer sets a three-thread global capacity**. +Different positive values have the same effect and coexist without changing +each other's scheduler. When both options are supplied, +``parquet.read.use-threads`` takes precedence; a malformed legacy value is still +rejected. + +Replace a legacy zero with ``parquet.read.use-threads=false``. Replace a positive +value with ``parquet.read.use-threads=true`` and move the capacity setting to +application startup if a fixed budget is required. For example, a former value +of ``6`` becomes ``use-threads=true`` with an application CPU capacity of six. + +Interaction with prefetch +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Paimon's prefetch executor and Arrow's CPU pool are separate resources. +``ReadContextBuilder::WithExecutor`` and ``SetPrefetchMaxParallelNum`` control +outer file/range parallelism; ``parquet.read.use-threads`` controls inner column +parallelism. Parallel construction of prefetch sub-readers does not create or +resize CPU pools. With many concurrent readers, measure both settings together; +disabling inner column threading can reduce scheduling overhead. + +The supported Arrow 17 synchronous ``GetRecordBatchReader`` API has no scoped +CPU executor parameter. Its asynchronous generator does, but using it would +change the batch/row-group read path. This configuration change keeps the +synchronous path and its decoding behavior. Do not run synchronous threaded +Parquet reads on the same Arrow CPU pool they submit column tasks to: a worker +waiting on its own saturated pool can deadlock. The Paimon prefetch executor is +independent of that pool. + Planning with a custom memory pool ---------------------------------- diff --git a/include/paimon/write_context.h b/include/paimon/write_context.h index e0a940eb0..41067de9a 100644 --- a/include/paimon/write_context.h +++ b/include/paimon/write_context.h @@ -302,7 +302,9 @@ class PAIMON_EXPORT WriteContextBuilder { /// Set the thread number for write buffer spill operations. (default is 0) /// If <= 0, threading is disabled for spill IPC read/write. - /// If > 0, sets arrow CPU thread pool capacity for spill operations. + /// If > 0, Finish() sets the process-wide Arrow CPU pool capacity and enables threaded + /// spill IPC read/write. Parquet readers and other Arrow CPU consumers share this pool. + /// Coordinate positive values with application CPU configuration before concurrent work. /// @param thread_number The thread number to use for spill operations. /// @return Reference to this builder for method chaining. WriteContextBuilder& SetWriteBufferSpillThreadNumber(int32_t thread_number); diff --git a/src/paimon/format/parquet/CMakeLists.txt b/src/paimon/format/parquet/CMakeLists.txt index 937e7124a..a74bf7f78 100644 --- a/src/paimon/format/parquet/CMakeLists.txt +++ b/src/paimon/format/parquet/CMakeLists.txt @@ -66,7 +66,7 @@ if(PAIMON_BUILD_TESTS) column_index_filter_test.cpp variant_parquet_test.cpp STATIC_LINK_LIBS - paimon_shared + paimon_static test_utils_static ${PAIMON_PARQUET_FILE_FORMAT_STATIC_LINK_LIBS} ${PAIMON_AVRO_FILE_FORMAT_STATIC_LINK_LIBS} diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp index 50f3b9735..c4f232bf8 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp @@ -40,7 +40,6 @@ #include "arrow/record_batch.h" #include "arrow/type.h" #include "arrow/util/range.h" -#include "arrow/util/thread_pool.h" #include "fmt/format.h" #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/utils/arrow/arrow_utils.h" @@ -823,21 +822,21 @@ Result<::parquet::ArrowReaderProperties> ParquetFileBatchReader::CreateArrowRead const std::map& options, int32_t batch_size, const std::optional& hints) { PAIMON_ASSIGN_OR_RAISE( - uint32_t executor_thread_count, + uint32_t legacy_thread_count, OptionsUtils::GetValueFromMap(options, PARQUET_READ_EXECUTOR_THREAD_COUNT, DEFAULT_PARQUET_READ_EXECUTOR_THREAD_COUNT)); + PAIMON_ASSIGN_OR_RAISE(bool use_threads, + OptionsUtils::GetValueFromMap(options, PARQUET_READ_USE_THREADS, + legacy_thread_count != 0)); ::parquet::ArrowReaderProperties arrow_reader_props; // TODO(jinli.zjw): set more ArrowReaderProperties (compare with java) PAIMON_ASSIGN_OR_RAISE(bool enable_pre_buffer, ResolvePreBufferEnabled(options, hints)); arrow_reader_props.set_pre_buffer(enable_pre_buffer); arrow_reader_props.set_batch_size(static_cast(batch_size)); - if (executor_thread_count != 0) { - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::SetCpuThreadPoolCapacity(executor_thread_count)); - arrow_reader_props.set_use_threads(true); - } else { - arrow_reader_props.set_use_threads(false); - } + // Arrow 17's synchronous reader uses the application-owned global CPU pool. + // Reader properties must not change its capacity. + arrow_reader_props.set_use_threads(use_threads); PAIMON_ASSIGN_OR_RAISE(bool cache_lazy, OptionsUtils::GetValueFromMap( options, PARQUET_READ_CACHE_OPTION_LAZY, false)); PAIMON_ASSIGN_OR_RAISE( diff --git a/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp b/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp index 703541d19..f3e307bf1 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp @@ -43,14 +43,17 @@ #include "arrow/io/memory.h" #include "arrow/ipc/api.h" #include "arrow/ipc/json_simple.h" +#include "arrow/util/thread_pool.h" #include "fmt/format.h" #include "gtest/gtest.h" #include "paimon/common/data/columnar/columnar_array.h" #include "paimon/common/data/columnar/columnar_row.h" #include "paimon/common/data/columnar/columnar_row_ref.h" +#include "paimon/common/executor/future.h" #include "paimon/common/io/cache_input_stream.h" #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/predicate/equal.h" +#include "paimon/common/reader/prefetch_file_batch_reader_impl.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/arrow/arrow_input_stream_adapter.h" #include "paimon/common/utils/arrow/arrow_utils.h" @@ -59,6 +62,8 @@ #include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/read_ahead_cache.h" +#include "paimon/common/utils/scope_guard.h" +#include "paimon/core/io/arrow_ipc_file.h" #include "paimon/defs.h" #include "paimon/format/parquet/parquet_field_id_converter.h" #include "paimon/format/parquet/parquet_format_defs.h" @@ -77,6 +82,7 @@ #include "paimon/testing/utils/testharness.h" #include "paimon/testing/utils/timezone_guard.h" #include "paimon/utils/roaring_bitmap32.h" +#include "paimon/write_context.h" #include "parquet/arrow/reader.h" #include "parquet/file_reader.h" #include "parquet/metadata.h" @@ -1137,41 +1143,175 @@ TEST_F(ParquetFileBatchReaderTest, TestCreateReaderProperties) { } } -TEST_F(ParquetFileBatchReaderTest, TestCreateArrowReaderProperties) { - { - // test default options - std::map options; - int32_t batch_size = 1024; - ASSERT_OK_AND_ASSIGN( - auto arrow_reader_properties, - ParquetFileBatchReader::CreateArrowReaderProperties(pool_, options, batch_size, - /*hints=*/std::nullopt)); - ASSERT_EQ(arrow_reader_properties.pre_buffer(), true); - ASSERT_EQ(arrow_reader_properties.batch_size(), 1024); - ASSERT_EQ(arrow_reader_properties.use_threads(), true); - ASSERT_EQ(arrow::GetCpuThreadPoolCapacity(), 3); - auto expected_cache_options = arrow::io::CacheOptions::Defaults(); - expected_cache_options.hole_size_limit = DEFAULT_PARQUET_READ_CACHE_OPTION_HOLE_SIZE_LIMIT; - ASSERT_EQ(arrow_reader_properties.cache_options(), expected_cache_options); +TEST_F(ParquetFileBatchReaderTest, TestConcurrentReadersPreserveGlobalCpuPool) { + WriteArray(file_path_, struct_array_, schema_, /*write_batch_size=*/2, + /*enable_dictionary=*/true, /*max_row_group_length=*/2); + const int32_t original_capacity = arrow::GetCpuThreadPoolCapacity(); + ScopeGuard restore_capacity([original_capacity]() { + EXPECT_TRUE(arrow::SetCpuThreadPoolCapacity(original_capacity).ok()); + }); + ASSERT_OK_AND_ASSIGN(auto executor, CreateDefaultExecutor(4)); + const std::vector> configurations = { + {}, + {{PARQUET_READ_EXECUTOR_THREAD_COUNT, "0"}}, + {{PARQUET_READ_EXECUTOR_THREAD_COUNT, "1"}}, + {{PARQUET_READ_EXECUTOR_THREAD_COUNT, "6"}}, + {{PARQUET_READ_USE_THREADS, "false"}}, + {{PARQUET_READ_USE_THREADS, "true"}}}; + for (int32_t spill_threads : {0, 2}) { + SCOPED_TRACE(spill_threads); + ASSERT_TRUE(arrow::SetCpuThreadPoolCapacity(5).ok()); + WriteContextBuilder write_builder(dir_->Str(), "test_user"); + write_builder.SetWriteBufferSpillThreadNumber(spill_threads); + ASSERT_OK_AND_ASSIGN(auto write_context, write_builder.Finish()); + ASSERT_EQ(write_context->EnableMultiThreadSpill(), spill_threads > 0); + const int32_t expected_capacity = spill_threads > 0 ? spill_threads : 5; + ASSERT_EQ(arrow::GetCpuThreadPoolCapacity(), expected_capacity); + + std::vector>>> reader_futures; + for (const auto& options : configurations) { + reader_futures.push_back( + Via(executor.get(), [&, options]() -> Result> { + ParquetReaderBuilder builder(options, /*batch_size=*/1); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr input, + fs_->Open(file_path_)); + return builder.Build(input); + })); + } + std::vector> readers; + for (auto& result : CollectAll(reader_futures)) { + ASSERT_OK(result.status()); + readers.push_back(std::move(result).value()); + ASSERT_EQ(arrow::GetCpuThreadPoolCapacity(), expected_capacity); + } + std::vector>>> futures; + futures.push_back( + Via(executor.get(), [&]() -> Result> { + const bool use_threads = write_context->EnableMultiThreadSpill(); + const std::string spill_path = + PathUtil::JoinPath(dir_->Str(), fmt::format("spill-{}.arrow", spill_threads)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr writer, + ArrowIpcFileWriter::Create(fs_, spill_path, schema_, "zstd", + /*compression_level=*/1, use_threads, pool_)); + auto batch = arrow::RecordBatch::Make(schema_, struct_array_->length(), + struct_array_->fields()); + PAIMON_RETURN_NOT_OK(writer->WriteBatch(batch)); + PAIMON_RETURN_NOT_OK(writer->Close()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr reader, + ArrowIpcFileReader::Open(fs_, spill_path, use_threads, pool_)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr actual, + reader->ReadRecordBatch(0)); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr array, + actual->ToStructArray()); + return std::make_shared(array); + })); + for (const auto& reader : readers) { + futures.push_back(Via(executor.get(), [reader = reader.get()]() { + return paimon::test::ReadResultCollector::CollectResult(reader); + })); + } + for (const auto& result : CollectAll(futures)) { + ASSERT_OK(result.status()); + ASSERT_TRUE(result.value()->Equals(arrow::ChunkedArray(struct_array_))); + } + ASSERT_EQ(arrow::GetCpuThreadPoolCapacity(), expected_capacity); + for (auto& reader : readers) { + reader->Close(); + reader.reset(); + ASSERT_EQ(arrow::GetCpuThreadPoolCapacity(), expected_capacity); + } } - { - std::map options = {{PARQUET_READ_EXECUTOR_THREAD_COUNT, "0"}}; - int32_t batch_size = 1024; - ASSERT_OK_AND_ASSIGN( - auto arrow_reader_properties, - ParquetFileBatchReader::CreateArrowReaderProperties(pool_, options, batch_size, - /*hints=*/std::nullopt)); - ASSERT_EQ(arrow_reader_properties.use_threads(), false); +} + +TEST_F(ParquetFileBatchReaderTest, TestPrefetchReadersPreserveGlobalCpuPool) { + WriteArray(file_path_, struct_array_, schema_, /*write_batch_size=*/2, + /*enable_dictionary=*/true, /*max_row_group_length=*/2); + ASSERT_OK_AND_ASSIGN(auto file_status, fs_->GetFileStatus(file_path_)); + const int32_t original_capacity = arrow::GetCpuThreadPoolCapacity(); + ScopeGuard restore_capacity([original_capacity]() { + EXPECT_TRUE(arrow::SetCpuThreadPoolCapacity(original_capacity).ok()); + }); + ASSERT_TRUE(arrow::SetCpuThreadPoolCapacity(1).ok()); + const std::vector> configurations = { + {}, + {{PARQUET_READ_EXECUTOR_THREAD_COUNT, "0"}}, + {{PARQUET_READ_EXECUTOR_THREAD_COUNT, "6"}}, + {{PARQUET_READ_USE_THREADS, "false"}}, + {{PARQUET_READ_USE_THREADS, "true"}}}; + for (uint32_t outer_threads : {1, 4}) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr executor, + CreateDefaultExecutor(outer_threads)); + for (bool read_ahead : {false, true}) { + for (const auto& options : configurations) { + SCOPED_TRACE(::testing::PrintToString(options)); + SCOPED_TRACE(outer_threads); + SCOPED_TRACE(read_ahead); + ParquetReaderBuilder builder(options, /*batch_size=*/1); + builder.WithReadHints(ReadHints{true, read_ahead}); + ASSERT_OK_AND_ASSIGN( + auto reader, + PrefetchFileBatchReaderImpl::Create( + file_path_, file_status.GetLen(), &builder, fs_, + /*prefetch_max_parallel_num=*/4, /*batch_size=*/1, + /*prefetch_batch_count=*/8, /*enable_adaptive_prefetch_strategy=*/false, + executor, /*initialize_read_ranges=*/true, + /*read_ahead_cache_enabled=*/read_ahead, CacheConfig(), + /*enable_io_metrics=*/true, WarmupLevel::DECODED, GetDefaultPool(), pool_)); + ASSERT_EQ(arrow::GetCpuThreadPoolCapacity(), 1); + reader->Warmup(); + ASSERT_OK_AND_ASSIGN(auto result, paimon::test::ReadResultCollector::CollectResult( + std::move(reader))); + ASSERT_TRUE(result->Equals(arrow::ChunkedArray(struct_array_))); + ASSERT_EQ(arrow::GetCpuThreadPoolCapacity(), 1); + } + } } - { - std::map options = {{PARQUET_READ_EXECUTOR_THREAD_COUNT, "6"}}; - int32_t batch_size = 1024; - ASSERT_OK_AND_ASSIGN( - auto arrow_reader_properties, - ParquetFileBatchReader::CreateArrowReaderProperties(pool_, options, batch_size, - /*hints=*/std::nullopt)); - ASSERT_EQ(arrow_reader_properties.use_threads(), true); - ASSERT_EQ(arrow::GetCpuThreadPoolCapacity(), 6); +} + +TEST_F(ParquetFileBatchReaderTest, TestCreateArrowReaderProperties) { + const int32_t original_capacity = arrow::GetCpuThreadPoolCapacity(); + ScopeGuard restore_capacity([original_capacity]() { + EXPECT_TRUE(arrow::SetCpuThreadPoolCapacity(original_capacity).ok()); + }); + ASSERT_TRUE(arrow::SetCpuThreadPoolCapacity(8).ok()); + auto expected_cache_options = arrow::io::CacheOptions::Defaults(); + expected_cache_options.hole_size_limit = DEFAULT_PARQUET_READ_CACHE_OPTION_HOLE_SIZE_LIMIT; + const std::vector, bool>> threading_options = { + {{}, true}, + {{{PARQUET_READ_EXECUTOR_THREAD_COUNT, "0"}}, false}, + {{{PARQUET_READ_EXECUTOR_THREAD_COUNT, "1"}}, true}, + {{{PARQUET_READ_EXECUTOR_THREAD_COUNT, "6"}}, true}, + {{{PARQUET_READ_EXECUTOR_THREAD_COUNT, "4294967295"}}, true}, + {{{PARQUET_READ_USE_THREADS, "false"}}, false}, + {{{PARQUET_READ_USE_THREADS, "true"}}, true}, + {{{PARQUET_READ_EXECUTOR_THREAD_COUNT, "0"}, {PARQUET_READ_USE_THREADS, "true"}}, true}, + {{{PARQUET_READ_EXECUTOR_THREAD_COUNT, "6"}, {PARQUET_READ_USE_THREADS, "false"}}, false}}; + for (const auto& [options, use_threads] : threading_options) { + SCOPED_TRACE(::testing::PrintToString(options)); + ASSERT_OK_AND_ASSIGN(auto properties, ParquetFileBatchReader::CreateArrowReaderProperties( + pool_, options, 1024, /*hints=*/std::nullopt)); + ASSERT_TRUE(properties.pre_buffer()); + ASSERT_EQ(properties.batch_size(), 1024); + ASSERT_EQ(properties.use_threads(), use_threads); + ASSERT_EQ(properties.cache_options(), expected_cache_options); + ASSERT_EQ(arrow::GetCpuThreadPoolCapacity(), 8); + } + const std::vector, std::string>> invalid_options = + {{{{PARQUET_READ_USE_THREADS, "invalid"}}, PARQUET_READ_USE_THREADS}, + {{{PARQUET_READ_EXECUTOR_THREAD_COUNT, "0"}, {PARQUET_READ_USE_THREADS, "invalid"}}, + PARQUET_READ_USE_THREADS}, + {{{PARQUET_READ_EXECUTOR_THREAD_COUNT, "-1"}}, PARQUET_READ_EXECUTOR_THREAD_COUNT}, + {{{PARQUET_READ_EXECUTOR_THREAD_COUNT, "4294967296"}}, PARQUET_READ_EXECUTOR_THREAD_COUNT}, + {{{PARQUET_READ_EXECUTOR_THREAD_COUNT, "invalid"}, {PARQUET_READ_USE_THREADS, "true"}}, + PARQUET_READ_EXECUTOR_THREAD_COUNT}}; + for (const auto& [options, invalid_option] : invalid_options) { + SCOPED_TRACE(::testing::PrintToString(options)); + ASSERT_NOK_WITH_MSG(ParquetFileBatchReader::CreateArrowReaderProperties( + pool_, options, 1024, /*hints=*/std::nullopt), + invalid_option); + ASSERT_EQ(arrow::GetCpuThreadPoolCapacity(), 8); } { std::map options = { @@ -1210,6 +1350,7 @@ TEST_F(ParquetFileBatchReaderTest, TestCreateArrowReaderProperties) { "parquet.read.cache-option.range-size-limit must be greater than " "parquet.read.cache-option.hole-size-limit"); } + ASSERT_EQ(arrow::GetCpuThreadPoolCapacity(), 8); } TEST_F(ParquetFileBatchReaderTest, TestPreBufferReadHints) { diff --git a/src/paimon/format/parquet/parquet_format_defs.h b/src/paimon/format/parquet/parquet_format_defs.h index f45603fc5..b9860833c 100644 --- a/src/paimon/format/parquet/parquet_format_defs.h +++ b/src/paimon/format/parquet/parquet_format_defs.h @@ -59,6 +59,10 @@ static inline const char PARQUET_WRITE_ENABLE_PAGE_INDEX[] = "parquet.write.enab static constexpr bool DEFAULT_PARQUET_WRITE_ENABLE_PAGE_INDEX = true; // read +// Enable parallel column reads on the application's Arrow CPU pool (default true). +static inline const char PARQUET_READ_USE_THREADS[] = "parquet.read.use-threads"; +// Deprecated fallback when PARQUET_READ_USE_THREADS is absent: 0 disables column threads, +// positive values enable them. This option no longer changes the global CPU pool capacity. static inline const char PARQUET_READ_EXECUTOR_THREAD_COUNT[] = "parquet.read.executor.thread-count"; static constexpr uint32_t DEFAULT_PARQUET_READ_EXECUTOR_THREAD_COUNT = 3;