diff --git a/benchmark/parquet_format_benchmark.cpp b/benchmark/parquet_format_benchmark.cpp index 40094d31d..cead815d0 100644 --- a/benchmark/parquet_format_benchmark.cpp +++ b/benchmark/parquet_format_benchmark.cpp @@ -574,6 +574,8 @@ Result WriteParquetFile(const std::shared_ptr& fs, const st struct ReadStats { int64_t rows = 0; + double first_batch_ns = 0; + int64_t peak_memory_bytes = 0; // Reader-side counterpart of file size: makes pruning and skipping visible apart from CPU. uint64_t storage_bytes = 0; // Without these a filtered case shows only that it got faster, not that pruning is why. @@ -598,15 +600,18 @@ Result ReadParquetFile(const std::shared_ptr& fs, const s const std::map& options, int32_t batch_size, const std::shared_ptr& pool) { + const auto started = std::chrono::steady_clock::now(); + auto read_pool = std::make_shared(pool.get()); PAIMON_ASSIGN_OR_RAISE(FileStatus file_status, fs->GetFileStatus(path)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr input, fs->Open(path)); - auto in_stream = std::make_shared(input, file_status.GetLen(), pool); + auto in_stream = + std::make_shared(input, file_status.GetLen(), read_pool); // Held separately so the counter outlives the adapter the reader takes ownership of. std::shared_ptr> storage_read_bytes = in_stream->StorageReadBytes(); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr reader, ParquetFileBatchReader::Create(std::move(in_stream), options, batch_size, - /*file_metadata=*/nullptr, storage_read_bytes, pool, + /*file_metadata=*/nullptr, storage_read_bytes, read_pool, /*hints=*/std::nullopt)); // SetReadSchema imports the C schema and takes ownership of it. @@ -620,6 +625,12 @@ Result ReadParquetFile(const std::shared_ptr& fs, const s if (BatchReader::IsEofBatch(batch)) { break; } + if (stats.batches == 0) { + stats.first_batch_ns = + std::chrono::duration(std::chrono::steady_clock::now() - started) + .count(); + } + ++stats.batches; stats.rows += batch.first->length; ArrowArrayRelease(batch.first.get()); ArrowSchemaRelease(batch.second.get()); @@ -636,6 +647,7 @@ Result ReadParquetFile(const std::shared_ptr& fs, const s ReadCounter(metrics, paimon::parquet::ParquetMetrics::READ_BATCH_COUNT)); reader->Close(); stats.storage_bytes = storage_read_bytes->load(); + stats.peak_memory_bytes = read_pool->max_memory(); return stats; } @@ -1201,6 +1213,8 @@ void RunReadBenchmark(::benchmark::State& state, const ReadFixture& fixture, } ReadStats stats; + double first_batch_ns = 0; + int64_t peak_memory_bytes = 0; Timer timer; for (auto _ : state) { Result result = @@ -1210,6 +1224,8 @@ void RunReadBenchmark(::benchmark::State& state, const ReadFixture& fixture, return; } stats = result.value(); + first_batch_ns += stats.first_batch_ns; + peak_memory_bytes = std::max(peak_memory_bytes, stats.peak_memory_bytes); } if (stats.rows < expected.min || stats.rows > expected.max) { FailBenchmark(state, Status::Invalid("read " + std::to_string(stats.rows) + @@ -1239,6 +1255,10 @@ void RunReadBenchmark(::benchmark::State& state, const ReadFixture& fixture, state.counters["row_groups_after_filter"] = ::benchmark::Counter(static_cast(stats.row_groups_after_filter)); state.counters["batches"] = ::benchmark::Counter(static_cast(stats.batches)); + state.counters["first_batch_ns"] = + ::benchmark::Counter(first_batch_ns / static_cast(state.iterations())); + state.counters["peak_memory_bytes"] = + ::benchmark::Counter(static_cast(peak_memory_bytes)); } // Sparse lookup into a large row group of 1 KiB values. Page size controls how much @@ -1344,6 +1364,19 @@ void BM_ParquetRead_Filtered(::benchmark::State& state) { RowExpectation{threshold, row_group_bound}); } +void BM_ParquetRead_FilteredBatchSize(::benchmark::State& state) { + const int64_t threshold = kRowGroupLength / 2; + const bool enable_page_index = state.range(1) != 0; + std::shared_ptr predicate = + PredicateBuilder::LessThan(0, "id", FieldType::BIGINT, Literal(threshold)); + RunReadBenchmark( + state, PlainFlatFixture(), FlatSchema(), predicate, std::nullopt, + {{paimon::parquet::PARQUET_READ_ENABLE_PAGE_INDEX_FILTER, + enable_page_index ? "true" : "false"}}, + static_cast(state.range(0)), + RowExpectation{threshold, enable_page_index ? kRowGroupLength - 1 : kRowGroupLength}); +} + // arg: distance between selected rows. The strides straddle the coalesce hole limit (32 rows by // default), which splits this into two regimes: at or below it neighbouring single-row ranges // merge into long spans, so rows_read runs far ahead of selected_rows and nothing is skipped; @@ -1603,6 +1636,16 @@ BENCHMARK(BM_ParquetRead_Filtered) ->Args({50, 0}) ->Unit(benchmark::kMillisecond) ->UseRealTime(); +BENCHMARK(BM_ParquetRead_FilteredBatchSize) + ->ArgNames({"batch_rows", "page_index"}) + ->Args({64, 1}) + ->Args({64, 0}) + ->Args({1024, 1}) + ->Args({1024, 0}) + ->Args({4096, 1}) + ->Args({4096, 0}) + ->Unit(benchmark::kMillisecond) + ->UseRealTime(); BENCHMARK(BM_ParquetRead_SkipHeavy) ->ArgName("stride") ->Arg(8) diff --git a/docs/source/examples/benchmark.rst b/docs/source/examples/benchmark.rst index a1833d9df..ca8cc9ea9 100644 --- a/docs/source/examples/benchmark.rst +++ b/docs/source/examples/benchmark.rst @@ -165,6 +165,14 @@ visible in both directions. Read cases additionally report ``rows_read``, ``batches``, and ``row_groups`` / ``row_groups_after_filter`` from the reader's own metrics. +Read cases also report ``first_batch_ns``, the mean time from opening the file +through returning the first batch, including metadata, schema setup and +pre-buffering. ``peak_memory_bytes`` is the maximum tracked Arrow allocation +across iterations, including pre-buffered data and decoder state. It excludes +fixture generation and allocations outside the reader's Arrow pool, so it is +not a process-wide memory measurement. Returned batches are released as they +are consumed. + Compare filtered cases on ``ns_per_input_row`` and ``bytes_per_input_row``, not ``ns_per_row`` and ``bytes_per_row``. The latter pair divides by the rows a case actually materialized, so pruning shrinks numerator and denominator together and @@ -208,6 +216,16 @@ Read batch size sweep, repeated for a stable comparison:: --benchmark_repetitions=5 \ --benchmark_report_aggregates_only=true +Page-filtered decoding with 64, 1024 and 4096 rows per batch, paired with +page-index filtering disabled. The predicate partially selects the first row +group; compare ``first_batch_ns`` and ``peak_memory_bytes`` across batch sizes, +and check ``read_bytes`` to distinguish decoding changes from page pruning:: + + paimon-parquet-format-benchmark \ + --benchmark_filter=BM_ParquetRead_FilteredBatchSize \ + --benchmark_repetitions=5 \ + --benchmark_report_aggregates_only=true + Null density on both sides, to see what definition levels cost:: paimon-parquet-format-benchmark --benchmark_filter='Parquet(Write|Read)_Nulls' diff --git a/src/paimon/format/parquet/file_reader_wrapper.cpp b/src/paimon/format/parquet/file_reader_wrapper.cpp index 56128328f..6f575a531 100644 --- a/src/paimon/format/parquet/file_reader_wrapper.cpp +++ b/src/paimon/format/parquet/file_reader_wrapper.cpp @@ -517,6 +517,8 @@ void FileReaderWrapper::DispatchPreBuffer(std::vector<::arrow::io::ReadRange> ra Status FileReaderWrapper::PrepareForReading(const std::vector& target_row_groups, const std::vector& column_indices) { try { + current_page_filtered_reader_.reset(); + filtered_global_offset_ = 0; target_row_groups_ = target_row_groups; target_column_indices_ = column_indices; diff --git a/src/paimon/format/parquet/file_reader_wrapper.h b/src/paimon/format/parquet/file_reader_wrapper.h index e9edca16c..971aac7c0 100644 --- a/src/paimon/format/parquet/file_reader_wrapper.h +++ b/src/paimon/format/parquet/file_reader_wrapper.h @@ -122,10 +122,6 @@ class FileReaderWrapper { /// Prepare for immediate reading of the specified row groups and columns. /// Initializes the reader and starts pre-buffering I/O. - /// - /// Note: when the read schema has nested sub-field projection, - /// page-level filtering is disabled temporarily due to known offset - /// calculation issues for nested pages. Status PrepareForReading(const std::vector& target_row_groups, const std::vector& column_indices); diff --git a/src/paimon/format/parquet/file_reader_wrapper_test.cpp b/src/paimon/format/parquet/file_reader_wrapper_test.cpp index e1ad5ac39..9ef953fa7 100644 --- a/src/paimon/format/parquet/file_reader_wrapper_test.cpp +++ b/src/paimon/format/parquet/file_reader_wrapper_test.cpp @@ -35,6 +35,7 @@ #include "arrow/c/bridge.h" #include "arrow/io/caching.h" #include "arrow/memory_pool.h" +#include "fmt/format.h" #include "gtest/gtest.h" #include "paimon/common/utils/arrow/arrow_input_stream_adapter.h" #include "paimon/common/utils/arrow/mem_utils.h" @@ -517,19 +518,13 @@ TEST_F(FileReaderWrapperTest, PageFilteredZeroBatchSizeDoesNotHang) { ASSERT_GE(batch_count, 1); } -/// SeekToRow back to a previously-consumed page-filtered row group must rebuild the -/// per-RG streaming reader from row_group_row_ranges_ and re-yield the same rows. -/// The page-filter path holds no per-RG cache that consumption could destroy; the -/// reader is constructed on demand each time, mirroring Arrow's stateless -/// GetRecordBatchReader for the fully-matched path. TEST_F(FileReaderWrapperTest, SeekBackToConsumedPageFilteredRowGroup) { std::string file_path = PathUtil::JoinPath(dir_->Str(), "seek_back.parquet"); - // 2000 rows produces 2 row groups (max_row_group_length=1000) with page index enabled. PrepareParquetFile(file_path, /*row_count=*/2000, /*enable_page_index=*/true); - ASSERT_OK_AND_ASSIGN(auto reader_wrapper, PrepareReaderWrapper(file_path)); + constexpr int64_t kBatchSize = 7; + ASSERT_OK_AND_ASSIGN(auto reader_wrapper, PrepareReaderWrapper(file_path, kBatchSize)); ASSERT_EQ(2, reader_wrapper->GetNumberOfRowGroups()); - // Both RGs page-filtered. RowRanges are RG-local: RG0 keeps 40 rows, RG1 keeps 50. std::map row_ranges_map; row_ranges_map[0] = RowRanges(RowRanges::Range(10, 49)); row_ranges_map[1] = RowRanges(RowRanges::Range(100, 149)); @@ -542,13 +537,15 @@ TEST_F(FileReaderWrapperTest, SeekBackToConsumedPageFilteredRowGroup) { /*ranges=*/row_ranges_map[1])}, all_columns)); - auto count_all_rows = [&](int64_t* out_total) { + auto count_all_rows = [&](int64_t* out_total) -> Status { int64_t total = 0; while (true) { - auto next = reader_wrapper->Next(); - if (!next.ok()) return next.status(); - auto batch = std::move(next).value(); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr batch, + reader_wrapper->Next()); if (!batch) break; + EXPECT_TRUE(batch->ValidateFull().ok()); + EXPECT_GT(batch->num_rows(), 0); + EXPECT_LE(batch->num_rows(), kBatchSize); total += batch->num_rows(); } *out_total = total; @@ -557,24 +554,34 @@ TEST_F(FileReaderWrapperTest, SeekBackToConsumedPageFilteredRowGroup) { int64_t first_total = 0; ASSERT_OK(count_all_rows(&first_total)); - ASSERT_EQ(90, first_total); // 40 + 50 + ASSERT_EQ(90, first_total); - // Seek back to row 0 (start of RG0). The on-demand reader construction means RG0 - // is read again from scratch, producing the same 90 rows total. ASSERT_OK(reader_wrapper->SeekToRow(0)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr first_batch, reader_wrapper->Next()); + ASSERT_NE(nullptr, first_batch); + ASSERT_EQ(kBatchSize, first_batch->num_rows()); + ASSERT_EQ("str_10", + checked_cast(*first_batch->column(0)).GetString(0)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr second_batch, reader_wrapper->Next()); + ASSERT_NE(nullptr, second_batch); + ASSERT_EQ("str_17", + checked_cast(*second_batch->column(0)).GetString(0)); + + ASSERT_OK(reader_wrapper->SeekToRow(1000)); + int64_t remaining_total = 0; + ASSERT_OK(count_all_rows(&remaining_total)); + ASSERT_EQ(50, remaining_total); + ASSERT_OK(reader_wrapper->SeekToRow(0)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr restarted, reader_wrapper->Next()); + ASSERT_NE(nullptr, restarted); + ASSERT_TRUE(first_batch->Equals(*restarted)); + ASSERT_OK(reader_wrapper->SeekToRow(0)); int64_t second_total = 0; ASSERT_OK(count_all_rows(&second_total)); ASSERT_EQ(90, second_total); } -/// When the page-level predicate matches more rows than the wrapper's batch_size, -/// the page-filtered streaming path must split the filtered rows across multiple -/// Next() calls. Pages are written 3 rows wide (write_batch_size=3 with -/// data_pagesize=1) so that filtered rows span multiple page-sized chunks; the -/// emitted batches must (a) sum to the RowRanges row count and (b) never exceed -/// the configured batch_size — TableBatchReader additionally caps each batch at -/// the underlying chunk boundary, which is fine as long as the cap holds. TEST_F(FileReaderWrapperTest, PageFilteredRespectsBatchSize) { constexpr int32_t kRowCount = 60; constexpr int32_t kPageRowCount = 3; @@ -584,29 +591,33 @@ TEST_F(FileReaderWrapperTest, PageFilteredRespectsBatchSize) { PrepareParquetFile(file_path, kRowCount, /*enable_page_index=*/true, /*write_batch_size=*/kPageRowCount); - // Keep rows [0, 29] — the first 10 pages of the row group. - RowRanges rr({RowRanges::Range(0, kExpectedTotal - 1)}); + RowRanges row_ranges(RowRanges::Range(0, kExpectedTotal - 1)); for (int64_t batch_size : {int64_t{1}, int64_t{2}, int64_t{3}, int64_t{5}, int64_t{10}}) { SCOPED_TRACE("batch_size=" + std::to_string(batch_size)); - ASSERT_OK_AND_ASSIGN(auto reader_wrapper, PrepareReaderWrapper(file_path, batch_size)); - ASSERT_OK(reader_wrapper->PrepareForReading( - {TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/rr)}, - {0, 1, 2})); - - int64_t total = 0; - int64_t batch_count = 0; - while (true) { - ASSERT_OK_AND_ASSIGN(auto batch, reader_wrapper->Next()); - if (!batch) break; - ASSERT_GT(batch->num_rows(), 0); - ASSERT_LE(batch->num_rows(), batch_size); - total += batch->num_rows(); - ++batch_count; + for (const auto& columns : {std::vector{0, 1, 2}, std::vector{}}) { + SCOPED_TRACE(columns.size()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader_wrapper, + PrepareReaderWrapper(file_path, batch_size)); + ASSERT_OK( + reader_wrapper->PrepareForReading({TargetRowGroup(0, true, row_ranges)}, columns)); + + int64_t total = 0; + while (true) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr batch, + reader_wrapper->Next()); + if (!batch) break; + ASSERT_TRUE(batch->ValidateFull().ok()); + ASSERT_EQ(columns.size(), batch->num_columns()); + ASSERT_GT(batch->num_rows(), 0); + ASSERT_EQ(std::min(batch_size, kExpectedTotal - total), batch->num_rows()); + ASSERT_OK_AND_ASSIGN(uint64_t first_row, + reader_wrapper->GetPreviousBatchFirstRowNumber()); + ASSERT_EQ(total, first_row); + total += batch->num_rows(); + } + ASSERT_EQ(kExpectedTotal, total); } - ASSERT_EQ(kExpectedTotal, total); - const int64_t min_batches = (kExpectedTotal + batch_size - 1) / batch_size; - ASSERT_GE(batch_count, min_batches); } } @@ -721,49 +732,63 @@ TEST_F(FileReaderWrapperTest, ApplyReadRanges) { TEST_F(FileReaderWrapperTest, ApplyReadRangesWiderSecondCall) { std::string file_path = PathUtil::JoinPath(dir_->Str(), "test.parquet"); - PrepareParquetFile(file_path, /*row_count=*/5500); - ASSERT_OK_AND_ASSIGN(auto reader_wrapper, PrepareReaderWrapper(file_path)); - - // Prepare with row groups: {0, 1, 2, 4, 5} - std::vector initial_targets = { - TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/false, - /*ranges=*/RowRanges()), - TargetRowGroup(/*rg_index=*/1, /*is_partially_matched=*/false, - /*ranges=*/RowRanges()), - TargetRowGroup(/*rg_index=*/2, /*is_partially_matched=*/false, - /*ranges=*/RowRanges()), - TargetRowGroup(/*rg_index=*/4, /*is_partially_matched=*/false, - /*ranges=*/RowRanges()), - TargetRowGroup(/*rg_index=*/5, /*is_partially_matched=*/false, - /*ranges=*/RowRanges())}; - std::vector all_columns = {0, 1, 2}; - ASSERT_OK(reader_wrapper->PrepareForReadingLazy(initial_targets, all_columns)); - - // First ApplyReadRanges: narrow to RG 0 only. - ASSERT_OK(reader_wrapper->ApplyReadRanges({{0, 1000}})); - - // Second ApplyReadRanges: widen to RG 0, 1, 2. Previously excluded RG 1, 2 should restore. - ASSERT_OK(reader_wrapper->ApplyReadRanges({{0, 1000}, {1000, 2000}, {2000, 3000}})); + PrepareParquetFile(file_path, 5500, true); + constexpr int64_t kBatchSize = 7; + + for (bool partially_matched : {false, true}) { + SCOPED_TRACE(partially_matched); + ASSERT_OK_AND_ASSIGN(auto reader_wrapper, PrepareReaderWrapper(file_path, kBatchSize)); + std::vector initial_targets; + for (int32_t row_group : {0, 1, 2, 4, 5}) { + initial_targets.emplace_back(row_group, partially_matched, + RowRanges(RowRanges::Range(10, 29))); + } + ASSERT_OK(reader_wrapper->PrepareForReadingLazy(initial_targets, {0, 1, 2})); + ASSERT_OK(reader_wrapper->ApplyReadRanges({{0, 1000}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr first, reader_wrapper->Next()); + ASSERT_NE(nullptr, first); + ASSERT_GT(first->num_rows(), 0); + if (partially_matched) { + ASSERT_EQ(kBatchSize, first->num_rows()); + } - // Verify: reading should produce rows from RG 0 + 1 + 2 = 3000 rows. - int64_t total_rows = 0; - while (true) { - ASSERT_OK_AND_ASSIGN(auto batch, reader_wrapper->Next()); - if (!batch) break; - total_rows += batch->num_rows(); + ASSERT_OK(reader_wrapper->ApplyReadRanges({{0, 1000}, {1000, 2000}, {2000, 3000}})); + int64_t total_rows = 0; + const int64_t rows_per_group = partially_matched ? 20 : 1000; + while (true) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr batch, reader_wrapper->Next()); + if (!batch) { + break; + } + ASSERT_TRUE(batch->ValidateFull().ok()); + ASSERT_GT(batch->num_rows(), 0); + if (partially_matched) { + ASSERT_LE(batch->num_rows(), kBatchSize); + } + const auto& strings = checked_cast(*batch->column(0)); + for (int64_t row = 0; row < batch->num_rows(); ++row) { + const int64_t selected_row = total_rows + row; + const int64_t file_row = selected_row / rows_per_group * 1000 + + selected_row % rows_per_group + + (partially_matched ? 10 : 0); + ASSERT_EQ(fmt::format("str_{}", file_row), strings.GetString(row)); + } + total_rows += batch->num_rows(); + } + ASSERT_EQ(3 * rows_per_group, total_rows); } - ASSERT_EQ(3000, total_rows); } TEST_F(FileReaderWrapperTest, PrepareForReading) { std::string file_path = PathUtil::JoinPath(dir_->Str(), "test.parquet"); - PrepareParquetFile(file_path, /*row_count=*/5500); - ASSERT_OK_AND_ASSIGN(auto reader_wrapper, PrepareReaderWrapper(file_path)); + PrepareParquetFile(file_path, 5500, true); + constexpr int64_t kBatchSize = 7; + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader_wrapper, + PrepareReaderWrapper(file_path, kBatchSize)); ASSERT_OK(reader_wrapper->PrepareForReading( /*target_row_groups=*/{TargetRowGroup(/*rg_index=*/1, /*is_partially_matched=*/false, /*ranges=*/RowRanges())}, /*column_indices=*/{0})); - // seek before actual read range ASSERT_OK(reader_wrapper->SeekToRow(0)); ASSERT_EQ(1000, reader_wrapper->GetNextRowToRead()); ASSERT_EQ(std::numeric_limits::max(), @@ -781,7 +806,6 @@ TEST_F(FileReaderWrapperTest, PrepareForReading) { ASSERT_OK_AND_ASSIGN(record_batch, reader_wrapper->Next()); ASSERT_FALSE(record_batch); - // empty column indices ASSERT_OK(reader_wrapper->PrepareForReading( /*target_row_groups=*/{TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/false, /*ranges=*/RowRanges()), @@ -795,13 +819,51 @@ TEST_F(FileReaderWrapperTest, PrepareForReading) { ASSERT_EQ(512, record_batch->num_rows()); ASSERT_EQ(0, record_batch->num_columns()); - // empty row group indices ASSERT_OK(reader_wrapper->PrepareForReading( /*target_row_groups=*/{}, /*column_indices=*/{0})); ASSERT_EQ(5500, reader_wrapper->GetNextRowToRead()); ASSERT_EQ(std::numeric_limits::max(), reader_wrapper->GetPreviousBatchFirstRowNumber().value()); + + for (bool lazy : {false, true}) { + SCOPED_TRACE(lazy); + ASSERT_OK(reader_wrapper->PrepareForReading( + {TargetRowGroup(1, true, RowRanges(RowRanges::Range(10, 29)))}, {0, 1})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr retained, reader_wrapper->Next()); + ASSERT_NE(nullptr, retained); + ASSERT_EQ(kBatchSize, retained->num_rows()); + ASSERT_EQ(2, retained->num_columns()); + + const std::vector targets = { + TargetRowGroup(2, true, RowRanges(RowRanges::Range(50, 61)))}; + ASSERT_OK(lazy ? reader_wrapper->PrepareForReadingLazy(targets, {2}) + : reader_wrapper->PrepareForReading(targets, {2})); + int64_t total = 0; + while (true) { + ASSERT_OK_AND_ASSIGN(record_batch, reader_wrapper->Next()); + if (!record_batch) break; + ASSERT_TRUE(record_batch->ValidateFull().ok()); + ASSERT_GT(record_batch->num_rows(), 0); + ASSERT_EQ(std::min(kBatchSize, 12 - total), record_batch->num_rows()); + ASSERT_EQ(1, record_batch->num_columns()); + ASSERT_EQ("col3", record_batch->schema()->field(0)->name()); + ASSERT_OK_AND_ASSIGN(uint64_t first_row, + reader_wrapper->GetPreviousBatchFirstRowNumber()); + ASSERT_EQ(2050 + total, first_row); + const auto& values = checked_cast(*record_batch->column(0)); + for (int64_t row = 0; row < record_batch->num_rows(); ++row) { + ASSERT_EQ(static_cast((2050 + total + row) % 2), values.Value(row)); + } + total += record_batch->num_rows(); + } + ASSERT_EQ(12, total); + ASSERT_TRUE(retained->ValidateFull().ok()); + const auto& strings = checked_cast(*retained->column(0)); + for (int64_t row = 0; row < retained->num_rows(); ++row) { + ASSERT_EQ(fmt::format("str_{}", 1010 + row), strings.GetString(row)); + } + } } namespace { diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp index b712e54d6..8c8fcf2e7 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp @@ -20,9 +20,11 @@ #include "paimon/format/parquet/page_filtered_row_group_reader.h" #include +#include #include #include #include +#include #include "arrow/array.h" #include "arrow/builder.h" @@ -35,6 +37,7 @@ #include "paimon/common/utils/arrow/arrow_utils.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/saturating_cast.h" +#include "paimon/format/parquet/parquet_format_defs.h" #include "parquet/arrow/reader.h" #include "parquet/arrow/reader_internal.h" #include "parquet/arrow/schema.h" @@ -45,14 +48,9 @@ namespace paimon::parquet { namespace { -/// Multiple of a column chunk's compressed size that a leaf may reserve up front in value bytes. -/// The estimate is a heuristic over footer fields, and `total_uncompressed_size` is a bare claim a -/// forged footer can inflate to make a leaf eagerly allocate gigabytes for a near-empty file. -/// `total_compressed_size` is instead checked against the file length before any read (Arrow's -/// ComputeColumnChunkRange, and GetDataPageLayout below, bound a chunk and its pages by it), so a -/// factor of it is bounded by bytes that physically exist. A leaf decompressing within the factor -/// is reserved in full; a more compressible one is capped and left to the builder's amortized -/// doubling, which is cheap against the I/O and decode of a read that large. +/// Cap each window's value-byte reservation using the file-bounded compressed chunk size. +/// The uncompressed footer size may be forged; builders can grow beyond this estimate as +/// values are decoded. constexpr int64_t kMaxReservationDecompressionFactor = 8; struct DataPageLayout { @@ -105,9 +103,7 @@ std::optional GetDataPageLayout( return DataPageLayout{column_chunk_offset, first_data_page_offset}; } -/// Wraps an arrow::Table + TableBatchReader as a RecordBatchReader so the caller can -/// stream batches while ensuring every returned array offset is zero. The Table is held -/// to keep its ChunkedArrays alive for the inner TableBatchReader. +/// Aligns a decoding window's Arrow chunks and normalizes array offsets. class TableRecordBatchReader : public arrow::RecordBatchReader { public: TableRecordBatchReader(std::shared_ptr table, int64_t chunksize, @@ -211,7 +207,7 @@ PageFilteredRowGroupReader::MakeDataPageReadPlan( std::move(data_pages)}; } -std::pair PageFilteredRowGroupReader::ComputeCompressedRowRanges( +RowRanges PageFilteredRowGroupReader::ComputeCompressedRowRanges( const RowRanges& original_ranges, const std::shared_ptr<::parquet::OffsetIndex>& offset_index, int64_t row_group_row_count) { const auto& page_locations = offset_index->page_locations(); @@ -242,33 +238,7 @@ std::pair PageFilteredRowGroupReader::ComputeCompressedRowRa compressed_offset += page_size; } - return {compressed, compressed_offset}; -} - -Status PageFilteredRowGroupReader::ExecuteSkipReadPattern( - int col_idx, const RowRanges& ranges, int64_t total, int64_t reserve_values, - int64_t reserve_value_bytes, ::parquet::arrow::ColumnReader* column_reader) { - PAIMON_RETURN_NOT_OK_FROM_ARROW( - column_reader->ResetLeaf(col_idx, total, reserve_values, reserve_value_bytes)); - int64_t current = 0; - for (const auto& range : ranges.GetRanges()) { - int64_t skip = range.from > current ? range.from - current : 0; - int64_t skipped = column_reader->SkipRecords(col_idx, skip); - if (skipped != skip) { - return Status::Invalid(fmt::format( - "PageFilteredRowGroupReader: leaf {} expected to skip {} records but skipped {}", - col_idx, skip, skipped)); - } - int64_t to_read = range.Count(); - int64_t read = column_reader->ReadRecords(col_idx, to_read); - if (read != to_read) { - return Status::Invalid(fmt::format( - "PageFilteredRowGroupReader: leaf {} expected to read {} records but read {}", - col_idx, to_read, read)); - } - current = range.to + 1; - } - return Status::OK(); + return compressed; } Status PageFilteredRowGroupReader::WaitForPreBuffer( @@ -295,16 +265,209 @@ Status PageFilteredRowGroupReader::WaitForPreBuffer( return Status::OK(); } -Result> PageFilteredRowGroupReader::ReadFilteredField( +/// Retains the decoder and selected-row cursor of every leaf, including leaves whose page +/// plans use different compressed row coordinates. Only the current output window is decoded. +class PageFilteredRowGroupReader::FilteredRecordBatchReader : public arrow::RecordBatchReader { + public: + static Result> Create( + const RowRanges& row_ranges, int32_t row_group_index, + const std::vector& column_indices, int64_t batch_size, + const std::shared_ptr<::parquet::RowGroupPageIndexReader>& page_index_reader, + const std::shared_ptr& pool, + ::parquet::arrow::FileReader* arrow_file_reader) { + const auto& manifest = arrow_file_reader->manifest(); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::vector field_indices, + manifest.GetFieldIndices(std::vector( + column_indices.begin(), column_indices.end()))); + const int64_t row_group_row_count = + arrow_file_reader->parquet_reader()->metadata()->RowGroup(row_group_index)->num_rows(); + auto included_columns = + std::make_shared>(column_indices.begin(), column_indices.end()); + std::vector readers; + arrow::FieldVector fields; + readers.reserve(field_indices.size()); + fields.reserve(field_indices.size()); + for (int32_t field_index : field_indices) { + PAIMON_ASSIGN_OR_RAISE( + FieldReader field_reader, + CreateFieldReader(page_index_reader, row_group_index, field_index, included_columns, + row_ranges, row_group_row_count, arrow_file_reader)); + fields.push_back(field_reader.field); + readers.push_back(std::move(field_reader)); + } + return std::unique_ptr( + new FilteredRecordBatchReader(std::move(readers), arrow::schema(std::move(fields)), + row_ranges.RowCount(), batch_size, pool)); + } + + std::shared_ptr schema() const override { + return schema_; + } + + arrow::Status ReadNext(std::shared_ptr* out) override { + *out = nullptr; + if (!status_.ok()) { + return status_; + } + Result> result = ReadNextBatch(); + if (!result.ok()) { + status_ = ToArrowStatus(result.status()); + return status_; + } + *out = std::move(result).value(); + return arrow::Status::OK(); + } + + arrow::Status Close() override { + chunk_reader_.reset(); + readers_.clear(); + remaining_rows_ = 0; + return arrow::Status::OK(); + } + + private: + struct LeafReadState { + int32_t column_index; + RowRanges ranges; + size_t range_index = 0; + int64_t next_row = 0; + double bytes_per_value = 0; + int64_t max_reservation_bytes = 0; + }; + + struct FieldReader { + std::shared_ptr field; + std::unique_ptr<::parquet::arrow::ColumnReader> reader; + std::vector leaves; + }; + + static Result CreateFieldReader( + const std::shared_ptr<::parquet::RowGroupPageIndexReader>& rg_page_index_reader, + int32_t row_group_index, int32_t field_index, + std::shared_ptr> column_indices, const RowRanges& row_ranges, + int64_t row_group_row_count, ::parquet::arrow::FileReader* arrow_file_reader); + + FilteredRecordBatchReader(std::vector&& readers, + std::shared_ptr schema, int64_t remaining_rows, + int64_t batch_size, std::shared_ptr pool) + : readers_(std::move(readers)), + schema_(std::move(schema)), + remaining_rows_(remaining_rows), + batch_size_(batch_size), + pool_(std::move(pool)) {} + + static Status ReadLeaf(int64_t rows, LeafReadState* leaf, + ::parquet::arrow::ColumnReader* reader) { + const int64_t value_bytes = std::min( + SaturatingDoubleToInteger(leaf->bytes_per_value * static_cast(rows)), + leaf->max_reservation_bytes); + // Reset output buffers without discarding the decoder's unread page levels. + PAIMON_RETURN_NOT_OK_FROM_ARROW( + reader->ResetLeaf(leaf->column_index, rows, rows, value_bytes)); + int64_t remaining = rows; + const auto& ranges = leaf->ranges.GetRanges(); + while (remaining > 0) { + if (leaf->range_index >= ranges.size()) { + return Status::Invalid("PageFilteredRowGroupReader: selected leaf rows exhausted"); + } + const auto& range = ranges[leaf->range_index]; + const int64_t skip = std::max(0, range.from - leaf->next_row); + const int64_t skipped = reader->SkipRecords(leaf->column_index, skip); + if (skipped != skip) { + return Status::Invalid( + fmt::format("PageFilteredRowGroupReader: leaf {} expected to skip {} records " + "but skipped {}", + leaf->column_index, skip, skipped)); + } + leaf->next_row += skipped; + const int64_t to_read = std::min(remaining, range.to - leaf->next_row + 1); + const int64_t read = reader->ReadRecords(leaf->column_index, to_read); + if (read != to_read) { + return Status::Invalid(fmt::format( + "PageFilteredRowGroupReader: leaf {} expected to read {} records but read {}", + leaf->column_index, to_read, read)); + } + leaf->next_row += read; + remaining -= read; + if (leaf->next_row > range.to) { + ++leaf->range_index; + } + } + return Status::OK(); + } + + Result> ReadNextBatch() { + try { + while (true) { + if (chunk_reader_) { + std::shared_ptr batch; + PAIMON_RETURN_NOT_OK_FROM_ARROW(chunk_reader_->ReadNext(&batch)); + if (batch) { + return batch; + } + chunk_reader_.reset(); + } + if (remaining_rows_ == 0) { + readers_.clear(); + return std::shared_ptr(); + } + const int64_t rows = std::min(remaining_rows_, batch_size_); + arrow::ChunkedArrayVector columns; + arrow::ArrayVector arrays; + columns.reserve(readers_.size()); + arrays.reserve(readers_.size()); + bool single_chunk = true; + for (auto& field : readers_) { + for (auto& leaf : field.leaves) { + PAIMON_RETURN_NOT_OK(ReadLeaf(rows, &leaf, field.reader.get())); + } + std::shared_ptr column; + PAIMON_RETURN_NOT_OK_FROM_ARROW(field.reader->BuildArray(rows, &column)); + if (column->length() != rows) { + return Status::Invalid(fmt::format( + "PageFilteredRowGroupReader: field {} produced {} rows but expected {}", + field.field->name(), column->length(), rows)); + } + single_chunk = single_chunk && column->num_chunks() == 1; + if (column->num_chunks() == 1) { + arrays.push_back(column->chunk(0)); + } + columns.push_back(std::move(column)); + } + remaining_rows_ -= rows; + if (single_chunk) { + auto batch = arrow::RecordBatch::Make(schema_, rows, std::move(arrays)); + return ArrowUtils::NormalizeRecordBatchOffsets(batch, pool_.get()); + } + // Binary values can exceed Arrow's array limit within a single output window. + auto table = arrow::Table::Make(schema_, std::move(columns), rows); + chunk_reader_ = + std::make_unique(std::move(table), batch_size_, pool_); + } + } + PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("PageFilteredRowGroupReader::ReadNext") + } + + std::vector readers_; + std::shared_ptr schema_; + int64_t remaining_rows_; + int64_t batch_size_; + std::shared_ptr pool_; + std::unique_ptr chunk_reader_; + arrow::Status status_; +}; + +Result +PageFilteredRowGroupReader::FilteredRecordBatchReader::CreateFieldReader( const std::shared_ptr<::parquet::RowGroupPageIndexReader>& rg_page_index_reader, int32_t row_group_index, int32_t field_index, std::shared_ptr> column_indices, const RowRanges& row_ranges, int64_t row_group_row_count, ::parquet::arrow::FileReader* arrow_file_reader) { // Factory: set a direct data page read plan on every leaf (per-leaf OffsetIndex). // The plan lets Arrow jump over unselected page headers as well as page bodies. - std::unordered_set direct_read_columns; - auto factory = [row_group_index, &rg_page_index_reader, &row_ranges, row_group_row_count, - &direct_read_columns](int col_idx, ::parquet::ParquetFileReader* reader) + auto direct_read_columns = std::make_shared>(); + auto factory = [row_group_index, rg_page_index_reader, row_ranges, row_group_row_count, + direct_read_columns](int col_idx, ::parquet::ParquetFileReader* reader) -> ::parquet::arrow::FileColumnIterator* { bool has_data_page_read_plan = false; int64_t first_data_page_offset = 0; @@ -320,14 +483,12 @@ Result> PageFilteredRowGroupReader::ReadFil first_data_page_offset = plan->first_data_page_offset; data_pages = std::move(plan->data_pages); has_data_page_read_plan = true; - direct_read_columns.insert(col_idx); + direct_read_columns->insert(col_idx); } } } - // An empty selection still needs the projected Arrow type, especially for partial nested - // projection, but must not construct a PageReader: without a range cache Arrow eagerly - // reads the whole column chunk in GetColumnPageReader(). An iterator with no row groups - // builds the same reader/type tree and immediately reaches EOF without file I/O. + // Empty selections need the projected type, but creating a PageReader would read the + // column chunk. An iterator without row groups builds the reader tree without I/O. std::vector row_groups; if (!row_ranges.IsEmpty()) { row_groups.push_back(row_group_index); @@ -337,7 +498,6 @@ Result> PageFilteredRowGroupReader::ReadFil std::move(data_pages)); }; - // Build reader tree with leaf column filtering std::unique_ptr<::parquet::arrow::ColumnReader> column_reader; PAIMON_RETURN_NOT_OK_FROM_ARROW( arrow_file_reader->GetColumn(field_index, column_indices, factory, &column_reader)); @@ -349,68 +509,47 @@ Result> PageFilteredRowGroupReader::ReadFil field_index, row_group_index)); } - // Since leaf columns may have misaligned pages, we compute compressed row ranges and drive each - // leaf column independently - - for (int col_idx : column_reader->LeafColumnIndices()) { - RowRanges effective_ranges = row_ranges; - int64_t effective_total = row_ranges.IsEmpty() ? 0 : row_group_row_count; - std::unique_ptr<::parquet::ColumnChunkMetaData> column_chunk; + FieldReader field; + const std::vector leaf_indices = column_reader->LeafColumnIndices(); + field.leaves.reserve(leaf_indices.size()); + for (int32_t col_idx : leaf_indices) { + LeafReadState leaf; + leaf.column_index = col_idx; + leaf.ranges = row_ranges; if (!row_ranges.IsEmpty()) { auto row_group_metadata = arrow_file_reader->parquet_reader()->metadata()->RowGroup(row_group_index); - column_chunk = row_group_metadata->ColumnChunk(col_idx); - if (direct_read_columns.count(col_idx)) { + auto column_chunk = row_group_metadata->ColumnChunk(col_idx); + if (direct_read_columns->count(col_idx)) { auto offset_index = rg_page_index_reader->GetOffsetIndex(col_idx); - auto [compressed, total] = + leaf.ranges = ComputeCompressedRowRanges(row_ranges, offset_index, row_group_row_count); - effective_ranges = std::move(compressed); - effective_total = total; + } + const int64_t chunk_bytes = column_chunk->total_uncompressed_size(); + const int64_t compressed_bytes = column_chunk->total_compressed_size(); + const int64_t chunk_values = column_chunk->num_values(); + if (chunk_bytes > 0 && compressed_bytes > 0 && chunk_values > 0) { + leaf.bytes_per_value = + static_cast(chunk_bytes) / static_cast(chunk_values); + leaf.max_reservation_bytes = std::min( + chunk_bytes, + SaturatingDoubleToInteger(static_cast(compressed_bytes) * + kMaxReservationDecompressionFactor)); } } - - // Rows that will actually be appended, as opposed to effective_total, which is the - // compressed space SkipRecords walks: a highly selective predicate reads a handful of - // rows out of a row group, and reserving for the whole group would allocate tens of MiB - // of offsets nobody writes. - const int64_t reserve_values = effective_ranges.RowCount(); - int64_t reserve_value_bytes = 0; - const int64_t chunk_bytes = column_chunk ? column_chunk->total_uncompressed_size() : 0; - const int64_t chunk_compressed_bytes = - column_chunk ? column_chunk->total_compressed_size() : 0; - const int64_t chunk_values = column_chunk ? column_chunk->num_values() : 0; - if (chunk_bytes > 0 && chunk_values > 0 && reserve_values > 0) { - // A heuristic, not a bound. total_uncompressed_size is uncompressed but still - // ENCODED: a dictionary page stores each value once and its data pages only - // indices, DELTA_BYTE_ARRAY only prefix deltas, so both can decode into more Arrow - // payload than they occupy, while page headers, levels and BYTE_ARRAY length - // prefixes pull the other way. The average is also taken over the whole chunk, - // including the pages this selection skips, so it misleads when wide values sit in - // skipped pages. Either direction only costs performance — the reservation is a - // hint the builder grows past when short — but they are why the result is capped - // instead of trusted, by a factor of the file-bounded compressed size rather than the - // forgeable uncompressed one (see kMaxReservationDecompressionFactor). Fixed-width - // leaves ignore the byte count entirely. - const double avg = static_cast(chunk_bytes) / static_cast(chunk_values); - reserve_value_bytes = std::min( - {SaturatingDoubleToInteger(avg * static_cast(reserve_values)), - chunk_bytes, - SaturatingDoubleToInteger( - static_cast(chunk_compressed_bytes) * - static_cast(kMaxReservationDecompressionFactor))}); - } - - PAIMON_RETURN_NOT_OK(ExecuteSkipReadPattern(col_idx, effective_ranges, effective_total, - reserve_values, reserve_value_bytes, - column_reader.get())); + field.leaves.push_back(std::move(leaf)); } - - // Build the Arrow array (TransferColumnData for leaves + assemble for nested) - std::shared_ptr chunked_array; - PAIMON_RETURN_NOT_OK_FROM_ARROW( - column_reader->BuildArray(row_ranges.RowCount(), &chunked_array)); - - return chunked_array; + // Arrow's projected field can differ from the manifest (e.g. a pruned struct or restored + // dictionary). Assemble zero rows to obtain that type without decoding any data pages. + for (const auto& leaf : field.leaves) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(column_reader->ResetLeaf(leaf.column_index, 0, 0, 0)); + } + std::shared_ptr empty; + PAIMON_RETURN_NOT_OK_FROM_ARROW(column_reader->BuildArray(0, &empty)); + field.field = + arrow_file_reader->manifest().schema_fields[field_index].field->WithType(empty->type()); + field.reader = std::move(column_reader); + return field; } Result> PageFilteredRowGroupReader::ReadFilteredRowGroup( @@ -419,61 +558,19 @@ Result> PageFilteredRowGroupReader::Re const std::vector<::arrow::io::ReadRange>& page_ranges, int64_t max_chunksize, const std::shared_ptr<::parquet::RowGroupPageIndexReader>& row_group_page_index_reader, std::shared_ptr<::arrow::MemoryPool> pool, ::parquet::arrow::FileReader* arrow_file_reader) { - auto parquet_reader = arrow_file_reader->parquet_reader(); + if (max_chunksize <= 0) { + return Status::Invalid("PageFilteredRowGroupReader: batch size must be positive"); + } const auto& row_ranges = target_row_group.GetRowRanges(); - int32_t row_group_index = target_row_group.GetRowGroupIndex(); - - int64_t expected_rows = row_ranges.RowCount(); - + const int32_t row_group_index = target_row_group.GetRowGroupIndex(); if (!row_ranges.IsEmpty()) { PAIMON_RETURN_NOT_OK(WaitForPreBuffer(row_group_index, column_indices, cache_options, - pre_buffered, page_ranges, pool, parquet_reader)); - } - - auto rg_metadata = parquet_reader->metadata()->RowGroup(row_group_index); - int64_t row_group_row_count = rg_metadata->num_rows(); - - const auto& manifest = arrow_file_reader->manifest(); - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( - std::vector field_indices, - manifest.GetFieldIndices(std::vector(column_indices.begin(), column_indices.end()))); - - std::vector> result_arrays; - result_arrays.reserve(field_indices.size()); - - std::shared_ptr> col_indices_set = - std::make_shared>(column_indices.begin(), column_indices.end()); - // TODO(zhouhongfeng.zhf): This loop could be parallelized. - for (int field_idx : field_indices) { - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr chunked_array, - ReadFilteredField(row_group_page_index_reader, row_group_index, field_idx, - col_indices_set, row_ranges, row_group_row_count, arrow_file_reader)); - - if (chunked_array->length() != expected_rows) { - return Status::Invalid( - fmt::format("PageFilteredRowGroupReader: field {} produced {} rows but expected {} " - "(row_group={})", - field_idx, chunked_array->length(), expected_rows, row_group_index)); - } - - result_arrays.push_back(std::move(chunked_array)); - } - - std::vector> result_fields; - for (size_t i = 0; i < result_arrays.size(); ++i) { - const auto& field = manifest.schema_fields[field_indices[i]].field; - result_fields.push_back(arrow::field(field->name(), result_arrays[i]->type(), - field->nullable(), field->metadata())); + pre_buffered, page_ranges, pool, + arrow_file_reader->parquet_reader())); } - auto result_schema = arrow::schema(result_fields); - // TODO(zhouhongfeng.zhf): This decodes the whole filtered row group up front, while the - // fully-matched path decodes one batch at a time. As a result peak memory holds every - // projected column of the row group instead of a single batch. - // Decoding batch by batch would make every returned column single-chunk so that offset - // normalization becomes a no-op. - auto table = arrow::Table::Make(result_schema, std::move(result_arrays), expected_rows); - return std::make_unique(std::move(table), max_chunksize, pool); + return FilteredRecordBatchReader::Create(row_ranges, row_group_index, column_indices, + max_chunksize, row_group_page_index_reader, pool, + arrow_file_reader); } std::vector<::arrow::io::ReadRange> PageFilteredRowGroupReader::ComputePageRanges( diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.h b/src/paimon/format/parquet/page_filtered_row_group_reader.h index 0633473c3..1d6a998ca 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.h +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.h @@ -20,10 +20,8 @@ #pragma once #include -#include #include #include -#include #include #include @@ -56,10 +54,10 @@ class PageFilteredRowGroupReader { /// @param pre_buffered If true, assumes PreBuffer was already called externally /// and only waits via WhenBuffered (no redundant PreBuffer). /// @param page_ranges If non-empty, wait via WhenBufferedRanges instead of WhenBuffered - /// @param max_chunksize Per-batch row cap for the returned reader. + /// @param max_chunksize Positive per-batch row cap for the returned reader. /// @param row_group_page_index_reader Reusable page-index reader for the target row group /// @param pool Memory pool - /// @param arrow_file_reader The Arrow FileReader for ColumnReader tree creation + /// @param arrow_file_reader Arrow FileReader, which must outlive the returned reader /// @return A RecordBatchReader streaming the filtered rows. static Result> ReadFilteredRowGroup( const TargetRowGroup& target_row_group, const std::vector& column_indices, @@ -78,6 +76,8 @@ class PageFilteredRowGroupReader { ::parquet::ParquetFileReader* parquet_reader); private: + class FilteredRecordBatchReader; + struct DataPageReadPlan { int64_t first_data_page_offset; std::vector<::parquet::DataPageReadPlanEntry> data_pages; @@ -105,31 +105,9 @@ class PageFilteredRowGroupReader { const ::parquet::ColumnChunkMetaData& column_chunk, int64_t row_group_row_count); /// Compute compressed RowRanges after the direct read plan skips non-matching pages. - static std::pair ComputeCompressedRowRanges( + static RowRanges ComputeCompressedRowRanges( const RowRanges& original_ranges, const std::shared_ptr<::parquet::OffsetIndex>& offset_index, int64_t row_group_row_count); - - /// Reset the given leaf and replay the skip/read pattern derived from `ranges` - /// directly against the ColumnReader (ResetLeaf + SkipRecords/ReadRecords). - /// `total` is the leaf's compressed-space size, which bounds the levels - /// SkipRecords walks; `reserve_values` / `reserve_value_bytes` describe what the - /// pattern will actually append, so a variable-width leaf can size its Arrow - /// builder once instead of doubling it per decoded batch. - static Status ExecuteSkipReadPattern(int col_idx, const RowRanges& ranges, int64_t total, - int64_t reserve_values, int64_t reserve_value_bytes, - ::parquet::arrow::ColumnReader* column_reader); - - /// Read a field (flat or nested) using ColumnReader tree. - /// Sets a direct page read plan on all leaves via factory, then drives each leaf - /// independently via ResetLeaf/SkipRecords/ReadRecords using its own - /// compressed_ranges. - /// `column_indices` holds `int` rather than `int32_t` because the set is - /// handed straight to Arrow's `FileReader::GetColumn` (to avoid reconstruction and deep copy) - static Result> ReadFilteredField( - const std::shared_ptr<::parquet::RowGroupPageIndexReader>& rg_page_index_reader, - int32_t row_group_index, int32_t field_index, - std::shared_ptr> column_indices, const RowRanges& row_ranges, - int64_t row_group_row_count, ::parquet::arrow::FileReader* arrow_file_reader); }; } // namespace paimon::parquet diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp index cab1c1442..a26e27b9f 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include @@ -39,6 +40,7 @@ #include "arrow/c/bridge.h" #include "arrow/io/api.h" #include "arrow/ipc/json_simple.h" +#include "gmock/gmock.h" #include "gtest/gtest.h" #include "paimon/common/utils/arrow/arrow_input_stream_adapter.h" #include "paimon/common/utils/arrow/mem_utils.h" @@ -84,19 +86,120 @@ class SyntheticOffsetIndex : public ::parquet::OffsetIndex { class SyntheticPageIndexReader : public ::parquet::RowGroupPageIndexReader { public: - explicit SyntheticPageIndexReader(std::shared_ptr<::parquet::OffsetIndex> index) - : index_(std::move(index)) {} + explicit SyntheticPageIndexReader(std::vector> indexes) + : indexes_(std::move(indexes)) {} std::shared_ptr<::parquet::ColumnIndex> GetColumnIndex(int32_t) override { return nullptr; } - std::shared_ptr<::parquet::OffsetIndex> GetOffsetIndex(int32_t) override { - return index_; + std::shared_ptr<::parquet::OffsetIndex> GetOffsetIndex(int32_t column_index) override { + return indexes_.at(column_index); } private: - std::shared_ptr<::parquet::OffsetIndex> index_; + std::vector> indexes_; +}; + +class MockArrowFileReader : public ::parquet::arrow::FileReader { + public: + MOCK_METHOD(arrow::Status, GetColumn, + (int32_t, std::unique_ptr<::parquet::arrow::ColumnReader>*), (override)); + MOCK_METHOD(arrow::Status, GetColumn, + (int32_t, const std::shared_ptr>&, + ::parquet::arrow::FileColumnIteratorFactory, + std::unique_ptr<::parquet::arrow::ColumnReader>*), + (override)); + MOCK_METHOD(arrow::Status, GetSchema, (std::shared_ptr*), (override)); + MOCK_METHOD(arrow::Status, ReadColumn, (int32_t, std::shared_ptr*), + (override)); + MOCK_METHOD(arrow::Status, GetRecordBatchReader, (std::unique_ptr*), + (override)); + MOCK_METHOD(arrow::Status, GetRecordBatchReader, + (const std::vector&, std::unique_ptr*), + (override)); + MOCK_METHOD(arrow::Status, GetRecordBatchReader, + (const std::vector&, const std::vector&, + std::unique_ptr*), + (override)); + MOCK_METHOD(arrow::Result>()>>, + GetRecordBatchGenerator, + (std::shared_ptr<::parquet::arrow::FileReader>, const std::vector, + const std::vector, arrow::internal::Executor*, int64_t), + (override)); + MOCK_METHOD(arrow::Status, ReadTable, (std::shared_ptr*), (override)); + MOCK_METHOD(arrow::Status, ReadTable, + (const std::vector&, std::shared_ptr*), (override)); + MOCK_METHOD(arrow::Status, ReadRowGroup, + (int32_t, const std::vector&, std::shared_ptr*), (override)); + MOCK_METHOD(arrow::Status, ReadRowGroup, (int32_t, std::shared_ptr*), (override)); + MOCK_METHOD(arrow::Status, ReadRowGroups, + (const std::vector&, const std::vector&, + std::shared_ptr*), + (override)); + MOCK_METHOD(arrow::Status, ReadRowGroups, + (const std::vector&, std::shared_ptr*), (override)); + MOCK_METHOD(arrow::Status, ScanContents, (std::vector, int32_t, int64_t*), (override)); + MOCK_METHOD(std::shared_ptr<::parquet::arrow::RowGroupReader>, RowGroup, (int32_t), (override)); + MOCK_METHOD(int32_t, num_row_groups, (), (const, override)); + MOCK_METHOD(::parquet::ParquetFileReader*, parquet_reader, (), (const, override)); + MOCK_METHOD(void, set_use_threads, (bool), (override)); + MOCK_METHOD(void, set_batch_size, (int64_t), (override)); + MOCK_METHOD(const ::parquet::ArrowReaderProperties&, properties, (), (const, override)); + MOCK_METHOD(const ::parquet::arrow::SchemaManifest&, manifest, (), (const, override)); +}; + +// Keep real page filtering and decoding, but force small output chunks instead of requiring +// enough binary data to exceed Arrow's array size limit. +class RechunkingColumnReader : public ::parquet::arrow::ColumnReader { + public: + RechunkingColumnReader(std::unique_ptr<::parquet::arrow::ColumnReader> reader, + int64_t chunk_size, std::vector* decoded_windows) + : reader_(std::move(reader)), chunk_size_(chunk_size), decoded_windows_(decoded_windows) {} + + arrow::Status NextBatch(int64_t batch_size, + std::shared_ptr* out) override { + return reader_->NextBatch(batch_size, out); + } + + std::vector LeafColumnIndices() const override { + return reader_->LeafColumnIndices(); + } + + arrow::Status ResetLeaf(int32_t column_index, int64_t records, int64_t values, + int64_t value_bytes) override { + return reader_->ResetLeaf(column_index, records, values, value_bytes); + } + + int64_t SkipRecords(int32_t column_index, int64_t rows) override { + return reader_->SkipRecords(column_index, rows); + } + + int64_t ReadRecords(int32_t column_index, int64_t rows) override { + return reader_->ReadRecords(column_index, rows); + } + + arrow::Status BuildArray(int64_t rows, std::shared_ptr* out) override { + ARROW_RETURN_NOT_OK(reader_->BuildArray(rows, out)); + if (rows == 0) { + return arrow::Status::OK(); + } + decoded_windows_->push_back((*out)->length()); + arrow::ArrayVector chunks; + for (const auto& chunk : (*out)->chunks()) { + for (int64_t offset = 0; offset < chunk->length(); offset += chunk_size_) { + chunks.push_back( + chunk->Slice(offset, std::min(chunk_size_, chunk->length() - offset))); + } + } + *out = std::make_shared(std::move(chunks), (*out)->type()); + return arrow::Status::OK(); + } + + private: + std::unique_ptr<::parquet::arrow::ColumnReader> reader_; + int64_t chunk_size_; + std::vector* decoded_windows_; }; } // namespace @@ -852,80 +955,152 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesMultiplePages) { ASSERT_LT(ranges[0].offset, ranges[1].offset); } -/// Test: variable-length columns are streamed across multiple offset-normalized -/// RecordBatches when batch_size is smaller than the matched row count, instead of -/// being concatenated into a single RecordBatch via CombineChunks. -/// -/// This verifies the alignment with Arrow's standard TableBatchReader path: -/// multi-chunk binary/string columns split along chunk + batch_size boundaries. It -/// asserts correctness and the multi-batch shape. -TEST_F(PageFilteredRowGroupReaderTest, StringColumnMultiBatchStreaming) { - std::string file_name = dir_->Str() + "/string_multi_batch.parquet"; +TEST_F(PageFilteredRowGroupReaderTest, MultiColumnMultiBatchStreaming) { + std::string file_name = dir_->Str() + "/multi_column_multi_batch.parquet"; arrow::StringBuilder str_builder; ASSERT_TRUE(str_builder.Reserve(60).ok()); - // 6 pages of 10 rows each: prefix "p0_".."p5_" so each page has a distinct min/max. for (int32_t i = 0; i < 60; ++i) { std::string val = "p" + std::to_string(i / 10) + "_" + (i < 10 ? "0" : "") + std::to_string(i); ASSERT_TRUE(str_builder.Append(val).ok()); } auto str_array = str_builder.Finish().ValueOrDie(); - auto field = arrow::field("val", arrow::utf8()); - auto struct_arr = arrow::StructArray::Make({str_array}, {field}).ValueOrDie(); + arrow::FieldVector fields = {arrow::field("id", arrow::int32()), + arrow::field("val", arrow::utf8())}; + auto data = arrow::StructArray::Make({MakeSequentialIntData(60)->field(0), str_array}, fields) + .ValueOrDie(); - WriteTestFile(file_name, struct_arr, /*write_batch_size=*/10, /*max_row_group_length=*/60); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/60); - // Predicate matches pages 2..5 (40 rows: "p2_20".."p5_59"). batch_size=7 forces - // the wrapper to surface multiple batches per page-filtered RG. - auto read_schema = arrow::schema({field}); + auto read_schema = arrow::schema(fields); auto predicate = PredicateBuilder::GreaterOrEqual( - /*field_index=*/0, /*field_name=*/"val", FieldType::STRING, + /*field_index=*/1, /*field_name=*/"val", FieldType::STRING, Literal(FieldType::STRING, "p2", 2)); std::shared_ptr result; ReadWithPredicateImpl(file_name, read_schema, predicate, &result, /*batch_size=*/7); ASSERT_TRUE(result); ASSERT_EQ(40, result->length()); + ASSERT_EQ(6, result->num_chunks()); - // Multi-batch shape: with 40 matched rows and batch_size=7 we expect at least - // ceil(40/7)=6 chunks. Anything > 1 already proves we did not collapse to a single - // post-CombineChunks RecordBatch. - ASSERT_GT(result->num_chunks(), 1); - - // Content correctness: rows arrive in the original page order, "p2_20" through "p5_59". - int64_t seen = 0; - for (int i = 0; i < result->num_chunks(); ++i) { - auto struct_chunk = std::dynamic_pointer_cast(result->chunk(i)); - ASSERT_TRUE(struct_chunk); - auto str_chunk = std::dynamic_pointer_cast(struct_chunk->field(0)); - ASSERT_TRUE(str_chunk); - for (int64_t j = 0; j < str_chunk->length(); ++j) { - int32_t row = 20 + static_cast(seen); - std::string expected = - "p" + std::to_string(row / 10) + "_" + (row < 10 ? "0" : "") + std::to_string(row); - ASSERT_EQ(expected, str_chunk->GetString(j)); - ++seen; - } + int64_t rows_read = 0; + for (const auto& chunk : result->chunks()) { + ASSERT_TRUE(chunk->ValidateFull().ok()); + ASSERT_EQ(std::min(7, 40 - rows_read), chunk->length()); + ASSERT_TRUE(data->Slice(20 + rows_read, chunk->length())->Equals(chunk)); + rows_read += chunk->length(); } - ASSERT_EQ(40, seen); + ASSERT_EQ(40, rows_read); } -TEST_F(PageFilteredRowGroupReaderTest, NormalizesSlicedBatchOffsets) { - std::string file_name = dir_->Str() + "/normalized_sliced_offsets.parquet"; - std::shared_ptr data = MakeSequentialIntData(60); - WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/60); - - std::shared_ptr read_schema = - arrow::schema({arrow::field("val", arrow::int32())}); - std::shared_ptr predicate = PredicateBuilder::GreaterOrEqual( - /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(20)); - - std::shared_ptr result; - ReadWithPredicateImpl(file_name, read_schema, predicate, &result, /*batch_size=*/7); - ASSERT_TRUE(result); - ASSERT_EQ(40, result->length()); - ASSERT_GT(result->num_chunks(), 1); +TEST_F(PageFilteredRowGroupReaderTest, StreamingAlignsMisalignedChunksAcrossDecodingWindows) { + constexpr int32_t kRows = 26; + constexpr int64_t kBatchSize = 7; + arrow::StringBuilder short_strings; + arrow::StringBuilder long_strings; + for (int32_t row = 0; row < kRows; ++row) { + ASSERT_TRUE( + (row % 5 == 0 ? short_strings.AppendNull() : short_strings.Append(std::to_string(row))) + .ok()); + ASSERT_TRUE((row % 5 == 1 ? long_strings.AppendNull() + : long_strings.Append(std::string(row + 1, 'x'))) + .ok()); + } + auto data = + arrow::StructArray::Make( + {short_strings.Finish().ValueOrDie(), long_strings.Finish().ValueOrDie()}, + {arrow::field("short_text", arrow::utf8()), arrow::field("long_text", arrow::utf8())}) + .ValueOrDie(); + const std::string file_name = dir_->Str() + "/streaming_misaligned_chunks.parquet"; + WriteTestFile(file_name, data, 2, kRows); + ASSERT_OK_AND_ASSIGN(std::shared_ptr input, fs_->Open(file_name)); + ASSERT_OK_AND_ASSIGN(int64_t length, input->Length()); + auto stream = std::make_shared(input, length, arrow_pool_); + std::unique_ptr<::parquet::arrow::FileReader> file_reader; + ASSERT_TRUE(::parquet::arrow::OpenFile(stream, arrow_pool_.get(), &file_reader).ok()); + auto indexes = file_reader->parquet_reader()->GetPageIndexReader()->RowGroup(0); + const RowRanges ranges( + {RowRanges::Range(2, 5), RowRanges::Range(10, 15), RowRanges::Range(19, 23)}); + TargetRowGroup target(0, true, ranges); + const auto page_ranges = PageFilteredRowGroupReader::ComputePageRanges( + target, {0, 1}, indexes, file_reader->parquet_reader()); + + std::vector> decoded_windows(2); + auto chunked_reader = std::make_unique<::testing::StrictMock>(); + EXPECT_CALL(*chunked_reader, manifest()) + .WillRepeatedly(::testing::ReturnRef(file_reader->manifest())); + EXPECT_CALL(*chunked_reader, parquet_reader()) + .WillRepeatedly(::testing::Return(file_reader->parquet_reader())); + EXPECT_CALL(*chunked_reader, GetColumn(::testing::AnyOf(0, 1), ::testing::Ne(nullptr), + ::testing::Ne(nullptr), ::testing::Ne(nullptr))) + .Times(2) + .WillRepeatedly([&](int32_t field_index, + const std::shared_ptr>& columns, + ::parquet::arrow::FileColumnIteratorFactory factory, + std::unique_ptr<::parquet::arrow::ColumnReader>* out) { + ARROW_RETURN_NOT_OK(file_reader->GetColumn(field_index, columns, factory, out)); + *out = std::make_unique(std::move(*out), field_index + 2, + &decoded_windows[field_index]); + return arrow::Status::OK(); + }); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batches, + PageFilteredRowGroupReader::ReadFilteredRowGroup( + target, {0, 1}, arrow::io::CacheOptions::Defaults(), false, + page_ranges, kBatchSize, indexes, arrow_pool_, chunked_reader.get())); + ASSERT_TRUE(batches->schema()->Equals(*arrow::schema(data->type()->fields()))); + ASSERT_TRUE(decoded_windows[0].empty()); + ASSERT_TRUE(decoded_windows[1].empty()); + + // Each seven-row window has chunks [2, 2, 2, 1] and [3, 3, 1]. Their combined + // boundaries must yield five batches before decoding the next window. The final + // one-row window returns through the single-chunk path after draining the second. + const std::vector> expected_batch_sizes = { + {2, 1, 1, 2, 1}, {2, 1, 1, 2, 1}, {1}}; + std::vector> retained_batches; + std::vector expected_decoded_windows; + for (size_t window = 0; window < expected_batch_sizes.size(); ++window) { + SCOPED_TRACE(window); + expected_decoded_windows.push_back(window < 2 ? kBatchSize : 1); + for (int64_t batch_size : expected_batch_sizes[window]) { + std::shared_ptr batch; + ASSERT_TRUE(batches->ReadNext(&batch).ok()); + ASSERT_NE(nullptr, batch); + ASSERT_TRUE(batch->ValidateFull().ok()); + ASSERT_TRUE(batch->schema()->Equals(*batches->schema())); + ASSERT_EQ(batch_size, batch->num_rows()); + for (int32_t column = 0; column < batch->num_columns(); ++column) { + ASSERT_EQ(0, batch->column(column)->offset()); + ASSERT_EQ(expected_decoded_windows, decoded_windows[column]); + } + retained_batches.push_back(std::move(batch)); + } + } + for (int32_t attempt = 0; attempt < 2; ++attempt) { + std::shared_ptr eof = retained_batches.front(); + ASSERT_TRUE(batches->ReadNext(&eof).ok()); + ASSERT_EQ(nullptr, eof); + } + ASSERT_TRUE(batches->Close().ok()); + batches.reset(); + chunked_reader.reset(); + file_reader.reset(); + + const std::vector expected_rows = {2, 3, 4, 5, 10, 11, 12, 13, + 14, 15, 19, 20, 21, 22, 23}; + size_t selected_row = 0; + for (const auto& batch : retained_batches) { + ASSERT_TRUE(batch->ValidateFull().ok()); + for (int64_t row = 0; row < batch->num_rows(); ++row) { + ASSERT_LT(selected_row, expected_rows.size()); + for (int32_t column = 0; column < batch->num_columns(); ++column) { + ASSERT_TRUE(batch->column(column)->Slice(row, 1)->Equals( + data->field(column)->Slice(expected_rows[selected_row], 1))); + } + ++selected_row; + } + } + ASSERT_EQ(expected_rows.size(), selected_row); } /// Test: end-to-end page-filtered read produces correct results when using page-level PreBuffer. @@ -1421,10 +1596,11 @@ TEST_F(PageFilteredRowGroupReaderTest, DirectOffsetIndexJumpDoesNotReadUnselecte std::shared_ptr result; std::vector read_at_ranges; ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result, - options, /*batch_size=*/1024, &read_at_ranges); + options, 1, &read_at_ranges); ASSERT_TRUE(result); ASSERT_EQ(2, result->length()); + ASSERT_EQ(2, result->num_chunks()); auto flat = arrow::Concatenate(result->chunks()).ValueOrDie(); auto struct_arr = std::dynamic_pointer_cast(flat); ASSERT_TRUE(struct_arr); @@ -1634,10 +1810,11 @@ TEST_F(PageFilteredRowGroupReaderTest, DirectOffsetIndexJumpSupportsDictionaryFa std::shared_ptr result; std::vector read_at_ranges; ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result, - options, /*batch_size=*/1024, &read_at_ranges); + options, 1, &read_at_ranges); ASSERT_TRUE(result); ASSERT_EQ(2, result->length()); + ASSERT_EQ(2, result->num_chunks()); auto flat = arrow::Concatenate(result->chunks()).ValueOrDie(); auto struct_arr = std::dynamic_pointer_cast(flat); ASSERT_TRUE(struct_arr); @@ -1656,7 +1833,14 @@ TEST_F(PageFilteredRowGroupReaderTest, DirectOffsetIndexJumpSupportsDictionaryFa /// Empty row selection may read OffsetIndex metadata, but must not read dictionary/data pages. TEST_F(PageFilteredRowGroupReaderTest, DictionaryEmptySelectionDoesNotReadPages) { std::string file_name = dir_->Str() + "/dictionary_empty_bitmap.parquet"; - auto data = MakeSequentialIntData(100); + arrow::StringBuilder strings; + for (int32_t row = 0; row < 100; ++row) { + ASSERT_TRUE(strings.Append(std::to_string(row % 3)).ok()); + } + std::shared_ptr values; + ASSERT_TRUE(strings.Finish(&values).ok()); + auto data = + arrow::StructArray::Make({values}, {arrow::field("val", arrow::utf8())}).ValueOrDie(); WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100, /*enable_dictionary=*/true); @@ -1667,6 +1851,9 @@ TEST_F(PageFilteredRowGroupReaderTest, DictionaryEmptySelectionDoesNotReadPages) ::parquet::arrow::FileReaderBuilder builder; ASSERT_TRUE(builder.Open(in_stream).ok()); + ::parquet::ArrowReaderProperties properties; + properties.set_read_dictionary(0, true); + builder.properties(properties); builder.memory_pool(arrow_pool_.get()); auto arrow_file_reader_result = builder.Build(); ASSERT_TRUE(arrow_file_reader_result.ok()) << arrow_file_reader_result.status().ToString(); @@ -1701,9 +1888,14 @@ TEST_F(PageFilteredRowGroupReaderTest, DictionaryEmptySelectionDoesNotReadPages) /*pre_buffered=*/false, /*page_ranges=*/{}, /*max_chunksize=*/1024, /*row_group_page_index_reader=*/row_group_page_index, arrow_pool_, arrow_file_reader.get())); + ASSERT_TRUE(result_reader->schema()->Equals( + *arrow::schema({arrow::field("val", arrow::dictionary(arrow::int32(), arrow::utf8()))}))); std::shared_ptr batch; ASSERT_TRUE(result_reader->ReadNext(&batch).ok()); ASSERT_FALSE(batch); + ASSERT_TRUE(result_reader->ReadNext(&batch).ok()); + ASSERT_FALSE(batch); + ASSERT_TRUE(result_reader->Close().ok()); for (const auto& read_range : tracking_input->GetReadAtRanges()) { const int64_t read_end = read_range.offset + read_range.length; ASSERT_TRUE(read_end <= column_chunk_offset || read_range.offset >= column_chunk_end) @@ -1739,8 +1931,7 @@ TEST_F(PageFilteredRowGroupReaderTest, DirectOffsetIndexJumpDataPageV2AdjacentPa ASSERT_EQ(10, page_locations.size()); RoaringBitmap32 bitmap; - bitmap.Add(45); // Page 4. - bitmap.Add(55); // Adjacent page 5. + bitmap.AddRange(43, 57); std::map options; options[PARQUET_READ_ROW_RANGES_COALESCE_HOLE_SIZE_LIMIT] = "0"; @@ -1750,17 +1941,19 @@ TEST_F(PageFilteredRowGroupReaderTest, DirectOffsetIndexJumpDataPageV2AdjacentPa std::shared_ptr result; std::vector read_at_ranges; ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result, - options, /*batch_size=*/1024, &read_at_ranges); + options, 3, &read_at_ranges); ASSERT_TRUE(result); - ASSERT_EQ(2, result->length()); + ASSERT_EQ(14, result->length()); + ASSERT_EQ(5, result->num_chunks()); auto flat = arrow::Concatenate(result->chunks()).ValueOrDie(); auto struct_arr = std::dynamic_pointer_cast(flat); ASSERT_TRUE(struct_arr); auto values = std::dynamic_pointer_cast(struct_arr->field(0)); ASSERT_TRUE(values); - ASSERT_EQ(45, values->Value(0)); - ASSERT_EQ(55, values->Value(1)); + for (int64_t row = 0; row < values->length(); ++row) { + ASSERT_EQ(43 + row, values->Value(row)); + } AssertSelectedPagesRead(page_locations, /*selected_pages=*/{4, 5}, read_at_ranges); AssertUnselectedPageHeadersNotRead(page_locations, /*selected_pages=*/{4, 5}, read_at_ranges); } @@ -1789,10 +1982,11 @@ TEST_F(PageFilteredRowGroupReaderTest, MissingOffsetIndexFallsBackToSequentialRe auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); std::shared_ptr result; - ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result); + ReadWithPredicateAndBitmapImpl(file_name, read_schema, nullptr, bitmap, &result, {}, 1); ASSERT_TRUE(result); ASSERT_EQ(2, result->length()); + ASSERT_EQ(2, result->num_chunks()); auto flat = arrow::Concatenate(result->chunks()).ValueOrDie(); auto struct_arr = std::dynamic_pointer_cast(flat); ASSERT_TRUE(struct_arr); @@ -1804,13 +1998,19 @@ TEST_F(PageFilteredRowGroupReaderTest, MissingOffsetIndexFallsBackToSequentialRe TEST_F(PageFilteredRowGroupReaderTest, InvalidOffsetIndexFallsBackWithoutChangingSelectedRows) { std::string file_name = dir_->Str() + "/invalid_offset_index.parquet"; - auto data = MakeSequentialIntData(100); + auto nested = MakeNestedStructData(100); + auto data = + arrow::StructArray::Make({nested}, {arrow::field("info", nested->type())}).ValueOrDie(); WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); auto stream = std::make_shared(in, length, arrow_pool_); std::unique_ptr<::parquet::arrow::FileReader> reader; ASSERT_TRUE(::parquet::arrow::OpenFile(stream, arrow_pool_.get(), &reader).ok()); + auto valid_index = + reader->parquet_reader()->GetPageIndexReader()->RowGroup(0)->GetOffsetIndex(1); + ASSERT_NE(nullptr, valid_index); + ASSERT_EQ(10, valid_index->page_locations().size()); auto column = reader->parquet_reader()->metadata()->RowGroup(0)->ColumnChunk(0); int64_t offset = column->data_page_offset(); int64_t size = column->total_compressed_size(); @@ -1826,8 +2026,7 @@ TEST_F(PageFilteredRowGroupReaderTest, InvalidOffsetIndexFallsBackWithoutChangin rows.Add(RowRanges::Range(15, 19)); rows.Add(RowRanges::Range(85, 89)); TargetRowGroup target(0, true, rows); - // A missing index and each malformed layout must select the same rows using the - // sequential fallback, including through the streaming reader's schema API. + const std::vector expected_rows = {15, 16, 17, 18, 19, 85, 86, 87, 88, 89}; for (size_t i = 0; i <= invalid_pages.size(); ++i) { SCOPED_TRACE(i); std::shared_ptr<::parquet::OffsetIndex> index; @@ -1836,33 +2035,38 @@ TEST_F(PageFilteredRowGroupReaderTest, InvalidOffsetIndexFallsBackWithoutChangin ASSERT_FALSE( PageFilteredRowGroupReader::MakeDataPageReadPlan(rows, index, *column, 100)); } - auto page_index = std::make_shared(index); + auto page_index = std::make_shared( + std::vector>{index, valid_index}); std::vector ranges = PageFilteredRowGroupReader::ComputePageRanges( - target, {0}, page_index, reader->parquet_reader()); - ASSERT_EQ(1, ranges.size()); + target, {0, 1}, page_index, reader->parquet_reader()); + ASSERT_EQ(3, ranges.size()); ASSERT_EQ(offset, ranges[0].offset); ASSERT_EQ(size, ranges[0].length); + AssertSelectedPagesRead(valid_index->page_locations(), {1, 8}, ranges); + AssertUnselectedPageHeadersNotRead(valid_index->page_locations(), {1, 8}, ranges); ASSERT_OK_AND_ASSIGN(std::unique_ptr batches, PageFilteredRowGroupReader::ReadFilteredRowGroup( - target, {0}, arrow::io::CacheOptions::Defaults(), false, ranges, + target, {0, 1}, arrow::io::CacheOptions::Defaults(), false, ranges, /*max_chunksize=*/3, page_index, arrow_pool_, reader.get())); - ASSERT_TRUE( - batches->schema()->Equals(*arrow::schema({arrow::field("val", arrow::int32())}))); - std::vector selected; + ASSERT_TRUE(batches->schema()->Equals(*arrow::schema(data->type()->fields()))); + size_t selected_count = 0; while (true) { std::shared_ptr batch; ASSERT_TRUE(batches->ReadNext(&batch).ok()); if (!batch) { break; } + ASSERT_TRUE(batch->ValidateFull().ok()); + ASSERT_GT(batch->num_rows(), 0); ASSERT_LE(batch->num_rows(), 3); - auto values = std::dynamic_pointer_cast(batch->column(0)); - ASSERT_NE(nullptr, values); - for (int64_t row = 0; row < values->length(); ++row) { - selected.push_back(values->Value(row)); + for (int64_t row = 0; row < batch->num_rows(); ++row) { + ASSERT_LT(selected_count, expected_rows.size()); + ASSERT_TRUE(batch->column(0)->Slice(row, 1)->Equals( + data->field(0)->Slice(expected_rows[selected_count], 1))); + ++selected_count; } } - ASSERT_EQ((std::vector{15, 16, 17, 18, 19, 85, 86, 87, 88, 89}), selected); + ASSERT_EQ(expected_rows.size(), selected_count); } } @@ -2371,25 +2575,24 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedStructSubColumnProjection) { auto predicate = PredicateBuilder::GreaterOrEqual( /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(70)); - std::shared_ptr result; - ReadWithPredicateImpl(file_name, read_schema, predicate, &result); - - ASSERT_TRUE(result); - ASSERT_EQ(30, result->length()); - - // Result is struct> - auto result_struct = std::dynamic_pointer_cast(result->chunk(0)); - ASSERT_TRUE(result_struct); - ASSERT_EQ(1, result_struct->num_fields()); - - auto info_result = std::dynamic_pointer_cast(result_struct->field(0)); - ASSERT_TRUE(info_result); - ASSERT_EQ(1, info_result->num_fields()); - - auto x_arr = std::dynamic_pointer_cast(info_result->field(0)); - ASSERT_TRUE(x_arr); - for (int32_t i = 0; i < 30; ++i) { - ASSERT_EQ((70 + i) * 100, x_arr->Value(i)) << "Mismatch at index " << i; + for (int32_t batch_size : {7, 1024}) { + SCOPED_TRACE(batch_size); + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result, batch_size); + ASSERT_TRUE(result); + ASSERT_EQ(30, result->length()); + ASSERT_TRUE(result->type()->Equals(arrow::struct_(read_schema->fields()))); + if (batch_size < result->length()) { + ASSERT_GT(result->num_chunks(), 1); + } + auto combined = arrow::Concatenate(result->chunks()); + ASSERT_TRUE(combined.ok()) << combined.status().ToString(); + const auto& result_struct = checked_cast(*combined.ValueOrDie()); + const auto& info_result = checked_cast(*result_struct.field(0)); + const auto& values = checked_cast(*info_result.field(0)); + for (int32_t i = 0; i < 30; ++i) { + ASSERT_EQ((70 + i) * 100, values.Value(i)) << "Mismatch at index " << i; + } } } @@ -2637,63 +2840,332 @@ static std::shared_ptr MakeMisalignedNestedDataWithNulls(int .ValueOrDie(); } -/// Test: nested columns containing NULLs, with MISALIGNED leaf pages, split across -/// MULTIPLE row groups. This combines the three stress dimensions: null-driven -/// definition levels, byte-based misaligned pages, and row-group boundaries that the -/// selection bitmap crosses. Every returned row is deep-compared (nulls included) -/// against the original data identified by its non-null key. -/// Bitmap: [0,15), [77, 87) (to avoid bitmap hole filling) -/// Expected: 25 rows -> keys 0..14 and 77..86 TEST_F(PageFilteredRowGroupReaderTest, NestedColumnsWithNullsMisalignedPagesMultiRowGroup) { - std::string file_name = dir_->Str() + "/nested_nulls_misaligned_multi_rg.parquet"; - constexpr int32_t kNumRows = 100; - auto data = MakeMisalignedNestedDataWithNulls(kNumRows); - - // write_batch_size=1 + byte-based data page size -> misaligned leaf pages. - // max_row_group_length=40 -> 3 row groups (40, 40, 20). - WriteTestFile(file_name, data, /*write_batch_size=*/1, /*max_row_group_length=*/40, - /*enable_dictionary=*/false, /*data_page_size=*/40); - - auto read_schema = - arrow::schema({arrow::field("key", arrow::int64()), - arrow::field("s", arrow::struct_({arrow::field("x", arrow::int32()), - arrow::field("y", arrow::int64())})), - arrow::field("tags", arrow::list(arrow::field("item", arrow::int32()))), - arrow::field("props", arrow::map(arrow::utf8(), arrow::int32()))}); - - // bitmap: [0,15), [77, 87) -> spans all three row groups. - RoaringBitmap32 bitmap; - bitmap.AddRange(0, 15); - bitmap.AddRange(77, 87); + constexpr int32_t kRows = 100; + constexpr int32_t kBatchSize = 7; + auto data = MakeMisalignedNestedDataWithNulls(kRows); + const std::vector selections = { + RowRanges({RowRanges::Range(0, 14), RowRanges::Range(77, 86)}), + RowRanges({RowRanges::Range(2, 15), RowRanges::Range(43, 68), RowRanges::Range(83, 96)}), + RowRanges({RowRanges::Range(2, 15), RowRanges::Range(40, 79), RowRanges::Range(83, 96)})}; + for (bool enable_page_index : {true, false}) { + SCOPED_TRACE(enable_page_index); + const std::string file_name = + dir_->Str() + + (enable_page_index ? "/nested_indexed.parquet" : "/nested_no_index.parquet"); + WriteTestFile(file_name, data, 1, 40, false, 40, enable_page_index); + for (size_t selection_index = 0; selection_index < selections.size(); ++selection_index) { + SCOPED_TRACE(selection_index); + RoaringBitmap32 selection; + for (const auto& range : selections[selection_index].GetRanges()) { + selection.AddRange(range.from, range.to + 1); + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr input, fs_->Open(file_name)); + ASSERT_OK_AND_ASSIGN(int64_t length, input->Length()); + auto stream = std::make_shared(input, length, arrow_pool_); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr reader, + ParquetFileBatchReader::Create( + std::move(stream), {{PARQUET_READ_ROW_RANGES_COALESCE_HOLE_SIZE_LIMIT, "0"}}, + kBatchSize, nullptr, nullptr, arrow_pool_, std::nullopt)); + ArrowSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(data->type()->fields()), &schema).ok()); + ASSERT_OK(reader->SetReadSchema(&schema, nullptr, selection)); + std::vector actual_rows; + arrow::ArrayVector retained_batches; + while (true) { + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch()); + if (BatchReader::IsEofBatch(batch)) { + break; + } + ASSERT_OK(paimon::test::ReadResultCollector::CheckBatchOffset(batch)); + auto result = arrow::ImportArray(batch.first.get(), batch.second.get()); + ASSERT_TRUE(result.ok()) << result.status().ToString(); + std::shared_ptr array = std::move(result).ValueOrDie(); + ASSERT_TRUE(array->ValidateFull().ok()); + ASSERT_GT(array->length(), 0); + ASSERT_LE(array->length(), kBatchSize); + for (int64_t row = 0; row < array->length(); ++row) { + ASSERT_OK_AND_ASSIGN(uint64_t file_row, reader->GetPreviousBatchFileRowId(row)); + ASSERT_TRUE(selection.Contains(file_row)); + actual_rows.push_back(static_cast(file_row)); + } + retained_batches.push_back(std::move(array)); + } + std::vector expected_rows; + for (auto it = selection.Begin(); it != selection.End(); ++it) { + expected_rows.push_back(*it); + } + ASSERT_EQ(expected_rows, actual_rows); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch eof, reader->NextBatch()); + ASSERT_TRUE(BatchReader::IsEofBatch(eof)); + reader->Close(); + reader.reset(); + size_t selected_row = 0; + for (const auto& array : retained_batches) { + ASSERT_TRUE(array->ValidateFull().ok()); + for (int64_t row = 0; row < array->length(); ++row) { + const uint32_t file_row = expected_rows[selected_row++]; + ASSERT_TRUE(data->Slice(file_row, 1)->Equals(array->Slice(row, 1))) + << "row content mismatch at file row " << file_row; + } + } + } + } +} - std::shared_ptr result; - ReadWithPredicateAndBitmapImpl(file_name, read_schema, nullptr, bitmap, &result); - ASSERT_TRUE(result); +TEST_F(PageFilteredRowGroupReaderTest, StreamingPreservesDictionaryValuesAcrossSelections) { + constexpr int32_t kRows = 60; + constexpr int64_t kBatchSize = 7; + arrow::StringBuilder strings; + for (int32_t row = 0; row < kRows; ++row) { + if (row % 5 == 0) { + ASSERT_TRUE(strings.AppendNull().ok()); + } else { + ASSERT_TRUE(strings.Append(std::to_string(row % 4)).ok()); + } + } + std::shared_ptr values; + ASSERT_TRUE(strings.Finish(&values).ok()); + auto data = arrow::StructArray::Make( + {MakeSequentialIntData(kRows)->field(0), values}, + {arrow::field("id", arrow::int32()), arrow::field("text", arrow::utf8())}) + .ValueOrDie(); + const std::string file_name = dir_->Str() + "/streaming_dictionary.parquet"; + WriteTestFile(file_name, data, 10, kRows, true); + ASSERT_OK_AND_ASSIGN(std::shared_ptr input, fs_->Open(file_name)); + ASSERT_OK_AND_ASSIGN(int64_t length, input->Length()); + auto stream = std::make_shared(input, length, arrow_pool_); + ::parquet::ArrowReaderProperties properties; + properties.set_read_dictionary(1, true); + ::parquet::arrow::FileReaderBuilder builder; + ASSERT_TRUE(builder.Open(stream).ok()); + std::unique_ptr<::parquet::arrow::FileReader> file_reader; + ASSERT_TRUE( + builder.memory_pool(arrow_pool_.get())->properties(properties)->Build(&file_reader).ok()); + auto indexes = file_reader->parquet_reader()->GetPageIndexReader()->RowGroup(0); + RowRanges ranges({RowRanges::Range(2, 16), RowRanges::Range(32, 46), RowRanges::Range(52, 58)}); + TargetRowGroup target(0, true, ranges); + const auto page_ranges = PageFilteredRowGroupReader::ComputePageRanges( + target, {0, 1}, indexes, file_reader->parquet_reader()); + for (int64_t batch_size : {-1, 0}) { + auto result = PageFilteredRowGroupReader::ReadFilteredRowGroup( + target, {0, 1}, arrow::io::CacheOptions::Defaults(), false, page_ranges, batch_size, + indexes, arrow_pool_, file_reader.get()); + ASSERT_FALSE(result.ok()); + ASSERT_TRUE(result.status().IsInvalid()); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr batches, + PageFilteredRowGroupReader::ReadFilteredRowGroup( + target, {0, 1}, arrow::io::CacheOptions::Defaults(), false, + page_ranges, kBatchSize, indexes, arrow_pool_, file_reader.get())); + ASSERT_EQ(arrow::Type::DICTIONARY, batches->schema()->field(1)->type()->id()); + std::vector> retained_batches; + std::vector actual_rows; + while (true) { + std::shared_ptr batch; + ASSERT_TRUE(batches->ReadNext(&batch).ok()); + if (!batch) { + break; + } + ASSERT_TRUE(batch->ValidateFull().ok()); + ASSERT_GT(batch->num_rows(), 0); + ASSERT_LE(batch->num_rows(), kBatchSize); + ASSERT_TRUE(batch->schema()->Equals(*batches->schema())); + ASSERT_EQ(arrow::Type::DICTIONARY, batch->column(1)->type_id()); + const auto& ids = checked_cast(*batch->column(0)); + for (int64_t row = 0; row < batch->num_rows(); ++row) { + actual_rows.push_back(ids.Value(row)); + } + retained_batches.push_back(std::move(batch)); + } + std::vector expected_rows; + for (const auto& range : ranges.GetRanges()) { + for (int64_t row = range.from; row <= range.to; ++row) { + expected_rows.push_back(static_cast(row)); + } + } + ASSERT_EQ(expected_rows, actual_rows); + ASSERT_GT(retained_batches.size(), 1); + ASSERT_TRUE(batches->Close().ok()); + batches.reset(); + file_reader.reset(); + for (const auto& batch : retained_batches) { + const auto& ids = checked_cast(*batch->column(0)); + const auto& encoded = checked_cast(*batch->column(1)); + const auto& dictionary = checked_cast(*encoded.dictionary()); + for (int64_t row = 0; row < batch->num_rows(); ++row) { + const int32_t file_row = ids.Value(row); + ASSERT_EQ(file_row % 5 == 0, encoded.IsNull(row)); + if (!encoded.IsNull(row)) { + ASSERT_EQ(std::to_string(file_row % 4), + dictionary.GetString(encoded.GetValueIndex(row))); + } + } + } +} - std::vector keys; +TEST_F(PageFilteredRowGroupReaderTest, StreamingBoundsDecodedMemoryAndPreservesReturnedBatches) { + constexpr int32_t kRows = 16384; + constexpr int64_t kBatchSize = 64; + const std::string value(1024, 'x'); + arrow::StringBuilder strings; + for (int32_t row = 0; row < kRows; ++row) { + ASSERT_TRUE(strings.Append(value).ok()); + } + std::shared_ptr values; + ASSERT_TRUE(strings.Finish(&values).ok()); + auto data = arrow::StructArray::Make( + {MakeSequentialIntData(kRows)->field(0), values}, + {arrow::field("id", arrow::int32()), arrow::field("text", arrow::utf8())}) + .ValueOrDie(); + const std::string file_name = dir_->Str() + "/streaming_memory.parquet"; + WriteTestFile(file_name, data, 256, kRows, true); + + const RowRanges ranges(RowRanges::Range(3, kRows - 5)); + for (int64_t expected_rows : {int64_t{0}, 4 * kBatchSize, ranges.RowCount()}) { + SCOPED_TRACE(expected_rows); + auto read_pool = GetArrowPool(pool_); + ASSERT_OK_AND_ASSIGN(std::shared_ptr input, fs_->Open(file_name)); + ASSERT_OK_AND_ASSIGN(int64_t length, input->Length()); + auto stream = std::make_shared(input, length, read_pool); + std::unique_ptr<::parquet::arrow::FileReader> file_reader; + ASSERT_TRUE(::parquet::arrow::OpenFile(stream, read_pool.get(), &file_reader).ok()); + auto indexes = file_reader->parquet_reader()->GetPageIndexReader()->RowGroup(0); + TargetRowGroup target(0, true, ranges); + const auto page_ranges = PageFilteredRowGroupReader::ComputePageRanges( + target, {0, 1}, indexes, file_reader->parquet_reader()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batches, + PageFilteredRowGroupReader::ReadFilteredRowGroup( + target, {0, 1}, arrow::io::CacheOptions::Defaults(), false, + page_ranges, kBatchSize, indexes, read_pool, file_reader.get())); + ASSERT_TRUE(batches->schema()->Equals(*arrow::schema(data->type()->fields()))); - auto concated = arrow::Concatenate(result->chunks()).ValueOrDie(); - auto top = std::dynamic_pointer_cast(concated); - ASSERT_TRUE(top); - auto key_arr = std::dynamic_pointer_cast(top->field(0)); - ASSERT_TRUE(key_arr); - for (int64_t i = 0; i < key_arr->length(); ++i) { - ASSERT_FALSE(key_arr->IsNull(i)) << "key column must stay non-null"; - int64_t k = key_arr->Value(i); - // Deep compare including nulls across struct/list/map leaves. - ASSERT_TRUE(data->Slice(k, 1)->Equals(*top->Slice(i, 1))) - << "row content mismatch at key " << k; - keys.push_back(k); + std::shared_ptr first; + int64_t rows_read = 0; + while (rows_read < expected_rows) { + std::shared_ptr batch; + ASSERT_TRUE(batches->ReadNext(&batch).ok()); + ASSERT_NE(nullptr, batch); + ASSERT_TRUE(batch->ValidateFull().ok()); + ASSERT_EQ(std::min(kBatchSize, expected_rows - rows_read), batch->num_rows()); + const auto& ids = checked_cast(*batch->column(0)); + const auto& text = checked_cast(*batch->column(1)); + for (int64_t row = 0; row < batch->num_rows(); ++row) { + ASSERT_EQ(3 + rows_read + row, ids.Value(row)); + ASSERT_EQ(value, text.GetString(row)); + } + if (!first) { + first = batch; + ASSERT_LT(read_pool->max_memory(), 4 * 1024 * 1024); + } + rows_read += batch->num_rows(); + } + ASSERT_LT(read_pool->max_memory(), 4 * 1024 * 1024); + if (expected_rows == ranges.RowCount()) { + std::shared_ptr eof = first; + ASSERT_TRUE(batches->ReadNext(&eof).ok()); + ASSERT_EQ(nullptr, eof); + const int64_t bytes_at_eof = read_pool->bytes_allocated(); + ASSERT_TRUE(batches->ReadNext(&eof).ok()); + ASSERT_EQ(nullptr, eof); + ASSERT_TRUE(batches->Close().ok()); + ASSERT_EQ(bytes_at_eof, read_pool->bytes_allocated()); + } + ASSERT_TRUE(batches->Close().ok()); + const int64_t bytes_after_close = read_pool->bytes_allocated(); + ASSERT_TRUE(batches->Close().ok()); + std::shared_ptr after_close = first; + ASSERT_TRUE(batches->ReadNext(&after_close).ok()); + ASSERT_EQ(nullptr, after_close); + batches.reset(); + ASSERT_EQ(bytes_after_close, read_pool->bytes_allocated()); + file_reader.reset(); + indexes.reset(); + + if (first) { + const auto& ids = checked_cast(*first->column(0)); + const auto& text = checked_cast(*first->column(1)); + for (int64_t row = 0; row < kBatchSize; ++row) { + ASSERT_EQ(3 + row, ids.Value(row)); + ASSERT_EQ(value, text.GetString(row)); + } + } + first.reset(); + ASSERT_EQ(0, read_pool->bytes_allocated()); } +} - std::vector expected; - for (int64_t i = 0; i < 15; ++i) { - expected.push_back(i); +TEST_F(PageFilteredRowGroupReaderTest, StreamingDefersLaterPageDecodeErrors) { + const std::string file_name = dir_->Str() + "/streaming_corrupt_tail.parquet"; + auto data = MakeTwoColumnData(100); + WriteTestFile(file_name, data, 10, 100); + ASSERT_OK_AND_ASSIGN(std::shared_ptr input, fs_->Open(file_name)); + ASSERT_OK_AND_ASSIGN(int64_t length, input->Length()); + auto stream = std::make_shared(input, length, arrow_pool_); + std::unique_ptr<::parquet::arrow::FileReader> original; + ASSERT_TRUE(::parquet::arrow::OpenFile(stream, arrow_pool_.get(), &original).ok()); + auto index = original->parquet_reader()->GetPageIndexReader()->RowGroup(0)->GetOffsetIndex(1); + ASSERT_NE(nullptr, index); + ASSERT_EQ(10, index->page_locations().size()); + const auto last_page = index->page_locations().back(); + ASSERT_EQ(90, last_page.first_row_index); + ASSERT_GE(last_page.offset, 0); + ASSERT_LE(last_page.offset + last_page.compressed_page_size, length); + std::string contents(length, '\0'); + ASSERT_OK_AND_ASSIGN(int64_t read, input->Read(contents.data(), length, 0)); + ASSERT_EQ(length, read); + std::fill(contents.begin() + last_page.offset, + contents.begin() + last_page.offset + last_page.compressed_page_size, + static_cast(0xff)); + auto buffer = std::make_shared(reinterpret_cast(contents.data()), + contents.size()); + auto corrupted = std::make_shared(buffer); + std::unique_ptr<::parquet::arrow::FileReader> file_reader; + ASSERT_TRUE(::parquet::arrow::OpenFile(corrupted, arrow_pool_.get(), &file_reader).ok()); + auto indexes = file_reader->parquet_reader()->GetPageIndexReader()->RowGroup(0); + TargetRowGroup target(0, true, RowRanges::CreateSingle(100)); + const auto ranges = PageFilteredRowGroupReader::ComputePageRanges( + target, {0, 1}, indexes, file_reader->parquet_reader()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batches, + PageFilteredRowGroupReader::ReadFilteredRowGroup( + target, {0, 1}, arrow::io::CacheOptions::Defaults(), false, ranges, 5, + indexes, arrow_pool_, file_reader.get())); + int64_t rows = 0; + arrow::Status status; + std::shared_ptr first; + for (int32_t i = 0; i < 20; ++i) { + std::shared_ptr batch = first; + status = batches->ReadNext(&batch); + if (!status.ok()) { + ASSERT_EQ(nullptr, batch); + break; + } + ASSERT_NE(nullptr, batch); + ASSERT_TRUE(batch->ValidateFull().ok()); + ASSERT_EQ(5, batch->num_rows()); + ASSERT_EQ(2, batch->num_columns()); + for (int32_t column = 0; column < batch->num_columns(); ++column) { + ASSERT_TRUE(batch->column(column)->Equals(data->field(column)->Slice(rows, 5))); + } + if (!first) { + first = batch; + } + rows += batch->num_rows(); } - for (int64_t i = 77; i < 87; ++i) { - expected.push_back(i); + ASSERT_EQ(90, rows); + ASSERT_FALSE(status.ok()); + std::shared_ptr batch = first; + ASSERT_EQ(status, batches->ReadNext(&batch)); + ASSERT_EQ(nullptr, batch); + ASSERT_TRUE(batches->Close().ok()); + batches.reset(); + file_reader.reset(); + ASSERT_NE(nullptr, first); + ASSERT_TRUE(first->ValidateFull().ok()); + for (int32_t column = 0; column < first->num_columns(); ++column) { + ASSERT_TRUE(first->column(column)->Equals(data->field(column)->Slice(0, 5))); } - ASSERT_EQ(keys, expected); } } // namespace paimon::parquet::test