From 2a7516db9a01c89536f68f6521cca763815655d7 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 16 Nov 2026 11:00:18 +0800 Subject: [PATCH] fix(cpp): validate the aligned value page row count decode_cur_value_page_data() read the value page's uint32 row count, sized the not-null bitmap from it, and copied that many bytes out of the uncompressed page without checking against the page size. The bitmap index could also run past its end when the time page had more rows. Reject pages shorter than the count field plus bitmap, compute the bitmap size in 64-bit arithmetic so a count near UINT32_MAX cannot wrap to zero, and treat a row beyond a non-empty bitmap as corruption in the single-value decode loops. An empty bitmap still means every row is null. Also return the real error code from decode_cur_time_page_data()/decode_cur_value_page_data(), which || collapsed to 1. --- cpp/src/reader/aligned_chunk_reader.cc | 76 ++++- cpp/src/reader/aligned_chunk_reader.h | 10 + .../aligned_value_page_corruption_test.cc | 289 ++++++++++++++++++ 3 files changed, 360 insertions(+), 15 deletions(-) create mode 100644 cpp/test/reader/aligned_value_page_corruption_test.cc diff --git a/cpp/src/reader/aligned_chunk_reader.cc b/cpp/src/reader/aligned_chunk_reader.cc index 5f5a2d42f..414d8f896 100644 --- a/cpp/src/reader/aligned_chunk_reader.cc +++ b/cpp/src/reader/aligned_chunk_reader.cc @@ -408,7 +408,10 @@ int AlignedChunkReader::get_next_page(TsBlock* ret_tsblock, } } if (IS_SUCC(ret)) { - ret = decode_cur_time_page_data() || decode_cur_value_page_data(); + ret = decode_cur_time_page_data(); + } + if (IS_SUCC(ret)) { + ret = decode_cur_value_page_data(); } } if (IS_SUCC(ret)) { @@ -666,19 +669,38 @@ int AlignedChunkReader::decode_cur_value_page_data() { } // Step 3: get value_buf if (IS_SUCC(ret)) { - uint32_t value_uncompressed_buf_offset = 0; - value_page_data_num_ = - SerializationUtil::read_ui32(value_uncompressed_buf); - value_uncompressed_buf_offset += sizeof(uint32_t); - value_page_col_notnull_bitmap_.resize((value_page_data_num_ + 7) / 8); - for (unsigned char& i : value_page_col_notnull_bitmap_) { - i = *(value_uncompressed_buf + value_uncompressed_buf_offset); - value_uncompressed_buf_offset++; - } - cur_value_index = -1; - value_buf = value_uncompressed_buf + value_uncompressed_buf_offset; - value_buf_size = - value_uncompressed_buf_size - value_uncompressed_buf_offset; + // The page begins with a uint32 row count followed by a not-null + // bitmap of ceil(count/8) bytes. Validate both before the count is + // used to size the bitmap or to walk the buffer. + if (value_uncompressed_buf_size < sizeof(uint32_t)) { + ret = E_TSFILE_CORRUPTED; + } else { + uint32_t value_uncompressed_buf_offset = 0; + value_page_data_num_ = + SerializationUtil::read_ui32(value_uncompressed_buf); + value_uncompressed_buf_offset += sizeof(uint32_t); + // 64-bit arithmetic: value_page_data_num_ + 7 overflows uint32_t + // for counts near UINT32_MAX. + const uint64_t bitmap_bytes = + (static_cast(value_page_data_num_) + 7) / 8; + if (bitmap_bytes > + value_uncompressed_buf_size - value_uncompressed_buf_offset) { + ret = E_TSFILE_CORRUPTED; + } else { + value_page_col_notnull_bitmap_.resize( + static_cast(bitmap_bytes)); + for (unsigned char& i : value_page_col_notnull_bitmap_) { + i = *(value_uncompressed_buf + + value_uncompressed_buf_offset); + value_uncompressed_buf_offset++; + } + cur_value_index = -1; + value_buf = + value_uncompressed_buf + value_uncompressed_buf_offset; + value_buf_size = + value_uncompressed_buf_size - value_uncompressed_buf_offset; + } + } } value_decoder_->reset(); #ifdef DEBUG_SE @@ -738,6 +760,10 @@ int AlignedChunkReader::decode_time_value_buf_into_tsblock( CppType value; \ while (time_decoder_->has_remaining(time_in)) { \ cur_value_index++; \ + if (UNLIKELY(value_row_out_of_bitmap(cur_value_index))) { \ + ret = E_TSFILE_CORRUPTED; \ + break; \ + } \ if (value_page_col_notnull_bitmap_.empty() || \ ((value_page_col_notnull_bitmap_[cur_value_index / 8] & \ 0xFF) & \ @@ -858,6 +884,13 @@ int AlignedChunkReader::decode_tv_batch(ByteStream& time_in, if (time_decoder_->peek_next_block_range_int64( time_in, block_min, block_max, block_count)) { if (!filter->satisfy_start_end_time(block_min, block_max)) { + if (block_count > 0 && + UNLIKELY(value_row_out_of_bitmap( + cur_value_index + + static_cast(block_count)))) { + ret = E_TSFILE_CORRUPTED; + break; + } int skipped = 0; time_decoder_->skip_peeked_block_int64(time_in, skipped); int nonnull = 0; @@ -891,6 +924,12 @@ int AlignedChunkReader::decode_tv_batch(ByteStream& time_in, } if (time_count == 0) break; + if (UNLIKELY(value_row_out_of_bitmap( + cur_value_index + static_cast(time_count)))) { + ret = E_TSFILE_CORRUPTED; + break; + } + bool is_null[BATCH]; int nonnull_count = 0; for (int i = 0; i < time_count; ++i) { @@ -1049,6 +1088,10 @@ int AlignedChunkReader::STRING_DECODE_TYPED_TV_INTO_TSBLOCK( uint32_t mask = 1 << 7; while (time_decoder_->has_remaining(time_in)) { cur_value_index++; + if (UNLIKELY(value_row_out_of_bitmap(cur_value_index))) { + ret = E_TSFILE_CORRUPTED; + break; + } bool should_read_data = true; if (value_page_col_notnull_bitmap_.empty() || ((value_page_col_notnull_bitmap_[cur_value_index / 8] & 0xFF) & @@ -1181,7 +1224,10 @@ int AlignedChunkReader::get_next_page(TsBlock* ret_tsblock, } } if (IS_SUCC(ret)) { - ret = decode_cur_time_page_data() || decode_cur_value_page_data(); + ret = decode_cur_time_page_data(); + } + if (IS_SUCC(ret)) { + ret = decode_cur_value_page_data(); } } if (IS_SUCC(ret)) { diff --git a/cpp/src/reader/aligned_chunk_reader.h b/cpp/src/reader/aligned_chunk_reader.h index 176c0b020..d913a693b 100644 --- a/cpp/src/reader/aligned_chunk_reader.h +++ b/cpp/src/reader/aligned_chunk_reader.h @@ -248,6 +248,16 @@ class AlignedChunkReader : public IChunkReader { value_in_.has_remaining(); } + // True when a non-empty value-page bitmap does not cover @row. An empty + // bitmap means the page carried no values (every row is null); otherwise + // the bitmap covers exactly the rows the page declares, and a time page + // that yields more rows than that is corrupt. + FORCE_INLINE bool value_row_out_of_bitmap(int64_t row) const { + return !value_page_col_notnull_bitmap_.empty() && + static_cast(row / 8) >= + value_page_col_notnull_bitmap_.size(); + } + int decode_tv_buf_into_tsblock_by_datatype(common::ByteStream& time_in, common::ByteStream& value_in, common::TsBlock* ret_tsblock, diff --git a/cpp/test/reader/aligned_value_page_corruption_test.cc b/cpp/test/reader/aligned_value_page_corruption_test.cc new file mode 100644 index 000000000..c01604ebe --- /dev/null +++ b/cpp/test/reader/aligned_value_page_corruption_test.cc @@ -0,0 +1,289 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// An aligned value page begins with a uint32 row count that sizes the +// not-null bitmap. These tests corrupt a real file's value-page payload in +// place (the page is +// written UNCOMPRESSED, so its payload is stored verbatim) and read it back +// through AlignedChunkReader. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#else +#include +#endif + +#include "common/allocator/alloc_base.h" +#include "common/config/config.h" +#include "common/tablet.h" +#include "file/local_random_access_read_file.h" +#include "file/tsfile_io_reader.h" +#include "file/write_file.h" +#include "reader/aligned_chunk_reader.h" +#include "writer/tsfile_table_writer.h" + +namespace storage { +namespace { + +constexpr int kRows = 1000; + +class AlignedValuePageCorruptionTest : public ::testing::Test { + protected: + void SetUp() override { + using namespace common; + libtsfile_init(); + saved_ = g_config_value_; + // One page for the whole chunk, so the value page payload is a single + // contiguous run in the file that can be patched in place. + g_config_value_.page_writer_max_point_num_ = kRows; + + schema_ = std::make_shared( + "t", std::vector{ + {"v", INT64, UNCOMPRESSED, PLAIN, ColumnCategory::FIELD}}); + path_ = TempName("valid"); + { + WriteFile file; + int flags = O_WRONLY | O_CREAT | O_TRUNC; +#ifdef _WIN32 + flags |= O_BINARY; +#endif + ASSERT_EQ(file.create(path_, flags, 0666), E_OK); + TsFileTableWriter writer(&file, schema_.get()); + Tablet tablet("t", {"v"}, {INT64}, {ColumnCategory::FIELD}, kRows); + for (int r = 0; r < kRows; ++r) { + ASSERT_EQ(tablet.add_timestamp(r, 1000 + r), E_OK); + ASSERT_EQ(tablet.add_value(r, "v", static_cast(r)), + E_OK); + } + ASSERT_EQ(writer.write_table(tablet), E_OK); + ASSERT_EQ(writer.close(), E_OK); + } + + LocalRandomAccessReadFile file; + ASSERT_EQ(file.open(path_), E_OK); + TsFileIOReader io; + ASSERT_EQ(io.init(&file), E_OK); + TsFileMeta* metadata = nullptr; + ASSERT_EQ(io.get_tsfile_meta(metadata), E_OK); + PageArena arena; + arena.init(512, MOD_TSFILE_READER); + auto device = std::make_shared( + std::vector{"t"}); + std::vector indexes(1, nullptr); + ASSERT_EQ(io.get_timeseries_indexes(device, {"v"}, indexes, arena), + E_OK); + ASSERT_NE(indexes[0], nullptr); + time_meta_ = indexes[0]->get_time_chunk_meta_list()->begin().get(); + value_meta_ = indexes[0]->get_value_chunk_meta_list()->begin().get(); + ASSERT_NE(time_meta_, nullptr); + ASSERT_NE(value_meta_, nullptr); + value_payload_offset_ = PayloadOffset(file, value_meta_); + ASSERT_GT(value_payload_offset_, 0); + } + + void TearDown() override { + std::remove(path_.c_str()); + std::remove(corrupt_path_.c_str()); + common::g_config_value_ = saved_; + libtsfile_destroy(); + } + + static std::string TempName(const char* tag) { + static std::atomic counter{0}; +#ifdef _WIN32 + const auto process_id = static_cast(_getpid()); +#else + const auto process_id = static_cast(getpid()); +#endif + return "aligned_value_page_" + std::string(tag) + "_" + + std::to_string(std::chrono::steady_clock::now() + .time_since_epoch() + .count()) + + "_" + std::to_string(process_id) + "_" + + std::to_string(counter.fetch_add(1)) + ".tsfile"; + } + + // Offset of the first value page's payload: just past the chunk header and + // the page header. A one-page chunk omits the page statistics, so the + // header is only the two var-int sizes. + static int64_t PayloadOffset(RandomAccessReadFile& file, ChunkMeta* meta) { + std::vector bytes(4096); + int32_t actual = 0; + EXPECT_EQ(file.read(meta->offset_of_chunk_header_, bytes.data(), + static_cast(bytes.size()), actual), + common::E_OK); + common::ByteStream in; + in.wrap_from(bytes.data(), actual); + ChunkHeader chunk; + EXPECT_EQ(chunk.deserialize_from(in), common::E_OK); + PageHeader page; + EXPECT_EQ(page.deserialize_from( + in, + (chunk.chunk_type_ & ONLY_ONE_PAGE_CHUNK_HEADER_MARKER) != + ONLY_ONE_PAGE_CHUNK_HEADER_MARKER, + chunk.data_type_), + common::E_OK); + return meta->offset_of_chunk_header_ + in.read_pos(); + } + + // Copies the valid file with its value-page row count replaced by + // @row_count (stored big-endian, as SerializationUtil::write_ui32 does). + void WriteCorruptedFile(uint32_t row_count) { + std::ifstream input(path_, std::ios::binary); + ASSERT_TRUE(input.is_open()); + std::vector bytes((std::istreambuf_iterator(input)), + std::istreambuf_iterator()); + ASSERT_GE(bytes.size(), static_cast(value_payload_offset_) + 4); + for (int i = 0; i < 4; ++i) { + bytes[value_payload_offset_ + i] = + static_cast(row_count >> (24 - i * 8)); + } + corrupt_path_ = TempName("corrupt"); + std::ofstream output(corrupt_path_, std::ios::binary | std::ios::trunc); + ASSERT_TRUE(output.is_open()); + output.write(bytes.data(), static_cast(bytes.size())); + output.close(); + } + + // Reads the single-series ("v") chunk through AlignedChunkReader and + // returns the first non-OK status, or E_NO_MORE_DATA after every row. + int ReadAllRows(const std::string& path, int* rows_read) { + using namespace common; + LocalRandomAccessReadFile file; + if (file.open(path) != E_OK) { + return E_FILE_OPEN_ERR; + } + TsFileIOReader io; + if (io.init(&file) != E_OK) { + return -1; + } + TsFileMeta* metadata = nullptr; + if (io.get_tsfile_meta(metadata) != E_OK) { + return -2; + } + PageArena arena; + arena.init(512, MOD_TSFILE_READER); + auto device = std::make_shared( + std::vector{"t"}); + std::vector indexes(1, nullptr); + if (io.get_timeseries_indexes(device, {"v"}, indexes, arena) != E_OK || + indexes[0] == nullptr) { + return -3; + } + ChunkMeta* time_meta = + indexes[0]->get_time_chunk_meta_list()->begin().get(); + ChunkMeta* value_meta = + indexes[0]->get_value_chunk_meta_list()->begin().get(); + + void* memory = + common::mem_alloc(sizeof(AlignedChunkReader), MOD_CHUNK_READER); + if (memory == nullptr) { + return E_OOM; + } + auto* reader = new (memory) AlignedChunkReader(); + struct ReaderGuard { + AlignedChunkReader* reader; + ~ReaderGuard() { + reader->destroy(); + common::mem_free(reader); + } + } guard{reader}; + if (reader->init(&file, common::String("v", 1), INT64, nullptr) != + E_OK) { + return E_OOM; + } + if (reader->load_by_aligned_meta(time_meta, value_meta) != E_OK) { + return -4; + } + + TupleDesc tuple; + tuple.push_back(g_time_column_schema); + tuple.push_back(schema_->get_column_schema("v")); + TsBlock block(&tuple, 64); + if (block.init() != E_OK) { + return E_OOM; + } + int rows = 0; + for (int call = 0; call < 1000; ++call) { + block.reset(); + const int ret = reader->get_next_page(&block, nullptr, arena); + rows += static_cast(block.get_row_count()); + if (ret != E_OK) { + if (rows_read != nullptr) *rows_read = rows; + return ret == E_NO_MORE_DATA ? E_NO_MORE_DATA : ret; + } + // The chunk is exhausted once no page is left; like the series + // scan iterator, stop here instead of asking for another page. + if (!reader->has_more_data()) { + if (rows_read != nullptr) *rows_read = rows; + return E_NO_MORE_DATA; + } + } + if (rows_read != nullptr) *rows_read = rows; + return E_NO_MORE_DATA; + } + + common::ConfigValue saved_; + std::shared_ptr schema_; + std::string path_; + std::string corrupt_path_; + ChunkMeta* time_meta_ = nullptr; + ChunkMeta* value_meta_ = nullptr; + int64_t value_payload_offset_ = -1; +}; + +TEST_F(AlignedValuePageCorruptionTest, ValidFileReadsEveryRow) { + int rows = 0; + EXPECT_EQ(ReadAllRows(path_, &rows), common::E_NO_MORE_DATA); + EXPECT_EQ(rows, kRows); +} + +TEST_F(AlignedValuePageCorruptionTest, OversizedRowCountIsRejected) { + // Two orders of magnitude more rows than the page can hold: the bitmap + // alone (1 MiB / 8) does not fit in the 8129-byte page payload. + WriteCorruptedFile(1u << 20); + int rows = 0; + EXPECT_EQ(ReadAllRows(corrupt_path_, &rows), common::E_TSFILE_CORRUPTED); + EXPECT_EQ(rows, 0); +} + +TEST_F(AlignedValuePageCorruptionTest, RowCountThatWrapsBitmapSizeIsRejected) { + // UINT32_MAX + 7 wraps to 6 in 32-bit arithmetic, so the bitmap size used + // to become zero and the page was silently read as all-null rows. + WriteCorruptedFile(UINT32_MAX); + int rows = 0; + EXPECT_EQ(ReadAllRows(corrupt_path_, &rows), common::E_TSFILE_CORRUPTED); + EXPECT_EQ(rows, 0); +} + +} // namespace +} // namespace storage