diff --git a/.github/.rat-excludes b/.github/.rat-excludes index c1c5ddefc..b4d9af5ce 100644 --- a/.github/.rat-excludes +++ b/.github/.rat-excludes @@ -12,6 +12,7 @@ iwyu_tool\.py arrow\.diff jieba\.diff orc\.diff +oss_sdk_v2\.diff vortex\.diff licenses/* .codespell_ignore diff --git a/.github/actions/setup-release-assets/action.yml b/.github/actions/setup-release-assets/action.yml new file mode 100644 index 000000000..9a32bbd97 --- /dev/null +++ b/.github/actions/setup-release-assets/action.yml @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Prepare release assets +description: Cache and verify pinned GitHub release archives before building +runs: + using: composite + steps: + - uses: actions/cache@v4 + with: + path: ${{ runner.temp }}/paimon-release-assets + key: release-assets-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('third_party/versions.txt', 'ci/scripts/setup_rust.sh') }} + - shell: bash + run: | + set -eu + source third_party/versions.txt + cache_dir="${RUNNER_TEMP}/paimon-release-assets" + for entry in "${DEPENDENCIES[@]}"; do + read -r var archive url <<< "${entry}" + case "${var}" in + PAIMON_ZLIB_URL|PAIMON_ZSTD_URL|PAIMON_PROTOBUF_URL|PAIMON_ARROW_URL) + checksum_var="${var%_URL}_BUILD_SHA256_CHECKSUM" + bash ci/scripts/download_release_asset.sh "${url}" "${cache_dir}/${archive}" "${!checksum_var}" + echo "${var}=${cache_dir}/${archive}" >> "${GITHUB_ENV}" + ;; + esac + done + echo "PAIMON_RELEASE_ASSET_DIR=${cache_dir}" >> "${GITHUB_ENV}" diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index f9548716d..7e32fffe3 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -43,6 +43,9 @@ jobs: - name: Run CMake module tests shell: bash run: ci/scripts/test_cmake_modules.sh + - name: Test release asset downloads + shell: bash + run: bash ci/scripts/test_download_release_asset.sh - name: Run packaging argument tests shell: bash run: ci/scripts/test_packaging_args.sh @@ -88,6 +91,12 @@ jobs: cc: gcc-14 cxx: g++-14 build_args: --build_type Release + - name: gcc13-release-aarch64 + runner: ubuntu-24.04-arm + cc: gcc-13 + cxx: g++-13 + gcc_version: '13.3.0' + build_args: --build_type Release - name: clang-debug-aarch64 runner: ubuntu-24.04-arm - name: asan-ubsan-aarch64 @@ -109,6 +118,8 @@ jobs: uses: ./.github/actions/setup-ccache with: cache-key-prefix: ccache-${{ matrix.name }} + - name: Prepare release assets + uses: ./.github/actions/setup-release-assets - name: Install Rust and protobuf tools shell: bash run: ci/scripts/setup_rust.sh @@ -116,6 +127,19 @@ jobs: run: | sudo apt-get update sudo apt-get install -y gdb libcurl4-openssl-dev libssl-dev + - name: Install and verify regression compiler + if: matrix.gcc_version + shell: bash + env: + CC: ${{ matrix.cc }} + CXX: ${{ matrix.cxx }} + EXPECTED_GCC_VERSION: ${{ matrix.gcc_version }} + run: | + sudo apt-get install -y "${CC}" "${CXX}" + "${CC}" --version + "${CXX}" --version + test "$("${CC}" -dumpfullversion)" = "${EXPECTED_GCC_VERSION}" + test "$("${CXX}" -dumpfullversion)" = "${EXPECTED_GCC_VERSION}" - name: Install flatc and libclang (Vortex) shell: bash run: ci/scripts/setup_flatc.sh diff --git a/.github/workflows/gcc8_test.yaml b/.github/workflows/gcc8_test.yaml index bdd6a039e..045c43669 100644 --- a/.github/workflows/gcc8_test.yaml +++ b/.github/workflows/gcc8_test.yaml @@ -38,12 +38,19 @@ jobs: container: image: ubuntu:20.04 options: --privileged + volumes: + # The minimal image has no CA bundle until apt installs ca-certificates. + - /etc/ssl/certs/ca-certificates.crt:/etc/apt/runner-ca-certificates.crt:ro timeout-minutes: 120 strategy: fail-fast: false steps: - name: Install dependencies run: | + sed -i -e 's|http://archive.ubuntu.com/ubuntu|https://archive.ubuntu.com/ubuntu|g' \ + -e 's|http://security.ubuntu.com/ubuntu|https://security.ubuntu.com/ubuntu|g' /etc/apt/sources.list + printf '%s\n' 'Acquire::https::CaInfo "/etc/apt/runner-ca-certificates.crt";' \ + 'Acquire::Retries "3";' > /etc/apt/apt.conf.d/80-ci-downloads apt-get update DEBIAN_FRONTEND=noninteractive apt-get install -y gcc-8 g++-8 ninja-build git tar curl libcurl4-openssl-dev libssl-dev tzdata zip unzip pkg-config build-essential python3-dev gdb sudo curl -L -O https://github.com/Kitware/CMake/releases/download/v3.28.3/cmake-3.28.3-linux-x86_64.tar.gz @@ -60,10 +67,13 @@ jobs: ls -la - name: Checkout paimon-cpp uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Prepare release assets + uses: ./.github/actions/setup-release-assets - name: Install Rust and protobuf tools shell: bash run: ci/scripts/setup_rust.sh - name: Setup ccache + id: setup-ccache uses: ./.github/actions/setup-ccache with: cache-key-prefix: ccache-gcc8-test @@ -77,5 +87,5 @@ jobs: CXX: g++-8 run: ci/scripts/build_paimon.sh --source_dir "$(pwd)" - name: Show ccache statistics - if: always() + if: always() && steps.setup-ccache.outcome == 'success' run: ccache -s diff --git a/LICENSE b/LICENSE index a672c4a10..e986c187b 100644 --- a/LICENSE +++ b/LICENSE @@ -426,6 +426,15 @@ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- +This product includes code from Alibaba Cloud OSS SDK for C++ V2 0.1.2. + +* Upstream source patch in cmake_modules/oss_sdk_v2.diff + +Home page: https://github.com/aliyun/alibabacloud-oss-cpp-sdk-v2 +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + This product includes code from xxHash. * MMH_rotl32 utility in src/paimon/common/utils/murmurhash_utils.h diff --git a/benchmark/parquet_format_benchmark.cpp b/benchmark/parquet_format_benchmark.cpp index eff80b39b..40094d31d 100644 --- a/benchmark/parquet_format_benchmark.cpp +++ b/benchmark/parquet_format_benchmark.cpp @@ -386,10 +386,11 @@ Result> MakeVectorColumn(int64_t num_rows, int64_t // A Parquet MAP is a LIST of a two-field STRUCT, so against LIST the difference is the key leaf. Result> MakeMapColumn(int64_t num_rows, int64_t offset, int32_t entries) { - auto key_builder = std::make_shared(); - auto item_builder = std::make_shared(); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, item_builder, - MapColumnType()); + arrow::MapBuilder builder(arrow::default_memory_pool(), + std::make_shared(), + std::make_shared(), MapColumnType()); + auto* key_builder = paimon::checked_cast(builder.key_builder()); + auto* item_builder = paimon::checked_cast(builder.item_builder()); PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Reserve(num_rows)); PAIMON_RETURN_NOT_OK_FROM_ARROW(key_builder->Reserve(num_rows * entries)); PAIMON_RETURN_NOT_OK_FROM_ARROW(item_builder->Reserve(num_rows * entries)); diff --git a/benchmark/parquet_format_benchmark_test.cpp b/benchmark/parquet_format_benchmark_test.cpp index 813c9b705..ca6397565 100644 --- a/benchmark/parquet_format_benchmark_test.cpp +++ b/benchmark/parquet_format_benchmark_test.cpp @@ -483,13 +483,15 @@ TEST_F(ParquetFormatBenchmarkTest, DecimalPrecisionRoundTrip) { // trip with the shape the read schema asks for. TEST_F(ParquetFormatBenchmarkTest, NestedRoundTrip) { constexpr int32_t kEntries = 4; - auto list_values = std::make_shared(); - arrow::ListBuilder list_builder(arrow::default_memory_pool(), list_values, + arrow::ListBuilder list_builder(arrow::default_memory_pool(), + std::make_shared(), arrow::list(arrow::int64())); - auto key_builder = std::make_shared(); - auto item_builder = std::make_shared(); - arrow::MapBuilder map_builder(arrow::default_memory_pool(), key_builder, item_builder, - arrow::map(arrow::utf8(), arrow::int64())); + auto* list_values = checked_cast(list_builder.value_builder()); + arrow::MapBuilder map_builder( + arrow::default_memory_pool(), std::make_shared(), + std::make_shared(), arrow::map(arrow::utf8(), arrow::int64())); + auto* key_builder = checked_cast(map_builder.key_builder()); + auto* item_builder = checked_cast(map_builder.item_builder()); for (int64_t i = 0; i < kRows; ++i) { ASSERT_TRUE(list_builder.Append().ok()); ASSERT_TRUE(map_builder.Append().ok()); diff --git a/ci/scripts/build_paimon.sh b/ci/scripts/build_paimon.sh index eacaf7485..06ea5d45a 100755 --- a/ci/scripts/build_paimon.sh +++ b/ci/scripts/build_paimon.sh @@ -145,9 +145,10 @@ if [[ "${enable_tsan}" == "true" ]]; then fi # CI always builds natively, so the host architecture is the target architecture. host_arch=$(uname -m) -if [[ "${host_arch}" != "x86_64" ]]; then +if [[ "$(uname -s)" != "Linux" || + ( "${host_arch}" != "x86_64" && "${host_arch}" != "aarch64" && "${host_arch}" != "arm64" ) ]]; then ENABLE_LUMINA="OFF" - echo "=== Lumina disabled: no prebuilt artifacts for ${host_arch} ===" + echo "=== Lumina disabled: no prebuilt artifacts for $(uname -s)-${host_arch} ===" fi if [[ "${verify_full_text_archives}" == "true" && "${ENABLE_FULL_TEXT}" != "ON" ]]; then echo "--verify_full_text_archives requires the full-text index, which this build disables" >&2 diff --git a/ci/scripts/download_release_asset.sh b/ci/scripts/download_release_asset.sh new file mode 100644 index 000000000..f8fb62a98 --- /dev/null +++ b/ci/scripts/download_release_asset.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Fetch a pinned GitHub release asset. Use the release API if the browser +# download endpoint times out, and publish a file only after verification. +set -euo pipefail + +url=$1 +output=$2 +sha256=$3 + +verify() { + if command -v sha256sum >/dev/null; then + echo "${sha256} $1" | sha256sum -c - + else + echo "${sha256} $1" | shasum -a 256 -c - + fi +} + +if [[ -f "${output}" ]] && verify "${output}"; then + exit 0 +fi + +mkdir -p "$(dirname "${output}")" +tmp=$(mktemp "${output}.XXXXXX") +trap 'rm -f "${tmp}"' EXIT +curl_args=(--proto '=https' --tlsv1.2 -fsSL --retry 3 --connect-timeout 15 --max-time 180) +if ! curl "${curl_args[@]}" "${url}" -o "${tmp}" || ! verify "${tmp}"; then + release=${url#https://github.com/} + repo=${release%%/releases/download/*} + asset=${release##*/} + tag=${release#*/releases/download/} + tag=${tag%/*} + asset_url=$(curl "${curl_args[@]}" "https://api.github.com/repos/${repo}/releases/tags/${tag}" | + python3 -c 'import json, sys +print(next(a["url"] for a in json.load(sys.stdin)["assets"] if a["name"] == sys.argv[1]))' "${asset}") + curl "${curl_args[@]}" -H 'Accept: application/octet-stream' "${asset_url}" -o "${tmp}" + verify "${tmp}" +fi +mv -f "${tmp}" "${output}" diff --git a/ci/scripts/setup_rust.sh b/ci/scripts/setup_rust.sh index a013e7f88..244f345bf 100755 --- a/ci/scripts/setup_rust.sh +++ b/ci/scripts/setup_rust.sh @@ -63,12 +63,11 @@ if ! command -v protoc >/dev/null 2>&1 || \ ;; esac protoc_home="${HOME}/.local/protoc-${PROTOC_VERSION}" - protoc_archive="/tmp/protoc-${PROTOC_VERSION}-${protoc_arch}.zip" + protoc_archive="${PAIMON_RELEASE_ASSET_DIR:-/tmp}/protoc-${PROTOC_VERSION}-${protoc_arch}.zip" if [[ ! -x "${protoc_home}/bin/protoc" ]]; then - curl --proto '=https' --tlsv1.2 -fL \ + bash "$(dirname "${BASH_SOURCE[0]}")/download_release_asset.sh" \ "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-${protoc_arch}.zip" \ - -o "${protoc_archive}" - echo "${protoc_sha256} ${protoc_archive}" | sha256sum --check + "${protoc_archive}" "${protoc_sha256}" mkdir -p "${protoc_home}" unzip -oq "${protoc_archive}" -d "${protoc_home}" fi diff --git a/ci/scripts/test_download_release_asset.sh b/ci/scripts/test_download_release_asset.sh new file mode 100644 index 000000000..3bf636fe1 --- /dev/null +++ b/ci/scripts/test_download_release_asset.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -euo pipefail + +script="$(dirname "${BASH_SOURCE[0]}")/download_release_asset.sh" +work=$(mktemp -d) +trap 'rm -rf "${work}"' EXIT +export PATH="${work}:${PATH}" +export REQUEST_LOG="${work}/requests" +cat > "${work}/curl" <<'MOCK' +#!/usr/bin/env bash +set -eu +echo request >> "${REQUEST_LOG}" +case "$*" in + *github.com/org/repo/releases/download/*) exit 22 ;; + *api.github.com/repos/org/repo/releases/tags/v1*) + echo '{"assets":[{"name":"archive","url":"https://api.github.com/assets/123"}]}' ;; + *application/octet-stream*https://api.github.com/assets/123*) + printf '%s' "${ASSET_CONTENT:-abc}" > "${!#}" ;; + *) exit 1 ;; +esac +MOCK +chmod +x "${work}/curl" +url=https://github.com/org/repo/releases/download/v1/archive +checksum=ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad +fetch() { + bash "${script}" "${url}" "${work}/archive" "${checksum}" +} + +fetch +[[ $(cat "${work}/archive") == abc ]] +fetch +[[ $(wc -l < "${REQUEST_LOG}") -eq 3 ]] +echo corrupt > "${work}/archive" +fetch +[[ $(cat "${work}/archive") == abc ]] +rm "${work}/archive" +if ASSET_CONTENT=invalid fetch; then + echo "Checksum mismatch accepted" >&2 + exit 1 +fi +[[ ! -e "${work}/archive" ]] diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake index 8aa8f1183..1149fed13 100644 --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake @@ -1089,8 +1089,10 @@ macro(build_boost) -sZLIB_LIBRARY_PATH=${ZLIB_PREFIX}/lib runtime-link=shared threading=multi variant=release --user-config=${BOOST_USER_CONFIG} - toolset=${BOOST_TOOLSET} cxxflags=${EP_CXX_FLAGS} - linkflags=${EP_CXX_FLAGS} install + toolset=${BOOST_TOOLSET} + cxxstd=${CMAKE_CXX_STANDARD} + cxxflags=${EP_CXX_FLAGS} linkflags=${EP_CXX_FLAGS} + install INSTALL_COMMAND "" DEPENDS zlib BUILD_BYPRODUCTS ${BOOST_BYPRODUCTS} @@ -1332,7 +1334,7 @@ endmacro() macro(build_lumina) # The release tarball ships one prebuilt artifact tree per platform; keep this # list in sync with the platforms the pinned release actually contains. - set(LUMINA_SUPPORTED_PLATFORMS "linux-x86_64") + set(LUMINA_SUPPORTED_PLATFORMS "linux-x86_64" "linux-aarch64") if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND PAIMON_TARGET_CPU_FAMILY STREQUAL "x86") set(LUMINA_ARTIFACT_PLATFORM "linux-x86_64") elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND PAIMON_TARGET_CPU_FAMILY STREQUAL @@ -1681,10 +1683,16 @@ macro(build_oss_sdk_v2) "-DCURL_LIBRARY_RELEASE=${OSS_SDK_V2_CURL_LIBRARY}") endif() + # Avoid copying the metadata variant, which triggers GCC 12's + # -Wmaybe-uninitialized warning in optimized builds. + set(OSS_SDK_V2_PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/oss_sdk_v2.diff") externalproject_add(oss_sdk_v2_ep ${EP_COMMON_OPTIONS} URL ${OSS_SDK_V2_SOURCE_URL} URL_HASH "SHA256=${PAIMON_OSS_SDK_V2_BUILD_SHA256_CHECKSUM}" + LOG_PATCH ON + PATCH_COMMAND ${CMAKE_COMMAND} -E chdir bash -c + "[ -f .patched ] && echo ' patch already applied, ignore...' || patch -s -N -p1 -i '${OSS_SDK_V2_PATCH_FILE}' && touch .patched" CMAKE_ARGS ${OSS_SDK_V2_CMAKE_ARGS} ${THIRDPARTY_LOG_OPTIONS} BUILD_BYPRODUCTS "${OSS_SDK_V2_STATIC_LIB}") diff --git a/cmake_modules/oss_sdk_v2.diff b/cmake_modules/oss_sdk_v2.diff new file mode 100644 index 000000000..1418d92f6 --- /dev/null +++ b/cmake_modules/oss_sdk_v2.diff @@ -0,0 +1,11 @@ +diff --git a/sdk/src/internal/ClientImplBase.cpp b/sdk/src/internal/ClientImplBase.cpp +--- a/sdk/src/internal/ClientImplBase.cpp ++++ b/sdk/src/internal/ClientImplBase.cpp +@@ -354,6 +354,6 @@ + context.signingContext.expirationInEpoch = 0; + if (input.opMetadata.find("EXPIRATION_TIME") != input.opMetadata.end()) { +- auto value = input.opMetadata.at("EXPIRATION_TIME"); ++ const auto& value = input.opMetadata.at("EXPIRATION_TIME"); + if (auto* val = std::get_if(&value)) { + context.signingContext.expirationInEpoch = static_cast(*val); + } diff --git a/docs/source/building.rst b/docs/source/building.rst index b0fe95e44..14973d13f 100644 --- a/docs/source/building.rst +++ b/docs/source/building.rst @@ -89,10 +89,9 @@ x86: the baseline stays whatever the toolchain defaults to. No persisted value depends on it, in particular the checksum written to SST files and to the B-tree global index is the same however Paimon was compiled. -One optional component is architecture-restricted: -``-DPAIMON_ENABLE_LUMINA=ON`` consumes a prebuilt binary that is only -published for Linux ``x86_64``, so configuring it on any other platform fails -with an explicit error. +The bundled Lumina package supports Linux ``x86_64`` and ``aarch64``. +Enable it with ``-DPAIMON_ENABLE_LUMINA=ON``. CMake selects the library +for the target architecture. Cross compilation is not wired up end to end: the bundled third-party builds are handed only the C and C++ compilers, without a toolchain file, sysroot or find @@ -191,7 +190,7 @@ to ``ON``, while every other component below defaults to ``OFF``. * ``-DPAIMON_ENABLE_OSS=ON``: Support for Alibaba Cloud OSS through OSS SDK V2 * ``-DPAIMON_ENABLE_S3=ON``: Support for Amazon S3-compatible filesystems * ``-DPAIMON_ENABLE_LUMINA=ON``: Support for the Lumina vector index. Requires - Linux ``x86_64``; see :ref:`cpp-building-platforms`. + Linux ``x86_64`` or ``aarch64``. See :ref:`cpp-building-platforms`. * ``-DPAIMON_ENABLE_LUCENE=ON``: Support for Lucene full-text search indexes * ``-DPAIMON_ENABLE_FULL_TEXT=ON``: Experimental full-text global index (index type ``full-text``) backed by the ``paimon-full-text-index`` 0.1.0 Rust engine. It uses the same index diff --git a/src/paimon/common/file_index/bitmap/bitmap_file_index_test.cpp b/src/paimon/common/file_index/bitmap/bitmap_file_index_test.cpp index e8f74c83d..f2f89718a 100644 --- a/src/paimon/common/file_index/bitmap/bitmap_file_index_test.cpp +++ b/src/paimon/common/file_index/bitmap/bitmap_file_index_test.cpp @@ -91,12 +91,12 @@ class BitmapIndexTest : public ::testing::Test { template Result> CreateArray(const std::shared_ptr& type, const std::vector& values) const { - auto value_builder = std::make_shared(); + ArrowBuilder value_builder; for (ValueType value : values) { - PAIMON_RETURN_NOT_OK_FROM_ARROW(value_builder->Append(value)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(value_builder.Append(value)); } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr value_array, - value_builder->Finish()); + value_builder.Finish()); PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( std::shared_ptr struct_array, arrow::StructArray::Make({value_array}, {arrow::field("f0", type)})); diff --git a/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index_test.cpp b/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index_test.cpp index 794f25c5b..c00c393d9 100644 --- a/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index_test.cpp +++ b/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index_test.cpp @@ -128,16 +128,16 @@ Result> RangeBitmapFileIndexTest::Cr const std::set& null_indices, const std::map& options, PAIMON_UNIQUE_PTR* serialized_bytes_out) { // Create Arrow array from test data, replacing values at null_indices with NULL - auto builder = std::make_shared(); + ArrowBuilder builder; for (size_t i = 0; i < test_data.size(); ++i) { if (null_indices.count(static_cast(i)) > 0) { - PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->AppendNull()); + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.AppendNull()); } else { - PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Append(test_data[i])); + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append(test_data[i])); } } std::shared_ptr arrow_array; - PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Finish(&arrow_array)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&arrow_array)); return CreateReaderFromArray(arrow_type, arrow_array, options, serialized_bytes_out); } diff --git a/src/paimon/common/file_index/rangebitmap/range_bitmap_io_test.cpp b/src/paimon/common/file_index/rangebitmap/range_bitmap_io_test.cpp index fd8e0c888..9ae5fbf44 100644 --- a/src/paimon/common/file_index/rangebitmap/range_bitmap_io_test.cpp +++ b/src/paimon/common/file_index/rangebitmap/range_bitmap_io_test.cpp @@ -57,12 +57,12 @@ class RangeBitmapIoTest : public ::testing::Test { const std::shared_ptr& arrow_type, const std::vector& test_data) { // Create Arrow array from test data - auto builder = std::make_shared(); + ArrowBuilder builder; for (const auto& value : test_data) { - PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Append(value)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append(value)); } std::shared_ptr arrow_array; - PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Finish(&arrow_array)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&arrow_array)); // Wrap in StructArray const std::shared_ptr field = arrow::field("test_field", arrow_type); diff --git a/src/paimon/core/casting/string_to_boolean_cast_executor.cpp b/src/paimon/core/casting/string_to_boolean_cast_executor.cpp index 951682f30..528821797 100644 --- a/src/paimon/core/casting/string_to_boolean_cast_executor.cpp +++ b/src/paimon/core/casting/string_to_boolean_cast_executor.cpp @@ -64,10 +64,10 @@ Result> StringToBooleanCastExecutor::Cast( const std::shared_ptr& array, const std::shared_ptr& target_type, arrow::MemoryPool* pool) const { auto* string_array = checked_cast(array.get()); - auto bool_builder = std::make_shared(pool); + arrow::BooleanBuilder bool_builder(pool); for (int64_t i = 0; i < string_array->length(); ++i) { if (string_array->IsNull(i)) { - PAIMON_RETURN_NOT_OK_FROM_ARROW(bool_builder->AppendNull()); + PAIMON_RETURN_NOT_OK_FROM_ARROW(bool_builder.AppendNull()); } else { std::optional bool_value = StringUtils::StringToValue(string_array->GetString(i)); @@ -76,11 +76,11 @@ Result> StringToBooleanCastExecutor::Cast( "StringToBooleanCastExecutor cast failed: STRING '{}' cannot cast to BOOLEAN", string_array->GetString(i))); } - PAIMON_RETURN_NOT_OK_FROM_ARROW(bool_builder->Append(bool_value.value())); + PAIMON_RETURN_NOT_OK_FROM_ARROW(bool_builder.Append(bool_value.value())); } } std::shared_ptr casted_array; - PAIMON_RETURN_NOT_OK_FROM_ARROW(bool_builder->Finish(&casted_array)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(bool_builder.Finish(&casted_array)); return casted_array; } diff --git a/src/paimon/core/casting/string_to_date_cast_executor.cpp b/src/paimon/core/casting/string_to_date_cast_executor.cpp index 66c4cbbed..372ad5758 100644 --- a/src/paimon/core/casting/string_to_date_cast_executor.cpp +++ b/src/paimon/core/casting/string_to_date_cast_executor.cpp @@ -57,18 +57,18 @@ Result> StringToDateCastExecutor::Cast( const std::shared_ptr& array, const std::shared_ptr& target_type, arrow::MemoryPool* pool) const { auto* string_array = checked_cast(array.get()); - auto date_builder = std::make_shared(pool); + arrow::Date32Builder date_builder(pool); for (int64_t i = 0; i < string_array->length(); ++i) { if (string_array->IsNull(i)) { - PAIMON_RETURN_NOT_OK_FROM_ARROW(date_builder->AppendNull()); + PAIMON_RETURN_NOT_OK_FROM_ARROW(date_builder.AppendNull()); } else { PAIMON_ASSIGN_OR_RAISE(int32_t date_value, StringUtils::StringToDate(string_array->GetString(i))); - PAIMON_RETURN_NOT_OK_FROM_ARROW(date_builder->Append(date_value)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(date_builder.Append(date_value)); } } std::shared_ptr casted_array; - PAIMON_RETURN_NOT_OK_FROM_ARROW(date_builder->Finish(&casted_array)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(date_builder.Finish(&casted_array)); return casted_array; } diff --git a/src/paimon/core/mergetree/compact/aggregate/field_ignore_retract_agg_test.cpp b/src/paimon/core/mergetree/compact/aggregate/field_ignore_retract_agg_test.cpp index 80348d64c..f084cdefd 100644 --- a/src/paimon/core/mergetree/compact/aggregate/field_ignore_retract_agg_test.cpp +++ b/src/paimon/core/mergetree/compact/aggregate/field_ignore_retract_agg_test.cpp @@ -82,14 +82,14 @@ TEST(FieldIgnoreRetractAggTest, ReversedAggBypassesWrappedOverride) { ASSERT_OK_AND_ASSIGN( std::unique_ptr collect_agg, FieldCollectAgg::Create(arrow::list(arrow::int32()), options, "f0", GetDefaultPool())); - auto agg = std::make_unique(std::move(collect_agg)); + FieldIgnoreRetractAgg agg(std::move(collect_agg)); VariantType accumulator = VariantType(checked_pointer_cast( std::make_shared(std::vector{int32_t{1}, int32_t{2}}))); VariantType input = VariantType(checked_pointer_cast( std::make_shared(std::vector{int32_t{3}, int32_t{4}}))); - ASSERT_OK_AND_ASSIGN(VariantType result, agg->AggReversed(accumulator, input)); + ASSERT_OK_AND_ASSIGN(VariantType result, agg.AggReversed(accumulator, input)); auto values = DataDefine::GetVariantValue>(result); ASSERT_EQ(4, values->Size()); ASSERT_EQ(3, values->GetInt(0)); diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_test.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_test.cpp index 9186eb1c4..cc8b3ef41 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_test.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_test.cpp @@ -253,8 +253,8 @@ class MergeTreeCompactManagerTest : public testing::Test { std::make_shared()); ASSERT_OK(manager->TriggerCompaction(/*full_compaction=*/false)); - ASSERT_OK_AND_ASSIGN(auto compact_result, manager->GetCompactionResult(/*blocking=*/true)); - (void)compact_result; + auto compact_result = manager->GetCompactionResult(/*blocking=*/true); + ASSERT_OK(compact_result); std::vector outputs; for (const auto& file : levels->AllFiles()) { diff --git a/src/paimon/core/mergetree/compact/universal_compaction_test.cpp b/src/paimon/core/mergetree/compact/universal_compaction_test.cpp index 9b2e6cd1f..a93c76ba1 100644 --- a/src/paimon/core/mergetree/compact/universal_compaction_test.cpp +++ b/src/paimon/core/mergetree/compact/universal_compaction_test.cpp @@ -273,7 +273,7 @@ TEST_F(UniversalCompactionTest, TestSizeAmplification) { if (unit) { auto files = GetFileSizeVecFromCompactUnit(unit.value()); int64_t total_size = std::accumulate(files.begin(), files.end(), 0l); - sizes = {total_size}; + sizes.assign(1, total_size); } ASSERT_EQ(sizes, expected_sizes); }; diff --git a/src/paimon/core/postpone/postpone_bucket_writer_test.cpp b/src/paimon/core/postpone/postpone_bucket_writer_test.cpp index dd20eca48..d0305204b 100644 --- a/src/paimon/core/postpone/postpone_bucket_writer_test.cpp +++ b/src/paimon/core/postpone/postpone_bucket_writer_test.cpp @@ -149,14 +149,15 @@ class PostponeBucketWriterTest : public ::testing::Test, }; std::vector GetTestValuesForPostponeBucketWriterTest() { - std::vector values = {"parquet"}; + return { + "parquet", #ifdef PAIMON_ENABLE_ORC - values.emplace_back("orc"); + "orc", #endif #ifdef PAIMON_ENABLE_AVRO - values.emplace_back("avro"); + "avro", #endif - return values; + }; } INSTANTIATE_TEST_SUITE_P(FileFormat, PostponeBucketWriterTest, diff --git a/src/paimon/core/realtime/arrow_realtime_store_test.cpp b/src/paimon/core/realtime/arrow_realtime_store_test.cpp index 27dd14b9d..a21c453bf 100644 --- a/src/paimon/core/realtime/arrow_realtime_store_test.cpp +++ b/src/paimon/core/realtime/arrow_realtime_store_test.cpp @@ -135,8 +135,8 @@ class ArrowRealtimeStoreTest : public testing::Test { } // namespace TEST_F(ArrowRealtimeStoreTest, TestWriteValidationAndSeal) { - ASSERT_OK_AND_ASSIGN(std::optional> empty_segment, - store_->SealForCommit()); + std::optional> empty_segment; + ASSERT_OK_AND_ASSIGN(empty_segment, store_->SealForCommit()); ASSERT_FALSE(empty_segment.has_value()); ASSERT_NOK_WITH_MSG(store_->Write(RealtimeWriteBatch{nullptr, OffsetRange(0, 1)}), @@ -181,8 +181,8 @@ TEST_F(ArrowRealtimeStoreTest, TestDataUsageTracksBuildingSealedAndReclaimedData ASSERT_EQ(usage.building_memory_bytes, store_->GetMemoryUsage()); const uint64_t first_segment_memory = usage.building_memory_bytes; - ASSERT_OK_AND_ASSIGN(std::optional> segment, - store_->SealForCommit()); + std::optional> segment; + ASSERT_OK_AND_ASSIGN(segment, store_->SealForCommit()); ASSERT_TRUE(segment.has_value()); usage = store_->GetDataUsage(); ASSERT_EQ(0, usage.building_memory_bytes); diff --git a/src/paimon/core/utils/nested_projection_utils_test.cpp b/src/paimon/core/utils/nested_projection_utils_test.cpp index 3d33910be..607d72ff2 100644 --- a/src/paimon/core/utils/nested_projection_utils_test.cpp +++ b/src/paimon/core/utils/nested_projection_utils_test.cpp @@ -895,9 +895,11 @@ class NestedProjectionUtilsMapArrayTest : public ::testing::Test { static std::shared_ptr BuildStringInt32MapArray( const std::vector>>& maps, const std::vector& null_mask = {}) { - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - arrow::MapBuilder map_builder(arrow::default_memory_pool(), key_builder, value_builder); + arrow::MapBuilder map_builder(arrow::default_memory_pool(), + std::make_shared(), + std::make_shared()); + auto* key_builder = checked_cast(map_builder.key_builder()); + auto* value_builder = checked_cast(map_builder.item_builder()); for (size_t i = 0; i < maps.size(); ++i) { if (!null_mask.empty() && !null_mask[i]) { EXPECT_TRUE(map_builder.AppendNull().ok()); diff --git a/src/paimon/format/blob/blob_format_writer_test.cpp b/src/paimon/format/blob/blob_format_writer_test.cpp index 2d06d94b9..866411363 100644 --- a/src/paimon/format/blob/blob_format_writer_test.cpp +++ b/src/paimon/format/blob/blob_format_writer_test.cpp @@ -1365,11 +1365,10 @@ class BlobFormatWriterArrayBlobTest : public BlobFormatWriterTestBase { Result> MakeArrayBlobRow( const std::optional& elements) const { - auto list_type = checked_pointer_cast(struct_type_->field(0)->type()); - auto list_builder = std::make_shared( - arrow::default_memory_pool(), std::make_shared(), list_type); - arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(), - {list_builder}); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::unique_ptr builder, + arrow::MakeBuilder(struct_type_)); + auto& struct_builder = checked_cast(*builder); + auto* list_builder = checked_cast(struct_builder.field_builder(0)); auto blob_builder = checked_cast(list_builder->value_builder()); PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder.Append()); if (!elements) { diff --git a/src/paimon/format/orc/read_range_generator_test.cpp b/src/paimon/format/orc/read_range_generator_test.cpp index 28fd96783..c43bc3b75 100644 --- a/src/paimon/format/orc/read_range_generator_test.cpp +++ b/src/paimon/format/orc/read_range_generator_test.cpp @@ -203,9 +203,7 @@ TEST_F(ReadRangeGeneratorTest, TestBytesPerGroup) { } TEST_F(ReadRangeGeneratorTest, TestGenReadRanges) { - ReadRangeGenerator::ReaderMeta meta; - meta.rows_per_group = 10000; - meta.rows_per_stripes = {20480, 20480, 20480, 20480, 2080}; + const ReadRangeGenerator::ReaderMeta meta{10000, {20480, 20480, 20480, 20480, 2080}}; auto read_ranges_vec = ReadRangeGenerator::DoGenReadRanges( /*begin_row_num=*/0, /*end_row_num=*/84000, /*range_size=*/10000, meta); ASSERT_TRUE(CheckEqual({{0, 10000}, @@ -225,45 +223,35 @@ TEST_F(ReadRangeGeneratorTest, TestGenReadRanges) { } TEST_F(ReadRangeGeneratorTest, TestGenReadRangesBasic) { - ReadRangeGenerator::ReaderMeta meta; - meta.rows_per_group = 3; - meta.rows_per_stripes = {4, 6}; + const ReadRangeGenerator::ReaderMeta meta{3, {4, 6}}; auto read_ranges_vec = ReadRangeGenerator::DoGenReadRanges( /*begin_row_num=*/0, /*end_row_num=*/10, /*range_size=*/3, meta); ASSERT_TRUE(CheckEqual({{0, 3}, {3, 4}, {4, 7}, {7, 10}}, read_ranges_vec)); } TEST_F(ReadRangeGeneratorTest, TestGenReadRangesMultiStripes) { - ReadRangeGenerator::ReaderMeta meta; - meta.rows_per_group = 4; - meta.rows_per_stripes = {10, 15}; + const ReadRangeGenerator::ReaderMeta meta{4, {10, 15}}; auto read_ranges_vec = ReadRangeGenerator::DoGenReadRanges( /*begin_row_num=*/5, /*end_row_num=*/25, /*range_size=*/5, meta); ASSERT_TRUE(CheckEqual({{5, 10}, {10, 15}, {15, 20}, {20, 25}}, read_ranges_vec)); } TEST_F(ReadRangeGeneratorTest, TestGenReadRangesInvalid) { - ReadRangeGenerator::ReaderMeta meta; - meta.rows_per_group = 4; - meta.rows_per_stripes = {10, 15}; + const ReadRangeGenerator::ReaderMeta meta{4, {10, 15}}; auto read_ranges_vec = ReadRangeGenerator::DoGenReadRanges( /*begin_row_num=*/3, /*end_row_num=*/2, /*range_size=*/5, meta); ASSERT_TRUE(CheckEqual({}, read_ranges_vec)); } TEST_F(ReadRangeGeneratorTest, TestGenReadRangesStripeBound) { - ReadRangeGenerator::ReaderMeta meta; - meta.rows_per_group = 5; - meta.rows_per_stripes = {10, 10}; + const ReadRangeGenerator::ReaderMeta meta{5, {10, 10}}; auto read_ranges_vec = ReadRangeGenerator::DoGenReadRanges( /*begin_row_num=*/3, /*end_row_num=*/18, /*range_size=*/5, meta); ASSERT_TRUE(CheckEqual({{3, 8}, {8, 10}, {10, 15}, {15, 18}}, read_ranges_vec)); } TEST_F(ReadRangeGeneratorTest, TestGenReadRangesNotMatch) { - ReadRangeGenerator::ReaderMeta meta; - meta.rows_per_group = 2; - meta.rows_per_stripes = {4, 6}; + const ReadRangeGenerator::ReaderMeta meta{2, {4, 6}}; auto read_ranges_vec = ReadRangeGenerator::DoGenReadRanges( /*begin_row_num=*/10, /*end_row_num=*/10, /*range_size=*/2, meta); ASSERT_TRUE(CheckEqual({}, read_ranges_vec)); diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp index b8af016b3..cab1c1442 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp @@ -42,6 +42,7 @@ #include "gtest/gtest.h" #include "paimon/common/utils/arrow/arrow_input_stream_adapter.h" #include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/stream_utils.h" #include "paimon/defs.h" #include "paimon/format/parquet/parquet_file_batch_reader.h" @@ -1216,8 +1217,9 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedStructColumnOnlyReadIdField) { /// Helper: build a list array (without id column). /// tags[i] = [i*10, i*10+1], for i in [0, N). static std::shared_ptr MakeListColumnData(int32_t num_rows) { - auto value_builder = std::make_shared(); - arrow::ListBuilder list_builder(arrow::default_memory_pool(), value_builder); + arrow::ListBuilder list_builder(arrow::default_memory_pool(), + std::make_shared()); + auto* value_builder = checked_cast(list_builder.value_builder()); for (int32_t i = 0; i < num_rows; ++i) { EXPECT_TRUE(list_builder.Append().ok()); EXPECT_TRUE(value_builder->Append(i * 10).ok()); @@ -1229,9 +1231,11 @@ static std::shared_ptr MakeListColumnData(int32_t num_rows) { /// Helper: build a map array (without id column). /// props[i] = {"k_i": i * 100}, for i in [0, N). static std::shared_ptr MakeMapColumnData(int32_t num_rows) { - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - arrow::MapBuilder map_builder(arrow::default_memory_pool(), key_builder, value_builder); + arrow::MapBuilder map_builder(arrow::default_memory_pool(), + std::make_shared(), + std::make_shared()); + auto* key_builder = checked_cast(map_builder.key_builder()); + auto* value_builder = checked_cast(map_builder.item_builder()); for (int32_t i = 0; i < num_rows; ++i) { EXPECT_TRUE(map_builder.Append().ok()); std::string key = "k_" + std::to_string(i); @@ -2567,10 +2571,11 @@ static std::shared_ptr MakeMisalignedNestedDataWithNulls(int // s: struct with nulls at both leaf and struct level. auto field_x = arrow::field("x", arrow::int32()); auto field_y = arrow::field("y", arrow::int64()); - auto x_builder = std::make_shared(); - auto y_builder = std::make_shared(); - arrow::StructBuilder s_builder(arrow::struct_({field_x, field_y}), arrow::default_memory_pool(), - {x_builder, y_builder}); + arrow::StructBuilder s_builder( + arrow::struct_({field_x, field_y}), arrow::default_memory_pool(), + {std::make_shared(), std::make_shared()}); + auto* x_builder = checked_cast(s_builder.field_builder(0)); + auto* y_builder = checked_cast(s_builder.field_builder(1)); for (int32_t i = 0; i < num_rows; ++i) { if (i % 11 == 0) { // AppendNull() also appends nulls to the child builders, keeping lengths in sync. @@ -2592,8 +2597,9 @@ static std::shared_ptr MakeMisalignedNestedDataWithNulls(int auto s_array = s_builder.Finish().ValueOrDie(); // tags: list with null lists. - auto item_builder = std::make_shared(); - arrow::ListBuilder tags_builder(arrow::default_memory_pool(), item_builder); + arrow::ListBuilder tags_builder(arrow::default_memory_pool(), + std::make_shared()); + auto* item_builder = checked_cast(tags_builder.value_builder()); for (int32_t i = 0; i < num_rows; ++i) { if (i % 6 == 0) { EXPECT_TRUE(tags_builder.AppendNull().ok()); @@ -2606,9 +2612,11 @@ static std::shared_ptr MakeMisalignedNestedDataWithNulls(int auto tags_array = tags_builder.Finish().ValueOrDie(); // props: map with null maps. - auto map_key_builder = std::make_shared(); - auto map_val_builder = std::make_shared(); - arrow::MapBuilder props_builder(arrow::default_memory_pool(), map_key_builder, map_val_builder); + arrow::MapBuilder props_builder(arrow::default_memory_pool(), + std::make_shared(), + std::make_shared()); + auto* map_key_builder = checked_cast(props_builder.key_builder()); + auto* map_val_builder = checked_cast(props_builder.item_builder()); for (int32_t i = 0; i < num_rows; ++i) { if (i % 8 == 0) { EXPECT_TRUE(props_builder.AppendNull().ok()); diff --git a/src/paimon/indexer/lumina/lumina_api_test.cpp b/src/paimon/indexer/lumina/lumina_api_test.cpp index c16aa22b7..8b7b6a480 100644 --- a/src/paimon/indexer/lumina/lumina_api_test.cpp +++ b/src/paimon/indexer/lumina/lumina_api_test.cpp @@ -41,7 +41,7 @@ class LuminaInterfaceTest : public ::testing::Test { { // create writer ::lumina::api::BuilderOptions builder_options; builder_options.Set(::lumina::core::kIndexType, ::lumina::core::kIndexTypeBruteforce) - .Set(::lumina::core::kDimension, 4) + .Set(::lumina::core::kDimension, static_cast(4)) .Set(::lumina::core::kDistanceMetric, ::lumina::core::kDistanceL2) .Set(::lumina::core::kEncodingType, ::lumina::core::kEncodingRawf32); auto builder_result = @@ -80,7 +80,7 @@ class LuminaInterfaceTest : public ::testing::Test { // create reader ::lumina::api::SearcherOptions searcher_options; searcher_options.Set(::lumina::core::kIndexType, ::lumina::core::kIndexTypeBruteforce) - .Set(::lumina::core::kDimension, 4); + .Set(::lumina::core::kDimension, static_cast(4)); auto reader_result = ::lumina::api::LuminaSearcher::Create(searcher_options, memory_resource); ASSERT_TRUE(reader_result.IsOk()); @@ -100,9 +100,10 @@ class LuminaInterfaceTest : public ::testing::Test { auto search = [&](int32_t parallel_number) { ::lumina::api::Query query(query_.data(), query_.size()); ::lumina::api::SearchOptions search_options; - search_options.Set(::lumina::core::kTopK, topk); + search_options.Set(::lumina::core::kTopK, static_cast(topk)); if (parallel_number > 0) { - search_options.Set(::lumina::core::kSearchParallelNumber, parallel_number); + search_options.Set(::lumina::core::kSearchParallelNumber, + static_cast(parallel_number)); } if (!filter) { diff --git a/src/paimon/indexer/lumina/lumina_global_index_test.cpp b/src/paimon/indexer/lumina/lumina_global_index_test.cpp index b8cb20dfe..99a7c8e10 100644 --- a/src/paimon/indexer/lumina/lumina_global_index_test.cpp +++ b/src/paimon/indexer/lumina/lumina_global_index_test.cpp @@ -28,6 +28,7 @@ #include "lumina/core/Constants.h" #include "lumina/extensions/experimental/BuildCombinedExtensionV0.h" #include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" @@ -235,8 +236,9 @@ class LuminaGlobalIndexTest : public ::testing::Test { std::shared_ptr CreateRandomVector(int32_t element_size, int32_t dimension) const { int64_t total_values = element_size * dimension; - auto float_builder = std::make_shared(); - arrow::ListBuilder list_builder(arrow::default_memory_pool(), float_builder); + arrow::ListBuilder list_builder(arrow::default_memory_pool(), + std::make_shared()); + auto* float_builder = checked_cast(list_builder.value_builder()); std::random_device rd; std::mt19937 gen(rd()); @@ -454,13 +456,13 @@ TEST_F(LuminaGlobalIndexTest, TestCheckpointFileManagement) { CreateCheckpointPathFactory(test_root, "lumina", "f0", Range(10, 20))); ASSERT_OK_AND_ASSIGN(std::shared_ptr path_factory, CreateFileStorePathFactory(test_root)); - auto file_manager = std::make_shared( - fs_, path_factory->CreateGlobalIndexFileFactory(), std::move(checkpoint_path_factory)); + GlobalIndexFileManager file_manager(fs_, path_factory->CreateGlobalIndexFileFactory(), + std::move(checkpoint_path_factory)); - ASSERT_OK_AND_ASSIGN(bool checkpoint_exists, file_manager->CheckpointExists()); + ASSERT_OK_AND_ASSIGN(bool checkpoint_exists, file_manager.CheckpointExists()); ASSERT_FALSE(checkpoint_exists); ASSERT_OK_AND_ASSIGN(std::unique_ptr first_output, - file_manager->CreateCheckpointOutputStream()); + file_manager.CreateCheckpointOutputStream()); ASSERT_OK_AND_ASSIGN(std::string first_path, first_output->GetUri()); ASSERT_TRUE(StringUtils::StartsWith(PathUtil::GetName(first_path), prefix)); ASSERT_TRUE(StringUtils::EndsWith(first_path, "-0.index.ckpt")); @@ -469,7 +471,7 @@ TEST_F(LuminaGlobalIndexTest, TestCheckpointFileManagement) { ASSERT_OK(first_output->Close()); ASSERT_OK_AND_ASSIGN(std::unique_ptr second_output, - file_manager->CreateCheckpointOutputStream()); + file_manager.CreateCheckpointOutputStream()); ASSERT_OK_AND_ASSIGN(std::string second_path, second_output->GetUri()); ASSERT_TRUE(StringUtils::StartsWith(PathUtil::GetName(second_path), prefix)); ASSERT_TRUE(StringUtils::EndsWith(second_path, "-1.index.ckpt")); @@ -491,24 +493,24 @@ TEST_F(LuminaGlobalIndexTest, TestCheckpointFileManagement) { ASSERT_OK_AND_ASSIGN(checkpoint_path_factory, CreateCheckpointPathFactory(test_root, "lumina", "f0", Range(10, 20))); - file_manager = std::make_shared( - fs_, path_factory->CreateGlobalIndexFileFactory(), std::move(checkpoint_path_factory)); + GlobalIndexFileManager reopened_manager(fs_, path_factory->CreateGlobalIndexFileFactory(), + std::move(checkpoint_path_factory)); ASSERT_OK_AND_ASSIGN(std::unique_ptr latest_input, - file_manager->OpenCheckpointInputStream()); + reopened_manager.OpenCheckpointInputStream()); char buffer[6]; ASSERT_OK_AND_ASSIGN(int64_t read_bytes, latest_input->Read(buffer, sizeof(buffer))); ASSERT_EQ(read_bytes, 6); ASSERT_EQ(std::string(buffer, sizeof(buffer)), "id-ten"); ASSERT_OK(latest_input->Close()); ASSERT_OK_AND_ASSIGN(std::unique_ptr next_output, - file_manager->CreateCheckpointOutputStream()); + reopened_manager.CreateCheckpointOutputStream()); ASSERT_OK_AND_ASSIGN(std::string next_path, next_output->GetUri()); ASSERT_TRUE(StringUtils::EndsWith(next_path, "-11.index.ckpt")); ASSERT_OK(next_output->Close()); - ASSERT_OK(file_manager->DeleteCheckpoint()); - ASSERT_OK_AND_ASSIGN(checkpoint_exists, file_manager->CheckpointExists()); + ASSERT_OK(reopened_manager.DeleteCheckpoint()); + ASSERT_OK_AND_ASSIGN(checkpoint_exists, reopened_manager.CheckpointExists()); ASSERT_FALSE(checkpoint_exists); ASSERT_OK_AND_ASSIGN(bool unrelated_exists, fs_->Exists(unrelated_file)); ASSERT_TRUE(unrelated_exists); diff --git a/src/paimon/indexer/lumina/lumina_tag_utils.cpp b/src/paimon/indexer/lumina/lumina_tag_utils.cpp index 979be3310..262193720 100644 --- a/src/paimon/indexer/lumina/lumina_tag_utils.cpp +++ b/src/paimon/indexer/lumina/lumina_tag_utils.cpp @@ -155,8 +155,9 @@ Status AppendTagValue(const std::shared_ptr& array, int64_t index, if (array->IsNull(index)) { return Status::OK(); } - auto validate_array_type = [&](arrow::Type::type expected_type, - const char* value_type_name) -> Status { + // The int32_t specialization validates its type in the switch below. + [[maybe_unused]] auto validate_array_type = [&](arrow::Type::type expected_type, + const char* value_type_name) -> Status { if (array->type_id() != expected_type) { return Status::Invalid(fmt::format("lumina {} tag field has unsupported arrow type {}", value_type_name, array->type()->ToString())); diff --git a/src/paimon/rest/rest_token_file_system_test.cpp b/src/paimon/rest/rest_token_file_system_test.cpp index da749a5b3..8a062a07d 100644 --- a/src/paimon/rest/rest_token_file_system_test.cpp +++ b/src/paimon/rest/rest_token_file_system_test.cpp @@ -635,10 +635,8 @@ TEST_F(RestTokenFileSystemTest, DefaultCacheEvictsAndRebuildsBackendWithoutReloa std::string path = WriteFile("data", "paimon"); ASSERT_OK(fs->Exists(path).status()); ASSERT_OK_AND_ASSIGN(RestToken token, fs->ValidToken()); - std::optional> cached = cache->GetIfPresent(token); - ASSERT_TRUE(cached.has_value()); - std::weak_ptr old_backend = cached.value(); - cached.reset(); + std::weak_ptr old_backend = cache->GetIfPresent(token).value_or(nullptr); + ASSERT_FALSE(old_backend.expired()); // fill the cache with other credentials to reach its real default capacity; the file // system asked for the credentials stays in use, only the delegate it built is evicted diff --git a/src/paimon/testing/utils/variant_test_data.h b/src/paimon/testing/utils/variant_test_data.h index e78b0cb96..72637e48b 100644 --- a/src/paimon/testing/utils/variant_test_data.h +++ b/src/paimon/testing/utils/variant_test_data.h @@ -25,6 +25,7 @@ #include "arrow/api.h" #include "paimon/common/data/variant/generic_variant.h" #include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/checked_cast.h" #include "paimon/memory/memory_pool.h" #include "paimon/result.h" @@ -50,10 +51,11 @@ inline Result> VariantTestData::BuildVariant const std::shared_ptr& variant_field, const std::vector& jsons, const std::shared_ptr& pool, int32_t id_offset) { arrow::Int32Builder id_builder; - auto value_builder = std::make_shared(); - auto metadata_builder = std::make_shared(); - arrow::StructBuilder variant_builder(variant_field->type(), arrow::default_memory_pool(), - {value_builder, metadata_builder}); + arrow::StructBuilder variant_builder( + variant_field->type(), arrow::default_memory_pool(), + {std::make_shared(), std::make_shared()}); + auto* value_builder = checked_cast(variant_builder.field_builder(0)); + auto* metadata_builder = checked_cast(variant_builder.field_builder(1)); for (size_t i = 0; i < jsons.size(); ++i) { PAIMON_RETURN_NOT_OK_FROM_ARROW(id_builder.Append(id_offset + static_cast(i))); if (jsons[i] == nullptr) { diff --git a/test/inte/append_compaction_inte_test.cpp b/test/inte/append_compaction_inte_test.cpp index 1424533de..ae5dcf086 100644 --- a/test/inte/append_compaction_inte_test.cpp +++ b/test/inte/append_compaction_inte_test.cpp @@ -139,7 +139,10 @@ class AppendCompactionInteTest : public testing::Test, for (const auto& data_file : restore_files->DataFiles()) { ASSERT_OK(dv_maintainer->NotifyNewDeletion(data_file->file_name, 0)); } - ASSERT_OK_AND_ASSIGN(auto index_file_meta, dv_maintainer->WriteDeletionVectorsIndex()); + auto index_file_meta_result = dv_maintainer->WriteDeletionVectorsIndex(); + ASSERT_OK(index_file_meta_result); + const auto& index_file_meta = index_file_meta_result.value(); + ASSERT_TRUE(index_file_meta.has_value()); auto commit_message = std::make_shared( partition, bucket, 2, DataIncrement({}, {}, {}, {index_file_meta.value()}, {}), diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp index c721658f5..fd706e640 100644 --- a/test/inte/blob_table_inte_test.cpp +++ b/test/inte/blob_table_inte_test.cpp @@ -584,16 +584,16 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter static Result> MakeArrayBlobArray( const std::shared_ptr& list_type, const std::vector>& rows) { - auto list_builder = std::make_shared( - arrow::default_memory_pool(), std::make_shared(), list_type); - auto element_builder = - checked_cast(list_builder->value_builder()); + arrow::ListBuilder list_builder(arrow::default_memory_pool(), + std::make_shared(), list_type); + auto* element_builder = + checked_cast(list_builder.value_builder()); for (const std::optional& row : rows) { if (!row) { - PAIMON_RETURN_NOT_OK_FROM_ARROW(list_builder->AppendNull()); + PAIMON_RETURN_NOT_OK_FROM_ARROW(list_builder.AppendNull()); continue; } - PAIMON_RETURN_NOT_OK_FROM_ARROW(list_builder->Append()); + PAIMON_RETURN_NOT_OK_FROM_ARROW(list_builder.Append()); for (const std::optional& element : *row) { if (element) { PAIMON_RETURN_NOT_OK_FROM_ARROW(element_builder->Append(*element)); @@ -603,7 +603,7 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter } } std::shared_ptr array; - PAIMON_RETURN_NOT_OK_FROM_ARROW(list_builder->Finish(&array)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(list_builder.Finish(&array)); return array; } @@ -612,10 +612,11 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter static Result> MakeMapBlobArray( const std::shared_ptr& map_type, const std::vector>& rows) { - auto key_builder = std::make_shared(); - auto item_builder = std::make_shared(); - arrow::MapBuilder map_builder(arrow::default_memory_pool(), key_builder, item_builder, - map_type); + arrow::MapBuilder map_builder(arrow::default_memory_pool(), + std::make_shared(), + std::make_shared(), map_type); + auto* key_builder = checked_cast(map_builder.key_builder()); + auto* item_builder = checked_cast(map_builder.item_builder()); for (const std::optional& row : rows) { if (!row) { PAIMON_RETURN_NOT_OK_FROM_ARROW(map_builder.AppendNull()); diff --git a/test/inte/primary_key_sorted_index_inte_test.cpp b/test/inte/primary_key_sorted_index_inte_test.cpp index f2811a699..bdf7825a4 100644 --- a/test/inte/primary_key_sorted_index_inte_test.cpp +++ b/test/inte/primary_key_sorted_index_inte_test.cpp @@ -556,11 +556,11 @@ TEST_P(PrimaryKeySortedIndexInteTest, ScoredSplitFailsBeforeForceKeepDeleteFallb } std::vector PrimaryKeySortedIndexFormats() { - std::vector formats = {"parquet"}; #ifdef PAIMON_ENABLE_ORC - formats.emplace_back("orc"); + return {"parquet", "orc"}; +#else + return {"parquet"}; #endif - return formats; } INSTANTIATE_TEST_SUITE_P(FileFormat, PrimaryKeySortedIndexInteTest,