From 5528f2f87c6c8dd38c4770d3389a9e58b325eec6 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Mon, 9 Nov 2026 14:48:06 +0800 Subject: [PATCH 01/10] fix: restore GCC 13.3 ARM64 Release builds and macOS regression coverage Fix optional assignment diagnostics and portability failures found by the macOS ARM64 full-feature regression. Incorporate the macOS CI coverage from #536. Closes #537 --- .github/actions/setup-ccache/action.yml | 7 +- .github/workflows/build_and_test.yaml | 50 +++++++ CMakeLists.txt | 39 +++++- build_support/asan_symbolize.py | 3 +- ci/scripts/build_paimon.sh | 4 +- ci/scripts/setup_flatc.sh | 15 +- ci/scripts/test_asan_symbolize.sh | 30 ++++ cmake_modules/BuildAwsAuth.cmake | 10 ++ cmake_modules/ThirdpartyToolchain.cmake | 81 +++++++++-- .../predicate/literal_converter_test.cpp | 17 +-- .../common/predicate/predicate_test.cpp | 129 +++++++++--------- .../predicate/predicate_validator_test.cpp | 20 +-- ...e_materializing_file_batch_reader_test.cpp | 66 ++++----- .../reader/predicate_batch_reader_test.cpp | 18 +-- .../prefetch_file_batch_reader_impl_test.cpp | 15 +- .../common/utils/date_time_utils_test.cpp | 26 ++-- .../common/utils/options_utils_test.cpp | 4 +- src/paimon/common/utils/string_utils.cpp | 16 ++- src/paimon/common/utils/string_utils_test.cpp | 10 ++ .../core/compact/compact_deletion_file.h | 4 +- .../operation/precise_file_index_read.cpp | 6 +- .../postpone/postpone_bucket_writer_test.cpp | 9 +- .../realtime/arrow_realtime_store_test.cpp | 8 +- src/paimon/rest/mock_rest_server.cpp | 22 ++- src/paimon/rest/rest_http_client_test.cpp | 11 ++ src/paimon/testing/utils/testharness.cpp | 5 +- test/inte/scan_inte_test.cpp | 4 +- .../write_and_read_inte_with_index_test.cpp | 42 +++--- test/inte/write_inte_test.cpp | 2 +- 29 files changed, 460 insertions(+), 213 deletions(-) diff --git a/.github/actions/setup-ccache/action.yml b/.github/actions/setup-ccache/action.yml index 62de2837a..fa4da936e 100644 --- a/.github/actions/setup-ccache/action.yml +++ b/.github/actions/setup-ccache/action.yml @@ -27,7 +27,12 @@ runs: steps: - name: Install ccache shell: bash - run: sudo apt-get update && sudo apt-get install -y ccache + run: | + if [[ "${RUNNER_OS}" == "macOS" ]]; then + brew install ccache + else + sudo apt-get update && sudo apt-get install -y ccache + fi - name: Restore ccache uses: actions/cache@v4 diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index f9548716d..1f0df9b90 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -50,6 +50,56 @@ jobs: shell: bash run: ci/scripts/test_asan_symbolize.sh + macos-build-and-test: + name: appleclang-release-arm64 + runs-on: macos-15 + timeout-minutes: 180 + env: + CC: /usr/bin/clang + CXX: /usr/bin/clang++ + PAIMON_BUILD_JOBS: '3' + CARGO_BUILD_JOBS: '3' + steps: + - name: Checkout paimon-cpp + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + lfs: true + - name: Install build dependencies + shell: bash + run: | + brew install cmake ninja protobuf openssl@3 curl + echo "$(brew --prefix protobuf)/bin" >> "$GITHUB_PATH" + echo "CMAKE_PREFIX_PATH=$(brew --prefix openssl@3):$(brew --prefix curl)" >> "$GITHUB_ENV" + - name: Setup ccache + uses: ./.github/actions/setup-ccache + with: + cache-key-prefix: ccache-appleclang-release-arm64 + - name: Install Rust and protobuf tools + shell: bash + run: ci/scripts/setup_rust.sh + - name: Install flatc and libclang (Vortex) + shell: bash + run: ci/scripts/setup_flatc.sh + - name: Build, test and verify installation + shell: bash + run: | + ci/scripts/build_paimon.sh \ + --source_dir "$(pwd)" \ + --build_type Release \ + --install_smoke + - name: Upload test logs + if: failure() + uses: actions/upload-artifact@v4 + with: + name: appleclang-release-arm64-test-logs + path: | + build/build/test-logs/ + build/Testing/Temporary/ + build/CMakeFiles/CMakeConfigureLog.yaml + - name: Show ccache statistics + if: always() + run: ccache -s + build-and-test: name: ${{ matrix.name }} runs-on: ${{ matrix.runner || 'ubuntu-24.04' }} diff --git a/CMakeLists.txt b/CMakeLists.txt index df1c51ea0..35a66f076 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -345,11 +345,46 @@ endif() if(PAIMON_ENABLE_LANCE) find_program(PAIMON_LANCE_PROTOC protoc REQUIRED) - corrosion_import_crate(MANIFEST_PATH crates/lance_ffi/Cargo.toml CRATES - paimon_lance_ffi) + set(PAIMON_LANCE_CORROSION_OPTIONS) + if(APPLE) + # Rust's host tools and Apple's native crypto dependencies require Apple's toolchain. + list(APPEND PAIMON_LANCE_CORROSION_OPTIONS NO_LINKER_OVERRIDE) + endif() + corrosion_import_crate(MANIFEST_PATH + crates/lance_ffi/Cargo.toml + CRATES + paimon_lance_ffi + ${PAIMON_LANCE_CORROSION_OPTIONS}) corrosion_experimental_cbindgen(TARGET paimon_lance_ffi HEADER_NAME paimon_lance_ffi.h) corrosion_set_env_vars(paimon_lance_ffi "PROTOC=${PAIMON_LANCE_PROTOC}") + if(APPLE) + target_link_libraries(paimon_lance_ffi INTERFACE "-framework Security" + "-framework CoreFoundation") + find_program(PAIMON_LANCE_CLANG clang + PATHS /usr/bin + NO_DEFAULT_PATH REQUIRED) + find_program(PAIMON_LANCE_CLANGXX clang++ + PATHS /usr/bin + NO_DEFAULT_PATH REQUIRED) + # cc-rs gives the hyphenated target variables priority over Corrosion's defaults. + # Preserve Apple's crypto-capable CPU baseline for these native dependencies. + corrosion_set_env_vars(paimon_lance_ffi + "CC_${Rust_CARGO_TARGET_CACHED}=${PAIMON_LANCE_CLANG}" + "CXX_${Rust_CARGO_TARGET_CACHED}=${PAIMON_LANCE_CLANGXX}" + "CFLAGS=" + "CXXFLAGS=" + "CFLAGS_${Rust_CARGO_TARGET_CACHED}=" + "CXXFLAGS_${Rust_CARGO_TARGET_CACHED}=") + if(CMAKE_OSX_SYSROOT) + corrosion_set_env_vars(paimon_lance_ffi "SDKROOT=${CMAKE_OSX_SYSROOT}") + endif() + if(CMAKE_OSX_DEPLOYMENT_TARGET) + corrosion_set_env_vars(paimon_lance_ffi + "MACOSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET}" + ) + endif() + endif() endif() if(PAIMON_ENABLE_LUCENE) diff --git a/build_support/asan_symbolize.py b/build_support/asan_symbolize.py index a119d9882..3b3323ad0 100755 --- a/build_support/asan_symbolize.py +++ b/build_support/asan_symbolize.py @@ -158,7 +158,8 @@ def open_atos(self): self.pipe = subprocess.Popen(cmdline, stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stderr=subprocess.PIPE, + encoding='utf-8', errors='surrogateescape') def symbolize(self, addr, binary, offset): """Overrides Symbolizer.symbolize.""" diff --git a/ci/scripts/build_paimon.sh b/ci/scripts/build_paimon.sh index eacaf7485..ba5237f94 100755 --- a/ci/scripts/build_paimon.sh +++ b/ci/scripts/build_paimon.sh @@ -145,9 +145,9 @@ if [[ "${enable_tsan}" == "true" ]]; then fi # CI always builds natively, so the host architecture is the target architecture. host_arch=$(uname -m) -if [[ "${host_arch}" != "x86_64" ]]; then +if [[ "$(uname -s)" != "Linux" || "${host_arch}" != "x86_64" ]]; then ENABLE_LUMINA="OFF" - echo "=== Lumina disabled: no prebuilt artifacts for ${host_arch} ===" + echo "=== Lumina disabled: no prebuilt artifacts for $(uname -s)-${host_arch} ===" fi if [[ "${verify_full_text_archives}" == "true" && "${ENABLE_FULL_TEXT}" != "ON" ]]; then echo "--verify_full_text_archives requires the full-text index, which this build disables" >&2 diff --git a/ci/scripts/setup_flatc.sh b/ci/scripts/setup_flatc.sh index ce8981f2f..099f3abc3 100755 --- a/ci/scripts/setup_flatc.sh +++ b/ci/scripts/setup_flatc.sh @@ -38,7 +38,13 @@ FLATC_INSTALL_DIR=${FLATC_INSTALL_DIR:-"${HOME}/.local/bin"} # Vortex's native dependencies invoke bindgen in their build scripts, which loads libclang at # build time. Images that only carry GCC (e.g. the gcc8 test container) may lack it. -if ! ldconfig -p 2>/dev/null | grep -q 'libclang'; then +if [[ "$(uname -s)" == "Darwin" ]]; then + if ! brew --prefix llvm >/dev/null 2>&1; then + brew install llvm + fi + export LIBCLANG_PATH="$(brew --prefix llvm)/lib" + echo "LIBCLANG_PATH=${LIBCLANG_PATH}" >>"${GITHUB_ENV:-/dev/null}" || true +elif ! ldconfig -p 2>/dev/null | grep -q 'libclang'; then if [[ "$(id -u)" -ne 0 ]] && command -v sudo >/dev/null 2>&1; then apt_prefix="sudo" else @@ -63,7 +69,12 @@ git clone --depth 1 --branch "v${FLATBUFFERS_VERSION}" \ cmake -S "${workdir}/flatbuffers" -B "${workdir}/build" \ -DCMAKE_BUILD_TYPE=Release \ -DFLATBUFFERS_BUILD_TESTS=OFF -cmake --build "${workdir}/build" --target flatc -j "$(nproc)" +if command -v nproc >/dev/null 2>&1; then + build_jobs=$(nproc) +else + build_jobs=$(sysctl -n hw.ncpu) +fi +cmake --build "${workdir}/build" --target flatc -j "${build_jobs}" mkdir -p "${FLATC_INSTALL_DIR}" install -m 0755 "${workdir}/build/flatc" "${FLATC_INSTALL_DIR}/flatc" diff --git a/ci/scripts/test_asan_symbolize.sh b/ci/scripts/test_asan_symbolize.sh index 0859559b4..2a40050cc 100755 --- a/ci/scripts/test_asan_symbolize.sh +++ b/ci/scripts/test_asan_symbolize.sh @@ -92,4 +92,34 @@ if [[ "${output}" != *"#0"* || "${output}" != *"tail"* || "${output}" != *" in"* status=1 fi +echo "=== Darwin symbolizer exchanges text with atos ===" +"${python}" - "${symbolizer}" <<'PY' || status=1 +import runpy +import subprocess +import sys +from unittest import mock + +symbolizer_path = sys.argv[1] +symbols = runpy.run_path(symbolizer_path) +sys.argv = [symbolizer_path] +real_popen = subprocess.Popen + + +def fake_atos(command, **kwargs): + assert command[0] == 'atos', command + return real_popen( + [sys.executable, '-c', + "assert input() == '0x123'; " + "print('foo(int) (in foo.dylib) (source.cc:42)')"], **kwargs) + + +symbolizer = symbols['DarwinSymbolizer']('0x123456789', '/blah/foo.dylib') +with mock.patch.object(subprocess, 'Popen', side_effect=fake_atos): + result = symbolizer.symbolize('0x123456789', '/blah/foo.dylib', '0x123') +assert result == ['0x123456789 in foo source.cc:42'], result +assert symbolizer.pipe.wait() == 0 +symbolizer.pipe.stdout.close() +symbolizer.pipe.stderr.close() +PY + exit "${status}" diff --git a/cmake_modules/BuildAwsAuth.cmake b/cmake_modules/BuildAwsAuth.cmake index 14b1f3ea9..a3275fc0f 100644 --- a/cmake_modules/BuildAwsAuth.cmake +++ b/cmake_modules/BuildAwsAuth.cmake @@ -79,6 +79,16 @@ function(paimon_add_aws_c_project endif() if(APPLE) list(APPEND AWS_PLATFORM_CMAKE_ARGS -DAWS_USE_SECITEM=ON) + if(CMAKE_C_COMPILER_ID STREQUAL "GNU") + # Apple's crypto and TLS backends require Clang's __builtin_available. + # These dependencies expose a C ABI, so Paimon can still use GCC. + find_program(AWS_C_CLANG_EXECUTABLE clang + PATHS /usr/bin + NO_DEFAULT_PATH REQUIRED) + list(APPEND AWS_PLATFORM_CMAKE_ARGS + -DCMAKE_C_COMPILER=${AWS_C_CLANG_EXECUTABLE} + -DCMAKE_OSX_SYSROOT=${CMAKE_OSX_SYSROOT}) + endif() endif() # aws-c-io's installed package unconditionally looks for s2n on Unix even # when it was built with Apple's native TLS backend. diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake index 8aa8f1183..1b51ceede 100644 --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake @@ -881,12 +881,20 @@ macro(build_lucene) "-DZLIB_ROOT=${LUCENE_ZLIB_ROOT}" "-DCMAKE_INSTALL_PREFIX=${LUCENE_PREFIX}") + set(LUCENE_BUILD_COMMAND ${CMAKE_COMMAND} --build --config $) + if(CMAKE_VERSION VERSION_GREATER_EQUAL "4.0") + # Cotire invokes CMake in script mode without forwarding the parent's policy cache. + set(LUCENE_BUILD_COMMAND ${CMAKE_COMMAND} -E env CMAKE_POLICY_VERSION_MINIMUM=3.5 + ${LUCENE_BUILD_COMMAND}) + endif() + set(LUCENE_LIB "${LUCENE_PREFIX}/lib/liblucene++.a") externalproject_add(lucene_ep ${EP_COMMON_OPTIONS} URL ${LUCENE_SOURCE_URL} URL_HASH "SHA256=${PAIMON_LUCENE_BUILD_SHA256_CHECKSUM}" CMAKE_ARGS ${LUCENE_CMAKE_ARGS} + BUILD_COMMAND ${LUCENE_BUILD_COMMAND} BUILD_BYPRODUCTS ${LUCENE_LIB} DEPENDS zlib boost_date_time @@ -1297,6 +1305,25 @@ macro(build_jindosdk_c) set(JINDOSDK_C_LIB_DIR "${JINDOSDK_C_PREFIX}/lib/native") set(JINDOSDK_C_DYNAMIC_LIB "${JINDOSDK_C_LIB_DIR}/${JINDOSDK_C_DYNAMIC_LIB_FILE}") + set(JINDOSDK_C_INSTALL_EXTRA) + set(JINDOSDK_C_INSTALL_PATTERN "${JINDOSDK_C_DYNAMIC_LIB_FILE}*") + set(JINDOSDK_C_BYPRODUCTS "${JINDOSDK_C_DYNAMIC_LIB}") + if(APPLE) + # The SDK's install name uses the major version, but its packaged symlink is broken. + string(REGEX MATCH "^[0-9]+" JINDOSDK_C_MAJOR_VERSION + "${PAIMON_JINDOSDK_C_BUILD_VERSION}") + set(JINDOSDK_C_SONAME "libjindosdk_c.${JINDOSDK_C_MAJOR_VERSION}.dylib") + set(JINDOSDK_C_INSTALL_EXTRA + COMMAND + ${CMAKE_COMMAND} + -E + create_symlink + "${JINDOSDK_C_DYNAMIC_LIB_FILE}" + "${JINDOSDK_C_LIB_DIR}/${JINDOSDK_C_SONAME}") + set(JINDOSDK_C_INSTALL_PATTERN "libjindosdk_c*.dylib") + list(APPEND JINDOSDK_C_BYPRODUCTS "${JINDOSDK_C_LIB_DIR}/${JINDOSDK_C_SONAME}") + endif() + # Extract and install jindosdk from precompiled package externalproject_add(jindosdk_ep URL ${JINDOSDK_C_SOURCE_URL} @@ -1308,7 +1335,8 @@ macro(build_jindosdk_c) "cp -r /include/* ${JINDOSDK_C_INCLUDE_DIR}" COMMAND bash -c "cp -r /lib/native/${JINDOSDK_C_DYNAMIC_LIB_FILE}* ${JINDOSDK_C_LIB_DIR}" - BUILD_BYPRODUCTS "${JINDOSDK_C_DYNAMIC_LIB}") + ${JINDOSDK_C_INSTALL_EXTRA} + BUILD_BYPRODUCTS ${JINDOSDK_C_BYPRODUCTS}) # The include directory must exist before it is referenced by a target. file(MAKE_DIRECTORY "${JINDOSDK_C_INCLUDE_DIR}") @@ -1325,7 +1353,7 @@ macro(build_jindosdk_c) install(DIRECTORY "${JINDOSDK_C_LIB_DIR}/" DESTINATION ${CMAKE_INSTALL_LIBDIR} FILES_MATCHING - PATTERN "${JINDOSDK_C_DYNAMIC_LIB_FILE}*") + PATTERN "${JINDOSDK_C_INSTALL_PATTERN}") endmacro() @@ -1508,6 +1536,18 @@ macro(build_vortex) file(MAKE_DIRECTORY "${VORTEX_INCLUDE_DIR}") file(MAKE_DIRECTORY "${VORTEX_LIB_DIR}") + set(VORTEX_PLATFORM_ENV) + if(APPLE) + list(APPEND VORTEX_PLATFORM_ENV "CFLAGS=${EP_C_FLAGS}" "CXXFLAGS=${EP_CXX_FLAGS}") + if(CMAKE_OSX_SYSROOT) + list(APPEND VORTEX_PLATFORM_ENV "SDKROOT=${CMAKE_OSX_SYSROOT}") + endif() + if(CMAKE_OSX_DEPLOYMENT_TARGET) + list(APPEND VORTEX_PLATFORM_ENV + "MACOSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET}") + endif() + endif() + # A local checkout (PAIMON_VORTEX_SOURCE_DIR) avoids re-downloading the full cargo workspace; # otherwise download the pinned source archive (with SHA256 when versions.txt provides one). if(DEFINED ENV{PAIMON_VORTEX_SOURCE_DIR}) @@ -1536,7 +1576,7 @@ macro(build_vortex) "/vortex-ffi/src/callback_io.rs" COMMAND ${CMAKE_COMMAND} -E env "FLATC=${VORTEX_FLATC}" "RUSTUP_TOOLCHAIN=stable" "CC=${CMAKE_C_COMPILER}" - "CXX=${CMAKE_CXX_COMPILER}" + "CXX=${CMAKE_CXX_COMPILER}" ${VORTEX_PLATFORM_ENV} "CARGO_TARGET_DIR=${VORTEX_CARGO_TARGET_DIR}" "RUSTFLAGS=-Crelocation-model=pic" ${PAIMON_CARGO_EXECUTABLE} rustc --locked --package @@ -1558,16 +1598,26 @@ macro(build_vortex) PROPERTIES IMPORTED_LOCATION "${VORTEX_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${VORTEX_INCLUDE_DIR}") - # System libraries required to link the Rust static archive on Linux (per Vortex's own - # vortex-ffi/cmake/SystemDependencies.cmake). - target_link_libraries(paimon_vortex_ffi - INTERFACE gcc_s - util - rt - Threads::Threads - m - ${CMAKE_DL_LIBS} - c) + if(APPLE) + target_link_libraries(paimon_vortex_ffi + INTERFACE iconv + "-framework Security" + "-framework CoreFoundation" + "-framework SystemConfiguration" + Threads::Threads + m + c + ${CMAKE_DL_LIBS}) + else() + target_link_libraries(paimon_vortex_ffi + INTERFACE gcc_s + util + rt + Threads::Threads + m + ${CMAKE_DL_LIBS} + c) + endif() add_dependencies(paimon_vortex_ffi vortex_ep) install(FILES "${VORTEX_STATIC_LIB}" DESTINATION ${CMAKE_INSTALL_LIBDIR}) @@ -2163,6 +2213,11 @@ macro(build_tbb) string(REPLACE "-Wdocumentation" "" TBB_CMAKE_CXX_FLAGS ${TBB_CMAKE_CXX_FLAGS}) string(REPLACE "-Wdocumentation" "" TBB_CMAKE_C_FLAGS ${TBB_CMAKE_C_FLAGS}) + if(APPLE AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + # TBB suppresses deprecated ucontext API warnings only for Clang on macOS. + string(APPEND TBB_CMAKE_CXX_FLAGS " -Wno-error=deprecated-declarations") + endif() + set(TBB_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/tbb_ep-install") if(LOWERCASE_BUILD_TYPE STREQUAL "debug") diff --git a/src/paimon/common/predicate/literal_converter_test.cpp b/src/paimon/common/predicate/literal_converter_test.cpp index 201f2aed6..c9b9f7f39 100644 --- a/src/paimon/common/predicate/literal_converter_test.cpp +++ b/src/paimon/common/predicate/literal_converter_test.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include "arrow/api.h" @@ -150,10 +151,10 @@ TEST_F(LiteralConverterTest, TestIntLiteral) { TEST_F(LiteralConverterTest, TestBigIntLiteral) { auto field_array = arrow::ipc::internal::json::ArrayFromJSON(arrow::int64(), R"([4, 5, null])").ValueOrDie(); - CheckResult(field_array, - std::vector({Literal(4l), Literal(5l), Literal(FieldType::BIGINT)})); + CheckResult(field_array, std::vector({Literal(int64_t{4}), Literal(int64_t{5}), + Literal(FieldType::BIGINT)})); CheckLiteralsFromString(FieldType::BIGINT, {"4", "5"}, - std::vector({Literal(4l), Literal(5l)})); + std::vector({Literal(int64_t{4}), Literal(int64_t{5})})); CheckLiteralFromRow(arrow::int64(), {static_cast(4), static_cast(5), NullType()}, FieldType::BIGINT, @@ -189,11 +190,11 @@ TEST_F(LiteralConverterTest, TestDoubleLiteral) { CheckLiteralsFromString(FieldType::DOUBLE, {"4.05", "5.17"}, std::vector({Literal(4.05), Literal(5.17)})); CheckLiteralFromRow(arrow::float64(), - {4.05, 5.17, static_cast INFINITY, static_cast(-INFINITY), - static_cast NAN, NullType()}, + {4.05, 5.17, static_cast(INFINITY), static_cast(-INFINITY), + static_cast(NAN), NullType()}, FieldType::DOUBLE, - {Literal(4.05), Literal(5.17), Literal(static_cast INFINITY), - Literal(static_cast(-INFINITY)), Literal(static_cast NAN), + {Literal(4.05), Literal(5.17), Literal(static_cast(INFINITY)), + Literal(static_cast(-INFINITY)), Literal(static_cast(NAN)), Literal(FieldType::DOUBLE)}); } @@ -525,7 +526,7 @@ TEST_F(LiteralConverterTest, TestLiteralsToArray) { arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), R"([456, null, -567])") .ValueOrDie()); CheckLiteralsToArray( - FieldType::BIGINT, {Literal(4l), Literal(FieldType::BIGINT), Literal(-5l)}, + FieldType::BIGINT, {Literal(int64_t{4}), Literal(FieldType::BIGINT), Literal(int64_t{-5})}, arrow::ipc::internal::json::ArrayFromJSON(arrow::int64(), R"([4, null, -5])").ValueOrDie()); CheckLiteralsToArray( FieldType::FLOAT, {Literal(4.5f), Literal(FieldType::FLOAT), Literal(-5.25f)}, diff --git a/src/paimon/common/predicate/predicate_test.cpp b/src/paimon/common/predicate/predicate_test.cpp index 97fef15d8..6c5cf2c37 100644 --- a/src/paimon/common/predicate/predicate_test.cpp +++ b/src/paimon/common/predicate/predicate_test.cpp @@ -195,7 +195,7 @@ TEST_F(PredicateTest, TestLeafFieldBoundsOnWideSlicedBatch) { TEST_F(PredicateTest, TestInvalidFieldIndex) { auto bigint_type = arrow::int64(); auto predicate_base = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f0", - FieldType::BIGINT, Literal(5l)); + FieldType::BIGINT, Literal(int64_t{5})); auto predicate = std::dynamic_pointer_cast(predicate_base); ASSERT_TRUE(predicate); @@ -220,7 +220,7 @@ TEST_F(PredicateTest, TestInvalidFieldIndex) { TEST_F(PredicateTest, TestEqual) { auto bigint_type = arrow::int64(); auto predicate_base = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(5l)); + FieldType::BIGINT, Literal(int64_t{5})); auto predicate = std::dynamic_pointer_cast(predicate_base); ASSERT_TRUE(predicate); auto f0 = @@ -237,14 +237,15 @@ TEST_F(PredicateTest, TestEqual) { ASSERT_EQ(*predicate->Negate(), *PredicateBuilder::NotEqual(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, - Literal(5l))); + Literal(int64_t{5}))); ASSERT_FALSE(*predicate->Negate() == *PredicateBuilder::NotEqual(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(10l))); + FieldType::BIGINT, Literal(int64_t{10}))); ASSERT_FALSE(*predicate->Negate() == *PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(10l))); + FieldType::BIGINT, + Literal(int64_t{10}))); // with internal row auto arrow_schema = arrow::schema(arrow::FieldVector({arrow::field("f0", bigint_type)})); ASSERT_FALSE(predicate->Test(arrow_schema, CreateBigIntRow({4})).value()); @@ -289,7 +290,7 @@ TEST_F(PredicateTest, TestEqualNull) { TEST_F(PredicateTest, TestNotEqual) { auto bigint_type = arrow::int64(); auto predicate_base = PredicateBuilder::NotEqual(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(5l)); + FieldType::BIGINT, Literal(int64_t{5})); auto predicate = std::dynamic_pointer_cast(predicate_base); ASSERT_TRUE(predicate); auto f0 = @@ -307,7 +308,7 @@ TEST_F(PredicateTest, TestNotEqual) { auto predicate_negate = std::dynamic_pointer_cast(predicate->Negate()); ASSERT_EQ(*predicate_negate, *PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(5l))); + FieldType::BIGINT, Literal(int64_t{5}))); // with internal row auto arrow_schema = arrow::schema(arrow::FieldVector({arrow::field("f0", bigint_type)})); @@ -354,7 +355,7 @@ TEST_F(PredicateTest, TestNotEqualNull) { TEST_F(PredicateTest, TestGreater) { auto bigint_type = arrow::int64(); auto predicate_base = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(5l)); + FieldType::BIGINT, Literal(int64_t{5})); auto predicate = std::dynamic_pointer_cast(predicate_base); ASSERT_TRUE(predicate); auto f0 = @@ -373,7 +374,7 @@ TEST_F(PredicateTest, TestGreater) { ASSERT_EQ(*predicate->Negate(), *PredicateBuilder::LessOrEqual(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(5l))); + FieldType::BIGINT, Literal(int64_t{5}))); // with internal row auto arrow_schema = arrow::schema(arrow::FieldVector({arrow::field("f0", bigint_type)})); ASSERT_FALSE(predicate->Test(arrow_schema, CreateBigIntRow({4})).value()); @@ -419,7 +420,7 @@ TEST_F(PredicateTest, TestGreaterNull) { TEST_F(PredicateTest, TestGreaterOrEqual) { auto bigint_type = arrow::int64(); auto predicate_base = PredicateBuilder::GreaterOrEqual( - /*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, Literal(5l)); + /*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, Literal(int64_t{5})); auto predicate = std::dynamic_pointer_cast(predicate_base); ASSERT_TRUE(predicate); ASSERT_EQ(predicate->GetFunction().ToString(), "GreaterOrEqual"); @@ -439,7 +440,7 @@ TEST_F(PredicateTest, TestGreaterOrEqual) { ASSERT_EQ(*predicate->Negate(), *PredicateBuilder::LessThan(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, - Literal(5l))); + Literal(int64_t{5}))); // with internal row auto arrow_schema = arrow::schema(arrow::FieldVector({arrow::field("f0", bigint_type)})); ASSERT_FALSE(predicate->Test(arrow_schema, CreateBigIntRow({4})).value()); @@ -485,7 +486,7 @@ TEST_F(PredicateTest, TestGreaterOrEqualNull) { TEST_F(PredicateTest, TestLess) { auto bigint_type = arrow::int64(); auto predicate_base = PredicateBuilder::LessThan(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(5l)); + FieldType::BIGINT, Literal(int64_t{5})); auto predicate = std::dynamic_pointer_cast(predicate_base); ASSERT_TRUE(predicate); auto f0 = @@ -504,7 +505,7 @@ TEST_F(PredicateTest, TestLess) { ASSERT_EQ(*predicate->Negate(), *PredicateBuilder::GreaterOrEqual( - /*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, Literal(5l))); + /*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, Literal(int64_t{5}))); // with internal row auto arrow_schema = arrow::schema(arrow::FieldVector({arrow::field("f0", bigint_type)})); @@ -549,7 +550,7 @@ TEST_F(PredicateTest, TestLessNull) { TEST_F(PredicateTest, TestLessOrEqual) { auto bigint_type = arrow::int64(); auto predicate_base = PredicateBuilder::LessOrEqual(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(5l)); + FieldType::BIGINT, Literal(int64_t{5})); auto predicate = std::dynamic_pointer_cast(predicate_base); ASSERT_TRUE(predicate); auto f0 = @@ -568,7 +569,7 @@ TEST_F(PredicateTest, TestLessOrEqual) { ASSERT_EQ(*predicate->Negate(), *PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(5l))); + FieldType::BIGINT, Literal(int64_t{5}))); // with internal row auto arrow_schema = arrow::schema(arrow::FieldVector({arrow::field("f0", bigint_type)})); @@ -673,8 +674,9 @@ TEST_F(PredicateTest, TestIsNotNull) { TEST_F(PredicateTest, TestIn) { auto bigint_type = arrow::int64(); - auto predicate_base = PredicateBuilder::In(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, {Literal(1l), Literal(3l)}); + auto predicate_base = + PredicateBuilder::In(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, + {Literal(int64_t{1}), Literal(int64_t{3})}); auto predicate = std::dynamic_pointer_cast(predicate_base); ASSERT_TRUE(predicate); auto f0 = @@ -693,7 +695,7 @@ TEST_F(PredicateTest, TestIn) { ASSERT_EQ(*predicate->Negate(), *PredicateBuilder::NotIn(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, - {Literal(1l), Literal(3l)})); + {Literal(int64_t{1}), Literal(int64_t{3})})); // with internal row auto arrow_schema = arrow::schema(arrow::FieldVector({arrow::field("f0", bigint_type)})); @@ -711,7 +713,7 @@ TEST_F(PredicateTest, TestInNull) { auto bigint_type = arrow::int64(); auto predicate_base = PredicateBuilder::In( /*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, - {Literal(1l), Literal(FieldType::BIGINT), Literal(3l)}); + {Literal(int64_t{1}), Literal(FieldType::BIGINT), Literal(int64_t{3})}); auto predicate = std::dynamic_pointer_cast(predicate_base); ASSERT_TRUE(predicate); auto f0 = @@ -742,8 +744,9 @@ TEST_F(PredicateTest, TestInNull) { TEST_F(PredicateTest, TestNotIn) { auto bigint_type = arrow::int64(); - auto predicate_base = PredicateBuilder::NotIn(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, {Literal(1l), Literal(3l)}); + auto predicate_base = + PredicateBuilder::NotIn(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, + {Literal(int64_t{1}), Literal(int64_t{3})}); auto predicate = std::dynamic_pointer_cast(predicate_base); ASSERT_TRUE(predicate); auto f0 = @@ -762,7 +765,7 @@ TEST_F(PredicateTest, TestNotIn) { ASSERT_EQ(*predicate->Negate(), *PredicateBuilder::In(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, - {Literal(1l), Literal(3l)})); + {Literal(int64_t{1}), Literal(int64_t{3})})); // with internal row auto arrow_schema = arrow::schema(arrow::FieldVector({arrow::field("f0", bigint_type)})); @@ -783,7 +786,7 @@ TEST_F(PredicateTest, TestNotInNull) { auto bigint_type = arrow::int64(); auto predicate_base = PredicateBuilder::NotIn( /*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, - {Literal(1l), Literal(FieldType::BIGINT), Literal(3l)}); + {Literal(int64_t{1}), Literal(FieldType::BIGINT), Literal(int64_t{3})}); auto predicate = std::dynamic_pointer_cast(predicate_base); ASSERT_TRUE(predicate); auto f0 = @@ -819,8 +822,8 @@ TEST_F(PredicateTest, TestLargeIn) { auto bigint_type = arrow::int64(); std::vector literals; literals.reserve(30); - literals.emplace_back(1l); - literals.emplace_back(3l); + literals.emplace_back(int64_t{1}); + literals.emplace_back(int64_t{3}); for (int64_t i = 10; i < 30; i++) { literals.emplace_back(i); } @@ -859,9 +862,9 @@ TEST_F(PredicateTest, TestLargeInNull) { auto bigint_type = arrow::int64(); std::vector literals; literals.reserve(30); - literals.emplace_back(1l); + literals.emplace_back(int64_t{1}); literals.emplace_back(FieldType::BIGINT); - literals.emplace_back(3l); + literals.emplace_back(int64_t{3}); for (int64_t i = 10; i < 30; i++) { literals.emplace_back(i); } @@ -900,8 +903,8 @@ TEST_F(PredicateTest, TestLargeNotIn) { auto bigint_type = arrow::int64(); std::vector literals; literals.reserve(30); - literals.emplace_back(1l); - literals.emplace_back(3l); + literals.emplace_back(int64_t{1}); + literals.emplace_back(int64_t{3}); for (int64_t i = 10; i < 30; i++) { literals.emplace_back(i); } @@ -943,9 +946,9 @@ TEST_F(PredicateTest, TestLargeNotInNull) { auto bigint_type = arrow::int64(); std::vector literals; literals.reserve(30); - literals.emplace_back(1l); + literals.emplace_back(int64_t{1}); literals.emplace_back(FieldType::BIGINT); - literals.emplace_back(3l); + literals.emplace_back(int64_t{3}); for (int64_t i = 10; i < 30; i++) { literals.emplace_back(i); } @@ -1017,8 +1020,9 @@ TEST_F(PredicateTest, TestLargeStringIn) { TEST_F(PredicateTest, TestInAfterRebind) { auto bigint_type = arrow::int64(); - auto predicate_base = PredicateBuilder::In(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, {Literal(1l), Literal(3l)}); + auto predicate_base = + PredicateBuilder::In(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, + {Literal(int64_t{1}), Literal(int64_t{3})}); auto leaf_predicate = std::dynamic_pointer_cast(predicate_base); ASSERT_TRUE(leaf_predicate); @@ -1080,9 +1084,9 @@ TEST_F(PredicateTest, TestAnd) { ASSERT_OK_AND_ASSIGN( auto predicate_base, PredicateBuilder::And({PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(3l)), + FieldType::BIGINT, Literal(int64_t{3})), PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", - FieldType::BIGINT, Literal(5l))})); + FieldType::BIGINT, Literal(int64_t{5}))})); auto predicate = std::dynamic_pointer_cast(predicate_base); auto f0 = arrow::ipc::internal::json::ArrayFromJSON(bigint_type, R"([4, 3, 3, null])").ValueOrDie(); @@ -1101,9 +1105,9 @@ TEST_F(PredicateTest, TestAnd) { ASSERT_OK_AND_ASSIGN( auto negate_predicate, PredicateBuilder::Or({PredicateBuilder::NotEqual(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(3l)), + FieldType::BIGINT, Literal(int64_t{3})), PredicateBuilder::NotEqual(/*field_index=*/1, /*field_name=*/"f1", - FieldType::BIGINT, Literal(5l))})); + FieldType::BIGINT, Literal(int64_t{5}))})); ASSERT_EQ(*predicate->Negate(), *negate_predicate); // with internal row @@ -1127,9 +1131,9 @@ TEST_F(PredicateTest, TestOr) { ASSERT_OK_AND_ASSIGN( auto predicate_base, PredicateBuilder::Or({PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(3l)), + FieldType::BIGINT, Literal(int64_t{3})), PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", - FieldType::BIGINT, Literal(5l))})); + FieldType::BIGINT, Literal(int64_t{5}))})); auto predicate = std::dynamic_pointer_cast(predicate_base); auto f0 = arrow::ipc::internal::json::ArrayFromJSON(bigint_type, R"([4, 3, 3, null])").ValueOrDie(); @@ -1145,12 +1149,12 @@ TEST_F(PredicateTest, TestOr) { predicate->Test(*struct_array, arrow::default_memory_pool())); ASSERT_EQ(is_valid, std::vector({0, 1, 1, 1})); - ASSERT_OK_AND_ASSIGN( - auto negate_predicate, - PredicateBuilder::And({PredicateBuilder::NotEqual(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(3l)), - PredicateBuilder::NotEqual(/*field_index=*/1, /*field_name=*/"f1", - FieldType::BIGINT, Literal(5l))})); + ASSERT_OK_AND_ASSIGN(auto negate_predicate, + PredicateBuilder::And( + {PredicateBuilder::NotEqual(/*field_index=*/0, /*field_name=*/"f0", + FieldType::BIGINT, Literal(int64_t{3})), + PredicateBuilder::NotEqual(/*field_index=*/1, /*field_name=*/"f1", + FieldType::BIGINT, Literal(int64_t{5}))})); ASSERT_EQ(*predicate->Negate(), *negate_predicate); // with internal row @@ -1171,8 +1175,9 @@ TEST_F(PredicateTest, TestOr) { TEST_F(PredicateTest, TestBetween) { auto bigint_type = arrow::int64(); - auto predicate_base = PredicateBuilder::Between(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(3l), Literal(5l)); + auto predicate_base = + PredicateBuilder::Between(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, + Literal(int64_t{3}), Literal(int64_t{5})); auto predicate = std::dynamic_pointer_cast(predicate_base); ASSERT_TRUE(predicate); auto f0 = arrow::ipc::internal::json::ArrayFromJSON(bigint_type, R"([3, 4, 5, 100, 1, null])") @@ -1189,9 +1194,9 @@ TEST_F(PredicateTest, TestBetween) { ASSERT_EQ(is_valid, std::vector({1, 1, 1, 0, 0, 0})); auto less_than = PredicateBuilder::LessThan(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(3l)); + FieldType::BIGINT, Literal(int64_t{3})); auto greater_than = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(5l)); + FieldType::BIGINT, Literal(int64_t{5})); ASSERT_OK_AND_ASSIGN(auto or_predicate, PredicateBuilder::Or({less_than, greater_than})); auto predicate_negate = std::dynamic_pointer_cast(predicate->Negate()); @@ -1217,7 +1222,7 @@ TEST_F(PredicateTest, TestBetweenNull) { auto bigint_type = arrow::int64(); auto predicate_base = PredicateBuilder::Between(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, - Literal(FieldType::BIGINT), Literal(5l)); + Literal(FieldType::BIGINT), Literal(int64_t{5})); auto predicate = std::dynamic_pointer_cast(predicate_base); ASSERT_TRUE(predicate); auto f0 = arrow::ipc::internal::json::ArrayFromJSON(bigint_type, R"([4, null])").ValueOrDie(); @@ -1817,12 +1822,12 @@ TEST_F(PredicateTest, TestCompound) { TEST_F(PredicateTest, TestPredicateToString) { { auto predicate = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(5l)); + FieldType::BIGINT, Literal(int64_t{5})); ASSERT_EQ(predicate->ToString(), "Equal(f0, 5)"); } { auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(5l)); + FieldType::BIGINT, Literal(int64_t{5})); ASSERT_EQ(predicate->ToString(), "GreaterThan(f0, 5)"); } { @@ -1859,20 +1864,20 @@ TEST_F(PredicateTest, TestPredicateToString) { { ASSERT_OK_AND_ASSIGN( - auto predicate, - PredicateBuilder::And({PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(3l)), - PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", - FieldType::BIGINT, Literal(5l))})); + auto predicate, PredicateBuilder::And( + {PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", + FieldType::BIGINT, Literal(int64_t{3})), + PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", + FieldType::BIGINT, Literal(int64_t{5}))})); ASSERT_EQ(predicate->ToString(), "And([Equal(f0, 3), Equal(f1, 5)])"); } { ASSERT_OK_AND_ASSIGN( - auto predicate, - PredicateBuilder::Or({PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(3l)), - PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", - FieldType::BIGINT, Literal(5l))})); + auto predicate, PredicateBuilder::Or( + {PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", + FieldType::BIGINT, Literal(int64_t{3})), + PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", + FieldType::BIGINT, Literal(int64_t{5}))})); ASSERT_EQ(predicate->ToString(), "Or([Equal(f0, 3), Equal(f1, 5)])"); } } diff --git a/src/paimon/common/predicate/predicate_validator_test.cpp b/src/paimon/common/predicate/predicate_validator_test.cpp index 89432b727..b3ecae673 100644 --- a/src/paimon/common/predicate/predicate_validator_test.cpp +++ b/src/paimon/common/predicate/predicate_validator_test.cpp @@ -93,7 +93,7 @@ TEST(PredicateValidatorTest, TestValidateLiterals) { auto predicate, PredicateBuilder::And({ PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, - Literal(3l)), + Literal(int64_t{3})), PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", FieldType::FLOAT, Literal(static_cast(5.5))), PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", FieldType::STRING, @@ -123,9 +123,9 @@ TEST(PredicateValidatorTest, TestValidateLiterals) { auto predicate, PredicateBuilder::And({ PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, - Literal(3l)), + Literal(int64_t{3})), PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", FieldType::FLOAT, - Literal(5l)), + Literal(int64_t{5})), PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", FieldType::STRING, Literal(FieldType::STRING, str.data(), str.size())), PredicateBuilder::Equal(/*field_index=*/3, /*field_name=*/"f3", FieldType::BOOLEAN, @@ -142,7 +142,7 @@ TEST(PredicateValidatorTest, TestValidateLiterals) { auto predicate, PredicateBuilder::And({ PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, - Literal(3l)), + Literal(int64_t{3})), PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", FieldType::FLOAT, Literal(static_cast(5.5))), PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", FieldType::STRING, @@ -161,7 +161,7 @@ TEST(PredicateValidatorTest, TestValidateLiterals) { auto predicate, PredicateBuilder::And({ PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, - Literal(3l)), + Literal(int64_t{3})), PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", FieldType::FLOAT, Literal(static_cast(5.5))), PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", FieldType::STRING, @@ -195,7 +195,7 @@ TEST(PredicateValidatorTest, TestValidateSchema) { auto predicate, PredicateBuilder::And({ PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, - Literal(3l)), + Literal(int64_t{3})), PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", FieldType::FLOAT, Literal(static_cast(5.5))), PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", FieldType::STRING, @@ -257,7 +257,7 @@ TEST(PredicateValidatorTest, TestValidateSchema) { auto predicate, PredicateBuilder::And({ PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, - Literal(3l)), + Literal(int64_t{3})), PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f1", FieldType::FLOAT, Literal(static_cast(5.5))), PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", FieldType::STRING, @@ -285,7 +285,7 @@ TEST(PredicateValidatorTest, TestValidateSchema) { auto predicate, PredicateBuilder::And({ PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, - Literal(3l)), + Literal(int64_t{3})), PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f1", FieldType::FLOAT, Literal(static_cast(5.5))), PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", FieldType::STRING, @@ -336,7 +336,7 @@ TEST(PredicateValidatorTest, TestValidateSchema) { auto predicate, PredicateBuilder::And({ PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, - Literal(3l)), + Literal(int64_t{3})), PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", FieldType::FLOAT, Literal(static_cast(5.5))), PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", FieldType::STRING, @@ -362,7 +362,7 @@ TEST(PredicateValidatorTest, TestValidateSchema) { auto predicate, PredicateBuilder::And({ PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, - Literal(3l)), + Literal(int64_t{3})), PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", FieldType::FLOAT, Literal(static_cast(5.5))), PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", FieldType::STRING, diff --git a/src/paimon/common/reader/late_materializing_file_batch_reader_test.cpp b/src/paimon/common/reader/late_materializing_file_batch_reader_test.cpp index 7b62862fa..0e677fe17 100644 --- a/src/paimon/common/reader/late_materializing_file_batch_reader_test.cpp +++ b/src/paimon/common/reader/late_materializing_file_batch_reader_test.cpp @@ -352,7 +352,7 @@ TEST_F(LateMaterializingFileBatchReaderTest, PassThroughWhenPayloadEmpty) { std::move(mock), GetArrowPool(pool_))); // read schema is just {k}; the predicate on k covers all columns -> payload empty auto predicate = PredicateBuilder::GreaterOrEqual(/*field_index=*/0, /*field_name=*/"k", - FieldType::BIGINT, Literal(0l)); + FieldType::BIGINT, Literal(int64_t{0})); ASSERT_OK(SetReadSchema(reader.get(), arrow::schema({k_field_}), predicate, std::nullopt)); ASSERT_OK_AND_ASSIGN(std::vector rows, Collect(reader.get())); @@ -372,7 +372,7 @@ TEST_F(LateMaterializingFileBatchReaderTest, ContiguousSubsetAcrossBatches) { ASSERT_OK_AND_ASSIGN(auto reader, LateMaterializingFileBatchReader::Create( std::move(mock), GetArrowPool(pool_))); auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"k", - FieldType::BIGINT, Literal(4l)); + FieldType::BIGINT, Literal(int64_t{4})); ASSERT_OK(SetReadSchema(reader.get(), arrow::schema(full_fields_), predicate, std::nullopt)); ASSERT_OK_AND_ASSIGN(std::vector rows, Collect(reader.get())); @@ -397,7 +397,7 @@ TEST_F(LateMaterializingFileBatchReaderTest, ScatteredAlternatingMatch) { ASSERT_OK_AND_ASSIGN(auto reader, LateMaterializingFileBatchReader::Create( std::move(mock), GetArrowPool(pool_))); auto predicate = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"k", - FieldType::BIGINT, Literal(1l)); + FieldType::BIGINT, Literal(int64_t{1})); ASSERT_OK(SetReadSchema(reader.get(), arrow::schema(full_fields_), predicate, std::nullopt)); ASSERT_OK_AND_ASSIGN(std::vector rows, Collect(reader.get())); @@ -421,7 +421,7 @@ TEST_F(LateMaterializingFileBatchReaderTest, MatchedIntersectsSelection) { ASSERT_OK_AND_ASSIGN(auto reader, LateMaterializingFileBatchReader::Create( std::move(mock), GetArrowPool(pool_))); auto predicate = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"k", - FieldType::BIGINT, Literal(1l)); + FieldType::BIGINT, Literal(int64_t{1})); // predicate hits {1,3,5,7,9,11}; selection keeps only {1,5,9} RoaringBitmap32 selection; selection.Add(1); @@ -447,7 +447,7 @@ TEST_F(LateMaterializingFileBatchReaderTest, EmptyMatchReturnsEof) { ASSERT_OK_AND_ASSIGN(auto reader, LateMaterializingFileBatchReader::Create( std::move(mock), GetArrowPool(pool_))); auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"k", - FieldType::BIGINT, Literal(100l)); + FieldType::BIGINT, Literal(int64_t{100})); ASSERT_OK(SetReadSchema(reader.get(), arrow::schema(full_fields_), predicate, std::nullopt)); ASSERT_OK_AND_ASSIGN(std::vector rows, Collect(reader.get())); @@ -461,7 +461,7 @@ TEST_F(LateMaterializingFileBatchReaderTest, SeekToRowRealignsProbeCursor) { ASSERT_OK_AND_ASSIGN(auto reader, LateMaterializingFileBatchReader::Create( std::move(mock), GetArrowPool(pool_))); auto predicate = PredicateBuilder::GreaterOrEqual(/*field_index=*/0, /*field_name=*/"k", - FieldType::BIGINT, Literal(5l)); + FieldType::BIGINT, Literal(int64_t{5})); ASSERT_OK(SetReadSchema(reader.get(), arrow::schema(full_fields_), predicate, std::nullopt)); // First payload batch triggers the probe scan; matched rows are 5..9. @@ -490,7 +490,7 @@ TEST_F(LateMaterializingFileBatchReaderTest, ReadRangesForwardedAcrossPhases) { ASSERT_OK_AND_ASSIGN(auto reader, LateMaterializingFileBatchReader::Create( std::move(mock), GetArrowPool(pool_))); auto predicate = PredicateBuilder::GreaterOrEqual(/*field_index=*/0, /*field_name=*/"k", - FieldType::BIGINT, Literal(2l)); + FieldType::BIGINT, Literal(int64_t{2})); ASSERT_OK(SetReadSchema(reader.get(), arrow::schema(full_fields_), predicate, std::nullopt)); std::vector> ranges = {{0, 8}}; @@ -520,7 +520,7 @@ TEST_F(LateMaterializingFileBatchReaderTest, ReentrantSetReadSchema) { std::move(mock), GetArrowPool(pool_))); auto predicate1 = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"k", - FieldType::BIGINT, Literal(7l)); + FieldType::BIGINT, Literal(int64_t{7})); ASSERT_OK(SetReadSchema(reader.get(), arrow::schema(full_fields_), predicate1, std::nullopt)); ASSERT_OK_AND_ASSIGN(std::vector rows1, Collect(reader.get())); ASSERT_EQ(rows1.size(), 2u); // k = 8,9 @@ -532,7 +532,7 @@ TEST_F(LateMaterializingFileBatchReaderTest, ReentrantSetReadSchema) { } auto predicate2 = PredicateBuilder::LessThan(/*field_index=*/0, /*field_name=*/"k", - FieldType::BIGINT, Literal(3l)); + FieldType::BIGINT, Literal(int64_t{3})); ASSERT_OK(SetReadSchema(reader.get(), arrow::schema(full_fields_), predicate2, std::nullopt)); ASSERT_OK_AND_ASSIGN(std::vector rows2, Collect(reader.get())); ASSERT_EQ(rows2.size(), 3u); // k = 0,1,2 @@ -567,10 +567,10 @@ TEST_F(LateMaterializingFileBatchReaderTest, MultiFieldPreservesColumnOrder) { ASSERT_OK_AND_ASSIGN(auto reader, LateMaterializingFileBatchReader::Create( std::move(mock), GetArrowPool(pool_))); // probe columns = {a (idx0), c (idx2)}; payload columns = {b, d, e} - auto pred_a = - PredicateBuilder::GreaterOrEqual(/*field_index=*/0, "a", FieldType::BIGINT, Literal(3l)); - auto pred_c = - PredicateBuilder::LessThan(/*field_index=*/2, "c", FieldType::BIGINT, Literal(700l)); + auto pred_a = PredicateBuilder::GreaterOrEqual(/*field_index=*/0, "a", FieldType::BIGINT, + Literal(int64_t{3})); + auto pred_c = PredicateBuilder::LessThan(/*field_index=*/2, "c", FieldType::BIGINT, + Literal(int64_t{700})); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({pred_a, pred_c})); ASSERT_OK(SetReadSchema(reader.get(), arrow::schema(type->fields()), predicate, std::nullopt)); @@ -611,8 +611,8 @@ TEST_F(LateMaterializingFileBatchReaderTest, NestedPayloadColumn) { ASSERT_OK_AND_ASSIGN(auto reader, LateMaterializingFileBatchReader::Create( std::move(mock), GetArrowPool(pool_))); // probe = {k}; payload = {arr (list), tag} - auto predicate = - PredicateBuilder::GreaterOrEqual(/*field_index=*/0, "k", FieldType::BIGINT, Literal(5l)); + auto predicate = PredicateBuilder::GreaterOrEqual(/*field_index=*/0, "k", FieldType::BIGINT, + Literal(int64_t{5})); ASSERT_OK(SetReadSchema(reader.get(), arrow::schema(type->fields()), predicate, std::nullopt)); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -650,8 +650,8 @@ TEST_F(LateMaterializingFileBatchReaderTest, DictionaryPayloadColumnKeepsEncodin std::move(mock), GetArrowPool(pool_))); // probe = {k}; payload = {v}. k >= 5 matches only row 5 of the batch holding rows 3..5, a run // at batch offset 2, and both rows of the batch holding rows 6..7, a run at batch offset 0. - auto predicate = - PredicateBuilder::GreaterOrEqual(/*field_index=*/0, "k", FieldType::BIGINT, Literal(5l)); + auto predicate = PredicateBuilder::GreaterOrEqual(/*field_index=*/0, "k", FieldType::BIGINT, + Literal(int64_t{5})); ASSERT_OK(SetReadSchema(reader.get(), arrow::schema(type->fields()), predicate, std::nullopt)); std::vector> rows; @@ -708,8 +708,8 @@ TEST_F(LateMaterializingFileBatchReaderTest, WorksAsInnerOfPrefetchReader) { /*enable_adaptive_prefetch_strategy=*/false, executor, /*initialize_read_ranges=*/false, /*read_ahead_cache_enabled=*/false, CacheConfig(), /*enable_io_metrics=*/false, WarmupLevel::DECODED, pool_, GetArrowPool(pool_))); - auto predicate = - PredicateBuilder::GreaterOrEqual(/*field_index=*/0, "k", FieldType::BIGINT, Literal(4l)); + auto predicate = PredicateBuilder::GreaterOrEqual(/*field_index=*/0, "k", FieldType::BIGINT, + Literal(int64_t{4})); ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(full_fields_), &c_schema).ok()); ASSERT_OK(impl->SetReadSchema(&c_schema, predicate, std::nullopt)); @@ -746,8 +746,8 @@ TEST_F(LateMaterializingFileBatchReaderTest, PrefetchInnerReentrantSetReadSchema /*enable_io_metrics=*/false, WarmupLevel::DECODED, pool_, GetArrowPool(pool_))); auto full_schema = arrow::schema(full_fields_); - auto predicate1 = - PredicateBuilder::GreaterThan(/*field_index=*/0, "k", FieldType::BIGINT, Literal(6l)); + auto predicate1 = PredicateBuilder::GreaterThan(/*field_index=*/0, "k", FieldType::BIGINT, + Literal(int64_t{6})); ::ArrowSchema c_schema1; ASSERT_TRUE(arrow::ExportSchema(*full_schema, &c_schema1).ok()); ASSERT_OK(impl->SetReadSchema(&c_schema1, predicate1, std::nullopt)); @@ -765,7 +765,7 @@ TEST_F(LateMaterializingFileBatchReaderTest, PrefetchInnerReentrantSetReadSchema } auto predicate2 = - PredicateBuilder::LessThan(/*field_index=*/0, "k", FieldType::BIGINT, Literal(3l)); + PredicateBuilder::LessThan(/*field_index=*/0, "k", FieldType::BIGINT, Literal(int64_t{3})); ::ArrowSchema c_schema2; ASSERT_TRUE(arrow::ExportSchema(*full_schema, &c_schema2).ok()); ASSERT_OK(impl->SetReadSchema(&c_schema2, predicate2, std::nullopt)); @@ -807,8 +807,8 @@ TEST_F(LateMaterializingFileBatchReaderTest, PrefetchInnerParallelReadersWithSee /*enable_adaptive_prefetch_strategy=*/false, executor, /*initialize_read_ranges=*/false, /*read_ahead_cache_enabled=*/false, CacheConfig(), /*enable_io_metrics=*/false, WarmupLevel::DECODED, pool_, GetArrowPool(pool_))); - auto predicate = - PredicateBuilder::GreaterOrEqual(/*field_index=*/0, "k", FieldType::BIGINT, Literal(5l)); + auto predicate = PredicateBuilder::GreaterOrEqual(/*field_index=*/0, "k", FieldType::BIGINT, + Literal(int64_t{5})); ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(full_fields_), &c_schema).ok()); ASSERT_OK(impl->SetReadSchema(&c_schema, predicate, std::nullopt)); @@ -862,8 +862,8 @@ TEST_F(LateMaterializingFileBatchReaderTest, PayloadPreBufferRangesReportedToCal reported.push_back(std::move(ranges)); batches_when_reported.push_back(batches); }); - auto predicate = - PredicateBuilder::GreaterOrEqual(/*field_index=*/0, "k", FieldType::BIGINT, Literal(4l)); + auto predicate = PredicateBuilder::GreaterOrEqual(/*field_index=*/0, "k", FieldType::BIGINT, + Literal(int64_t{4})); ASSERT_OK(SetReadSchema(reader.get(), arrow::schema(full_fields_), predicate, std::nullopt)); // What is known up front is the probe projection's ranges, not the payload's. ASSERT_OK_AND_ASSIGN(ByteRanges up_front, reader->PreBufferRange()); @@ -915,8 +915,8 @@ TEST_F(LateMaterializingFileBatchReaderTest, EmptyMatchDoesNotReportPayloadRange int32_t calls = 0; reader->SetPreBufferRangeCallback( [&calls](std::vector>&&) { calls++; }); - auto predicate = - PredicateBuilder::GreaterThan(/*field_index=*/0, "k", FieldType::BIGINT, Literal(100l)); + auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, "k", FieldType::BIGINT, + Literal(int64_t{100})); ASSERT_OK(SetReadSchema(reader.get(), arrow::schema(full_fields_), predicate, std::nullopt)); ASSERT_OK_AND_ASSIGN(std::vector rows, Collect(reader.get())); @@ -935,8 +935,8 @@ TEST_F(LateMaterializingFileBatchReaderTest, PayloadPreBufferRangeErrorFailsRead int32_t calls = 0; reader->SetPreBufferRangeCallback( [&calls](std::vector>&&) { calls++; }); - auto predicate = - PredicateBuilder::GreaterOrEqual(/*field_index=*/0, "k", FieldType::BIGINT, Literal(2l)); + auto predicate = PredicateBuilder::GreaterOrEqual(/*field_index=*/0, "k", FieldType::BIGINT, + Literal(int64_t{2})); ASSERT_OK(SetReadSchema(reader.get(), arrow::schema(full_fields_), predicate, std::nullopt)); ASSERT_NOK_WITH_MSG(reader->NextBatch(), "pre-buffer range unavailable"); @@ -974,7 +974,7 @@ TEST_F(LateMaterializingFileBatchReaderTest, DisabledPrefetchCacheSkipsRangeQuer /*initialize_read_ranges=*/false, /*read_ahead_cache_enabled=*/false, CacheConfig(), /*enable_io_metrics=*/false, level, pool_, GetArrowPool(pool_))); auto predicate = PredicateBuilder::GreaterOrEqual(/*field_index=*/0, "k", FieldType::BIGINT, - Literal(4l)); + Literal(int64_t{4})); ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(full_fields_), &c_schema).ok()); ASSERT_OK(reader->SetReadSchema(&c_schema, predicate, std::nullopt)); @@ -1016,8 +1016,8 @@ TEST_F(LateMaterializingFileBatchReaderTest, PrefetchInnerRegistersPayloadPreBuf /*enable_adaptive_prefetch_strategy=*/false, executor, /*initialize_read_ranges=*/false, /*read_ahead_cache_enabled=*/true, CacheConfig(), /*enable_io_metrics=*/false, WarmupLevel::DECODED, pool_, GetArrowPool(pool_))); - auto predicate = - PredicateBuilder::GreaterOrEqual(/*field_index=*/0, "k", FieldType::BIGINT, Literal(4l)); + auto predicate = PredicateBuilder::GreaterOrEqual(/*field_index=*/0, "k", FieldType::BIGINT, + Literal(int64_t{4})); ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(full_fields_), &c_schema).ok()); ASSERT_OK(impl->SetReadSchema(&c_schema, predicate, std::nullopt)); diff --git a/src/paimon/common/reader/predicate_batch_reader_test.cpp b/src/paimon/common/reader/predicate_batch_reader_test.cpp index a47159a8c..f872eee74 100644 --- a/src/paimon/common/reader/predicate_batch_reader_test.cpp +++ b/src/paimon/common/reader/predicate_batch_reader_test.cpp @@ -102,7 +102,7 @@ TEST_F(PredicateBatchReaderTest, TestSimple) { auto reader = std::make_unique(data_array, data_type_, /*batch_size=*/10); auto predicate = PredicateBuilder::LessThan(/*field_index=*/1, /*field_name=*/"f1", - FieldType::BIGINT, Literal(24l)); + FieldType::BIGINT, Literal(int64_t{24})); auto expected_array = std::make_shared(data_array->Slice(0, 24)); CheckResult(std::move(reader), predicate, expected_array); } @@ -110,7 +110,7 @@ TEST_F(PredicateBatchReaderTest, TestSimple) { auto reader = std::make_unique(data_array, data_type_, /*batch_size=*/10); auto predicate = PredicateBuilder::LessThan(/*field_index=*/1, /*field_name=*/"f1", - FieldType::BIGINT, Literal(1l)); + FieldType::BIGINT, Literal(int64_t{1})); auto expected_array = std::make_shared(data_array->Slice(0, 1)); CheckResult(std::move(reader), predicate, expected_array); } @@ -118,7 +118,7 @@ TEST_F(PredicateBatchReaderTest, TestSimple) { auto reader = std::make_unique(data_array, data_type_, /*batch_size=*/10); auto predicate = PredicateBuilder::LessThan(/*field_index=*/1, /*field_name=*/"f1", - FieldType::BIGINT, Literal(99l)); + FieldType::BIGINT, Literal(int64_t{99})); auto expected_array = std::make_shared(data_array->Slice(0, 99)); CheckResult(std::move(reader), predicate, expected_array); } @@ -126,7 +126,7 @@ TEST_F(PredicateBatchReaderTest, TestSimple) { auto reader = std::make_unique(data_array, data_type_, /*batch_size=*/10); auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/1, /*field_name=*/"f1", - FieldType::BIGINT, Literal(0l)); + FieldType::BIGINT, Literal(int64_t{0})); auto expected_array = std::make_shared(data_array->Slice(1, 99)); CheckResult(std::move(reader), predicate, expected_array); } @@ -134,7 +134,7 @@ TEST_F(PredicateBatchReaderTest, TestSimple) { auto reader = std::make_unique(data_array, data_type_, /*batch_size=*/10); auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/1, /*field_name=*/"f1", - FieldType::BIGINT, Literal(98l)); + FieldType::BIGINT, Literal(int64_t{98})); auto expected_array = std::make_shared(data_array->Slice(99, 1)); CheckResult(std::move(reader), predicate, expected_array); } @@ -154,7 +154,7 @@ TEST_F(PredicateBatchReaderTest, TestVariousBatchSize) { .ValueOrDie(); auto expected_array = std::make_shared(PrepareArray(8)); auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/1, /*field_name=*/"f1", - FieldType::BIGINT, Literal(-1l)); + FieldType::BIGINT, Literal(int64_t{-1})); for (auto batch_size : {5, 10, 11, 20}) { auto reader = std::make_unique(data_array, data_type_, batch_size); CheckResult(std::move(reader), predicate, expected_array); @@ -178,7 +178,7 @@ TEST_F(PredicateBatchReaderTest, TestBindFieldIndexByName) { auto data_array = PrepareArray(8); auto reader = std::make_unique(data_array, data_type_, /*batch_size=*/10); auto predicate = PredicateBuilder::LessThan( - /*field_index=*/0, /*field_name=*/"f1", FieldType::BIGINT, Literal(3l)); + /*field_index=*/0, /*field_name=*/"f1", FieldType::BIGINT, Literal(int64_t{3})); auto expected_array = std::make_shared(data_array->Slice(0, 3)); CheckResult(std::move(reader), predicate, expected_array); } @@ -189,7 +189,7 @@ TEST_F(PredicateBatchReaderTest, TestFullAndEmptyCase) { auto reader = std::make_unique(data_array, data_type_, /*batch_size=*/10); auto predicate = PredicateBuilder::LessThan(/*field_index=*/1, /*field_name=*/"f1", - FieldType::BIGINT, Literal(20l)); + FieldType::BIGINT, Literal(int64_t{20})); auto expected_array = std::make_shared(data_array); CheckResult(std::move(reader), predicate, expected_array); } @@ -197,7 +197,7 @@ TEST_F(PredicateBatchReaderTest, TestFullAndEmptyCase) { auto reader = std::make_unique(data_array, data_type_, /*batch_size=*/10); auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/1, /*field_name=*/"f1", - FieldType::BIGINT, Literal(20l)); + FieldType::BIGINT, Literal(int64_t{20})); CheckResult(std::move(reader), predicate, nullptr); } } diff --git a/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp b/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp index a741077e9..da69b6447 100644 --- a/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp +++ b/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp @@ -19,6 +19,7 @@ #include "paimon/common/reader/prefetch_file_batch_reader_impl.h" #include +#include #include #include @@ -1247,9 +1248,9 @@ TEST_P(PrefetchFileBatchReaderImplTest, TestPrefetchWithPredicatePushdownWithCom ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::Or({ PredicateBuilder::LessThan(/*field_index=*/1, /*field_name=*/"f1", - FieldType::BIGINT, Literal(20l)), + FieldType::BIGINT, Literal(int64_t{20})), PredicateBuilder::GreaterThan(/*field_index=*/1, /*field_name=*/"f1", - FieldType::BIGINT, Literal(70l)), + FieldType::BIGINT, Literal(int64_t{70})), })); auto reader = PreparePrefetchReader(file_format, schema.get(), predicate, @@ -1293,9 +1294,9 @@ TEST_P(PrefetchFileBatchReaderImplTest, ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::Or({ PredicateBuilder::LessThan(/*field_index=*/1, /*field_name=*/"f1", - FieldType::BIGINT, Literal(20l)), + FieldType::BIGINT, Literal(int64_t{20})), PredicateBuilder::GreaterThan(/*field_index=*/1, /*field_name=*/"f1", - FieldType::BIGINT, Literal(70l)), + FieldType::BIGINT, Literal(int64_t{70})), })); auto reader = PreparePrefetchReader(file_format, schema.get(), predicate, @@ -1357,9 +1358,9 @@ TEST_P(PrefetchFileBatchReaderImplTest, TestRowMapping) { auto predicate, PredicateBuilder::Or({ PredicateBuilder::Between(/*field_index=*/1, /*field_name=*/"f1", FieldType::BIGINT, - Literal(20l), Literal(29l)), + Literal(int64_t{20}), Literal(int64_t{29})), PredicateBuilder::Between(/*field_index=*/1, /*field_name=*/"f1", FieldType::BIGINT, - Literal(70l), Literal(79l)), + Literal(int64_t{70}), Literal(int64_t{79})), })); auto reader = PreparePrefetchReader(file_format, schema.get(), predicate, @@ -1389,7 +1390,7 @@ TEST_P(PrefetchFileBatchReaderImplTest, TestRowMapping) { std::unique_ptr c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*schema, c_schema.get()).ok()); predicate = PredicateBuilder::Between(/*field_index=*/1, /*field_name=*/"f1", FieldType::BIGINT, - Literal(30l), Literal(49l)); + Literal(int64_t{30}), Literal(int64_t{49})); ASSERT_OK(reader->SetReadSchema(c_schema.get(), predicate, std::nullopt)); std::shared_ptr metrics_after_schema = reader->GetReaderMetrics(); ASSERT_OK_AND_ASSIGN(uint64_t consumed_after_schema, diff --git a/src/paimon/common/utils/date_time_utils_test.cpp b/src/paimon/common/utils/date_time_utils_test.cpp index 7d32c6d9c..231c63f0d 100644 --- a/src/paimon/common/utils/date_time_utils_test.cpp +++ b/src/paimon/common/utils/date_time_utils_test.cpp @@ -19,7 +19,9 @@ #include "paimon/common/utils/date_time_utils.h" +#include #include +#include #include "gtest/gtest.h" #include "paimon/common/utils/checked_cast.h" @@ -32,71 +34,71 @@ TEST(DateTimeUtilsTest, TestTimestampConverter) { { auto ret = DateTimeUtils::TimestampConverter( 10L, DateTimeUtils::SECOND, DateTimeUtils::MILLISECOND, DateTimeUtils::NANOSECOND); - ASSERT_EQ(ret, std::make_pair(10000L, 0L)); + ASSERT_EQ(ret, (std::pair{10000, 0})); } { auto ret = DateTimeUtils::TimestampConverter( 10L, DateTimeUtils::SECOND, DateTimeUtils::NANOSECOND, DateTimeUtils::NANOSECOND); - ASSERT_EQ(ret, std::make_pair(10000000000L, 0L)); + ASSERT_EQ(ret, (std::pair{10000000000, 0})); } { auto ret = DateTimeUtils::TimestampConverter( 10L, DateTimeUtils::SECOND, DateTimeUtils::NANOSECOND, DateTimeUtils::SECOND); - ASSERT_EQ(ret, std::make_pair(10000000000L, 0L)); + ASSERT_EQ(ret, (std::pair{10000000000, 0})); } { auto ret = DateTimeUtils::TimestampConverter( 2567L, DateTimeUtils::MILLISECOND, DateTimeUtils::SECOND, DateTimeUtils::NANOSECOND); - ASSERT_EQ(ret, std::make_pair(2L, 567000000L)); + ASSERT_EQ(ret, (std::pair{2, 567000000})); } { auto ret = DateTimeUtils::TimestampConverter(2567L, DateTimeUtils::MILLISECOND, DateTimeUtils::SECOND, DateTimeUtils::SECOND); - ASSERT_EQ(ret, std::make_pair(2L, 0L)); + ASSERT_EQ(ret, (std::pair{2, 0})); } { auto ret = DateTimeUtils::TimestampConverter( 2567L, DateTimeUtils::MILLISECOND, DateTimeUtils::MICROSECOND, DateTimeUtils::SECOND); - ASSERT_EQ(ret, std::make_pair(2567000L, 0L)); + ASSERT_EQ(ret, (std::pair{2567000, 0})); } { auto ret = DateTimeUtils::TimestampConverter(2567L, DateTimeUtils::MILLISECOND, DateTimeUtils::MICROSECOND, DateTimeUtils::NANOSECOND); - ASSERT_EQ(ret, std::make_pair(2567000L, 0L)); + ASSERT_EQ(ret, (std::pair{2567000, 0})); } { auto ret = DateTimeUtils::TimestampConverter(12345678L, DateTimeUtils::NANOSECOND, DateTimeUtils::MILLISECOND, DateTimeUtils::NANOSECOND); - ASSERT_EQ(ret, std::make_pair(12L, 345678L)); + ASSERT_EQ(ret, (std::pair{12, 345678})); } { auto ret = DateTimeUtils::TimestampConverter(12345678L, DateTimeUtils::NANOSECOND, DateTimeUtils::MILLISECOND, DateTimeUtils::MICROSECOND); - ASSERT_EQ(ret, std::make_pair(12L, 345L)); + ASSERT_EQ(ret, (std::pair{12, 345})); } { auto ret = DateTimeUtils::TimestampConverter(12345678L, DateTimeUtils::NANOSECOND, DateTimeUtils::MILLISECOND, DateTimeUtils::SECOND); - ASSERT_EQ(ret, std::make_pair(12L, 0L)); + ASSERT_EQ(ret, (std::pair{12, 0})); } { auto ret = DateTimeUtils::TimestampConverter( -2240521239998998999L, DateTimeUtils::NANOSECOND, DateTimeUtils::MILLISECOND, DateTimeUtils::NANOSECOND); - ASSERT_EQ(ret, std::make_pair(-2240521239999L, 1001L)); + ASSERT_EQ(ret, (std::pair{-2240521239999, 1001})); } { // 9999-12-31 auto ret = DateTimeUtils::TimestampConverter(253402252995L, DateTimeUtils::SECOND, DateTimeUtils::MILLISECOND, DateTimeUtils::NANOSECOND); - ASSERT_EQ(ret, std::make_pair(253402252995000L, 0L)); + ASSERT_EQ(ret, (std::pair{253402252995000, 0})); } } diff --git a/src/paimon/common/utils/options_utils_test.cpp b/src/paimon/common/utils/options_utils_test.cpp index c1ff05572..39661c283 100644 --- a/src/paimon/common/utils/options_utils_test.cpp +++ b/src/paimon/common/utils/options_utils_test.cpp @@ -51,9 +51,9 @@ TEST(OptionsUtilsTest, TestGetValueFromMap) { ASSERT_NOK_WITH_MSG(OptionsUtils::GetValueFromMap(key_value_map, "key_int8"), "convert key key_int8, value 500 to signed char failed"); ASSERT_NOK_WITH_MSG(OptionsUtils::GetValueFromMap(key_value_map, "key_int64"), - "convert key key_int64, value ab to long failed"); + "convert key key_int64, value ab to "); ASSERT_NOK_WITH_MSG(OptionsUtils::GetValueFromMap(key_value_map, "key_int64", 10), - "convert key key_int64, value ab to long failed"); + "convert key key_int64, value ab to "); ASSERT_OK_AND_ASSIGN( auto nonexist, OptionsUtils::GetValueFromMap(key_value_map, "key_nonexist", 233)); diff --git a/src/paimon/common/utils/string_utils.cpp b/src/paimon/common/utils/string_utils.cpp index 2e3565153..681cf41c6 100644 --- a/src/paimon/common/utils/string_utils.cpp +++ b/src/paimon/common/utils/string_utils.cpp @@ -26,6 +26,7 @@ #include #include +#include "arrow/vendored/datetime.h" #include "fmt/format.h" #include "paimon/status.h" @@ -228,20 +229,21 @@ Result StringUtils::StringToDate(const std::string& str) { if (int_value) { return int_value.value(); } - std::tm timeinfo = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, nullptr}; + std::tm timeinfo{}; std::istringstream ss(str); ss >> std::get_time(&timeinfo, "%Y-%m-%d"); if (ss.fail()) { return Status::Invalid(fmt::format("failed to convert string '{}' to date", str)); } - int32_t orig_mon = timeinfo.tm_mon; - int32_t orig_mday = timeinfo.tm_mday; - std::time_t time = timegm(&timeinfo); - if (time == -1 || timeinfo.tm_mon != orig_mon || timeinfo.tm_mday != orig_mday) { + // Calendar arithmetic avoids platform-specific timegm limits for early years. + arrow_vendored::date::year_month_day date{ + arrow_vendored::date::year{timeinfo.tm_year + 1900}, + arrow_vendored::date::month{static_cast(timeinfo.tm_mon + 1)}, + arrow_vendored::date::day{static_cast(timeinfo.tm_mday)}}; + if (!date.ok()) { return Status::Invalid(fmt::format("failed to convert string '{}' to date", str)); } - static const int64_t SECONDS_PER_DAY = 86400l; // = 24 * 60 * 60 - return time / SECONDS_PER_DAY; + return arrow_vendored::date::sys_days{date}.time_since_epoch().count(); } /// Parses a timestamp string into unix milliseconds. diff --git a/src/paimon/common/utils/string_utils_test.cpp b/src/paimon/common/utils/string_utils_test.cpp index 89b0334f1..e95b65092 100644 --- a/src/paimon/common/utils/string_utils_test.cpp +++ b/src/paimon/common/utils/string_utils_test.cpp @@ -460,6 +460,14 @@ TEST_F(StringUtilsTest, TestStringToDate) { ASSERT_OK_AND_ASSIGN(auto date, StringUtils::StringToDate("0000-01-01")); ASSERT_EQ(date, -719528); } + { + ASSERT_OK_AND_ASSIGN(int32_t date, StringUtils::StringToDate("0000-02-29")); + ASSERT_EQ(date, -719469); + } + { + ASSERT_OK_AND_ASSIGN(int32_t date, StringUtils::StringToDate("0001-01-01")); + ASSERT_EQ(date, -719162); + } { ASSERT_OK_AND_ASSIGN(auto date, StringUtils::StringToDate("9999-12-31")); ASSERT_EQ(date, 2932896); @@ -472,6 +480,8 @@ TEST_F(StringUtilsTest, TestStringToDate) { ASSERT_NOK(StringUtils::StringToDate("1970-XX-02")); ASSERT_NOK(StringUtils::StringToDate("2023-02-29")); ASSERT_NOK(StringUtils::StringToDate("2024-00-01")); + ASSERT_NOK(StringUtils::StringToDate("0000-02-30")); + ASSERT_NOK(StringUtils::StringToDate("1900-02-29")); } TEST_F(StringUtilsTest, TestStringToTimestampMillis) { diff --git a/src/paimon/core/compact/compact_deletion_file.h b/src/paimon/core/compact/compact_deletion_file.h index 70d23720a..e97bb63d0 100644 --- a/src/paimon/core/compact/compact_deletion_file.h +++ b/src/paimon/core/compact/compact_deletion_file.h @@ -128,8 +128,8 @@ class LazyCompactDeletionFile : public CompactDeletionFile, inline Result> CompactDeletionFile::GenerateFiles( const std::shared_ptr& maintainer) { - PAIMON_ASSIGN_OR_RAISE(std::optional> file, - maintainer->WriteDeletionVectorsIndex()); + std::optional> file; + PAIMON_ASSIGN_OR_RAISE(file, maintainer->WriteDeletionVectorsIndex()); return std::make_shared(file.value_or(nullptr), maintainer->DvIndexFile()); } diff --git a/src/paimon/core/operation/precise_file_index_read.cpp b/src/paimon/core/operation/precise_file_index_read.cpp index 79daca660..8a9204dec 100644 --- a/src/paimon/core/operation/precise_file_index_read.cpp +++ b/src/paimon/core/operation/precise_file_index_read.cpp @@ -103,9 +103,9 @@ Status PreparePredicate( mapped_leaves->emplace(leaf, nullptr); return Status::OK(); } - PAIMON_ASSIGN_OR_RAISE( - std::optional> mapped, - FieldMappingBuilder::ReconstructPredicateWithDataFields(leaf, read_by_name, data_by_id)); + std::optional> mapped; + PAIMON_ASSIGN_OR_RAISE(mapped, FieldMappingBuilder::ReconstructPredicateWithDataFields( + leaf, read_by_name, data_by_id)); std::shared_ptr mapped_leaf; if (mapped) { mapped_leaf = std::dynamic_pointer_cast(*mapped); diff --git a/src/paimon/core/postpone/postpone_bucket_writer_test.cpp b/src/paimon/core/postpone/postpone_bucket_writer_test.cpp index dd20eca48..d0305204b 100644 --- a/src/paimon/core/postpone/postpone_bucket_writer_test.cpp +++ b/src/paimon/core/postpone/postpone_bucket_writer_test.cpp @@ -149,14 +149,15 @@ class PostponeBucketWriterTest : public ::testing::Test, }; std::vector GetTestValuesForPostponeBucketWriterTest() { - std::vector values = {"parquet"}; + return { + "parquet", #ifdef PAIMON_ENABLE_ORC - values.emplace_back("orc"); + "orc", #endif #ifdef PAIMON_ENABLE_AVRO - values.emplace_back("avro"); + "avro", #endif - return values; + }; } INSTANTIATE_TEST_SUITE_P(FileFormat, PostponeBucketWriterTest, diff --git a/src/paimon/core/realtime/arrow_realtime_store_test.cpp b/src/paimon/core/realtime/arrow_realtime_store_test.cpp index 27dd14b9d..a21c453bf 100644 --- a/src/paimon/core/realtime/arrow_realtime_store_test.cpp +++ b/src/paimon/core/realtime/arrow_realtime_store_test.cpp @@ -135,8 +135,8 @@ class ArrowRealtimeStoreTest : public testing::Test { } // namespace TEST_F(ArrowRealtimeStoreTest, TestWriteValidationAndSeal) { - ASSERT_OK_AND_ASSIGN(std::optional> empty_segment, - store_->SealForCommit()); + std::optional> empty_segment; + ASSERT_OK_AND_ASSIGN(empty_segment, store_->SealForCommit()); ASSERT_FALSE(empty_segment.has_value()); ASSERT_NOK_WITH_MSG(store_->Write(RealtimeWriteBatch{nullptr, OffsetRange(0, 1)}), @@ -181,8 +181,8 @@ TEST_F(ArrowRealtimeStoreTest, TestDataUsageTracksBuildingSealedAndReclaimedData ASSERT_EQ(usage.building_memory_bytes, store_->GetMemoryUsage()); const uint64_t first_segment_memory = usage.building_memory_bytes; - ASSERT_OK_AND_ASSIGN(std::optional> segment, - store_->SealForCommit()); + std::optional> segment; + ASSERT_OK_AND_ASSIGN(segment, store_->SealForCommit()); ASSERT_TRUE(segment.has_value()); usage = store_->GetDataUsage(); ASSERT_EQ(0, usage.building_memory_bytes); diff --git a/src/paimon/rest/mock_rest_server.cpp b/src/paimon/rest/mock_rest_server.cpp index e9f0b36b7..524633266 100644 --- a/src/paimon/rest/mock_rest_server.cpp +++ b/src/paimon/rest/mock_rest_server.cpp @@ -18,7 +18,9 @@ #include "paimon/rest/mock_rest_server.h" +#include #include +#include #include #include #include @@ -89,6 +91,12 @@ Result> MockRestServer::Start(Handler handler) { ::close(listen_fd); return Status::IOError("mock rest server: failed to listen: ", std::strerror(errno)); } + int32_t flags = ::fcntl(listen_fd, F_GETFL, 0); + if (flags < 0 || ::fcntl(listen_fd, F_SETFL, flags | O_NONBLOCK) < 0) { + ::close(listen_fd); + return Status::IOError("mock rest server: failed to set nonblocking mode: ", + std::strerror(errno)); + } socklen_t address_len = sizeof(address); if (::getsockname(listen_fd, reinterpret_cast(&address), &address_len) < 0) { ::close(listen_fd); @@ -106,9 +114,8 @@ void MockRestServer::Stop() { if (stopped_.exchange(true)) { return; } - // `shutdown` wakes the blocked `accept`; the fd is closed only after the accept - // thread has joined, so it cannot be recycled by another thread while the accept - // thread might still use it. + // Some platforms do not wake a listening socket on shutdown. The bounded poll + // in AcceptLoop also observes stopped_. Close only after joining to avoid fd reuse. ::shutdown(listen_fd_, SHUT_RDWR); if (accept_thread_.joinable()) { accept_thread_.join(); @@ -122,6 +129,10 @@ std::string MockRestServer::GetBaseUri() const { void MockRestServer::AcceptLoop() { while (!stopped_.load()) { + struct pollfd descriptor{listen_fd_, POLLIN, 0}; + if (::poll(&descriptor, 1, 100) <= 0 || stopped_.load()) { + continue; + } int32_t connection_fd = ::accept(listen_fd_, nullptr, nullptr); if (connection_fd < 0) { if (stopped_.load()) { @@ -129,6 +140,11 @@ void MockRestServer::AcceptLoop() { } continue; } + int32_t flags = ::fcntl(connection_fd, F_GETFL, 0); + if (flags < 0 || ::fcntl(connection_fd, F_SETFL, flags & ~O_NONBLOCK) < 0) { + ::close(connection_fd); + continue; + } // The connection is handled on the accept thread; the socket timeouts keep a // wedged peer from blocking `Stop()` indefinitely. struct timeval timeout; diff --git a/src/paimon/rest/rest_http_client_test.cpp b/src/paimon/rest/rest_http_client_test.cpp index 28d7c63dc..90df34202 100644 --- a/src/paimon/rest/rest_http_client_test.cpp +++ b/src/paimon/rest/rest_http_client_test.cpp @@ -53,6 +53,17 @@ TEST(RestHttpClientTest, BuildQueryString) { ASSERT_EQ("a=1&b=x+y%2Fz", RestHttpClient::BuildQueryString({{"a", "1"}, {"b", "x y/z"}})); } +TEST(RestHttpClientTest, StopIdleServer) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([](const MockRestServer::Request&) { + return MockRestServer::Response(); + })); + const auto start = std::chrono::steady_clock::now(); + server->Stop(); + server->Stop(); + ASSERT_LT(std::chrono::steady_clock::now() - start, std::chrono::seconds(2)); +} + TEST(RestHttpClientTest, GetWithHeadersAndQuery) { // the handler runs on the mock server's accept thread, so every state it shares with // the test body is mutex guarded, here and in the tests below diff --git a/src/paimon/testing/utils/testharness.cpp b/src/paimon/testing/utils/testharness.cpp index d018ab082..f9e598373 100644 --- a/src/paimon/testing/utils/testharness.cpp +++ b/src/paimon/testing/utils/testharness.cpp @@ -58,6 +58,7 @@ #include #include +#include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" #include "paimon/common/utils/uuid.h" #include "paimon/fs/file_system.h" @@ -140,7 +141,7 @@ std::unique_ptr UniqueTestDirectory::Create(const std::stri static const size_t kMaxTries = 1000; std::string tmp_dir = fs_identifier == "jindo" ? GetJindoTestDir() - : std::filesystem::temp_directory_path().string() + "/"; + : std::filesystem::temp_directory_path().string(); std::map fs_options = fs_identifier == "jindo" ? GetJindoTestOptions() : std::map(); auto fs = FileSystemFactory::Get(fs_identifier, tmp_dir, fs_options); @@ -152,7 +153,7 @@ std::unique_ptr UniqueTestDirectory::Create(const std::stri if (!UUID::Generate(&uuid)) { continue; } - std::string test_dir = tmp_dir + "paimon_test_" + uuid; + std::string test_dir = PathUtil::JoinPath(tmp_dir, "paimon_test_" + uuid); auto is_exist = fs.value()->Exists(test_dir); if (!is_exist.ok() || is_exist.value()) { continue; diff --git a/test/inte/scan_inte_test.cpp b/test/inte/scan_inte_test.cpp index 3f949484d..66619b2f5 100644 --- a/test/inte/scan_inte_test.cpp +++ b/test/inte/scan_inte_test.cpp @@ -2233,7 +2233,7 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable2) { // in index filter: predicate is removed as type for f1 is converted bigint->int, which is not // safe auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"f1", - FieldType::BIGINT, Literal(100l)); + FieldType::BIGINT, Literal(int64_t{100})); context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); @@ -2308,7 +2308,7 @@ TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTableWithEmptyResult) { // child1 will remove file1 for schema1 auto child1 = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f1", FieldType::BIGINT, - Literal(100l)); + Literal(int64_t{100})); // child2 will remove file0 for schema 0 auto child2 = PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f4", FieldType::STRING, Literal(FieldType::STRING, "David", 5)); diff --git a/test/inte/write_and_read_inte_with_index_test.cpp b/test/inte/write_and_read_inte_with_index_test.cpp index 8e7ba94fa..0ae0037bd 100644 --- a/test/inte/write_and_read_inte_with_index_test.cpp +++ b/test/inte/write_and_read_inte_with_index_test.cpp @@ -576,8 +576,8 @@ class WriteAndReadInteWithIndexTest // Test f1 (BIGINT) predicates { // Test greater than predicate: f1 > 300 -> rows 3,4,7 (values 400,500,600) - auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/1, /*field_name=*/"f1", - FieldType::BIGINT, Literal(300L)); + auto predicate = PredicateBuilder::GreaterThan( + /*field_index=*/1, /*field_name=*/"f1", FieldType::BIGINT, Literal(int64_t{300})); std::shared_ptr expected_array; auto array_status = arrow::ipc::internal::json::ChunkedArrayFromJSON(arrow_data_type, {R"([ @@ -644,7 +644,7 @@ class WriteAndReadInteWithIndexTest // Test f1 BETWEEN 200 AND 500 -> rows 1,2,3,4 (values 200,300,400,500) auto predicate = PredicateBuilder::Between(/*field_index=*/1, /*field_name=*/"f1", FieldType::BIGINT, - Literal(200L), Literal(500L)); + Literal(int64_t{200}), Literal(int64_t{500})); std::shared_ptr expected_array; auto array_status = arrow::ipc::internal::json::ChunkedArrayFromJSON(arrow_data_type, {R"([ @@ -687,7 +687,7 @@ class WriteAndReadInteWithIndexTest ASSERT_OK_AND_ASSIGN(auto or_predicate, PredicateBuilder::Or({predicate1, predicate2})); auto predicate3 = PredicateBuilder::LessThan(/*field_index=*/1, /*field_name=*/"f1", - FieldType::BIGINT, Literal(200L)); + FieldType::BIGINT, Literal(int64_t{200})); ASSERT_OK_AND_ASSIGN(auto and_predicate, PredicateBuilder::And({or_predicate, predicate3})); @@ -707,8 +707,8 @@ class WriteAndReadInteWithIndexTest // Combined AND: matches rows 3,4,7 auto predicate1 = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"f0", FieldType::INT, Literal(5)); - auto predicate2 = PredicateBuilder::GreaterThan(/*field_index=*/1, /*field_name=*/"f1", - FieldType::BIGINT, Literal(100L)); + auto predicate2 = PredicateBuilder::GreaterThan( + /*field_index=*/1, /*field_name=*/"f1", FieldType::BIGINT, Literal(int64_t{100})); ASSERT_OK_AND_ASSIGN(auto and_predicate, PredicateBuilder::And({predicate1, predicate2})); @@ -2052,7 +2052,7 @@ TEST_P(WriteAndReadInteWithIndexTest, TestWithAlterTable) { { // test equal predicate for f1 auto predicate = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f1", - FieldType::BIGINT, Literal(10l)); + FieldType::BIGINT, Literal(int64_t{10})); std::shared_ptr expected_array; auto array_status = arrow::ipc::internal::json::ChunkedArrayFromJSON(arrow_data_type, {R"([ [0, 10, "Alice", 11.1, null], @@ -2069,7 +2069,7 @@ TEST_P(WriteAndReadInteWithIndexTest, TestWithAlterTable) { { // test not equal predicate for f1 auto predicate = PredicateBuilder::NotEqual(/*field_index=*/0, /*field_name=*/"f1", - FieldType::BIGINT, Literal(10l)); + FieldType::BIGINT, Literal(int64_t{10})); std::shared_ptr expected_array; auto array_status = arrow::ipc::internal::json::ChunkedArrayFromJSON(arrow_data_type, {R"([ [0, 20, "Lucy", 15.1, null], @@ -2185,7 +2185,7 @@ TEST_P(WriteAndReadInteWithIndexTest, TestWithAlterTable) { { // test greater than predicate for f1, do not take effective in bitmap index auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"f1", - FieldType::BIGINT, Literal(10l)); + FieldType::BIGINT, Literal(int64_t{10})); std::shared_ptr expected_array; auto array_status = arrow::ipc::internal::json::ChunkedArrayFromJSON(arrow_data_type, {R"([ [0, 10, "Alice", 11.1, null], @@ -2207,9 +2207,9 @@ TEST_P(WriteAndReadInteWithIndexTest, TestWithAlterTable) { } { // test in predicate - auto predicate = - PredicateBuilder::In(/*field_index=*/0, /*field_name=*/"f1", FieldType::BIGINT, - {Literal(10l), Literal(30l), Literal(50l)}); + auto predicate = PredicateBuilder::In( + /*field_index=*/0, /*field_name=*/"f1", FieldType::BIGINT, + {Literal(int64_t{10}), Literal(int64_t{30}), Literal(int64_t{50})}); std::shared_ptr expected_array; auto array_status = arrow::ipc::internal::json::ChunkedArrayFromJSON(arrow_data_type, {R"([ [0, 10, "Alice", 11.1, null], @@ -2227,9 +2227,9 @@ TEST_P(WriteAndReadInteWithIndexTest, TestWithAlterTable) { } { // test not in predicate - auto predicate = - PredicateBuilder::NotIn(/*field_index=*/0, /*field_name=*/"f1", FieldType::BIGINT, - {Literal(10l), Literal(30l), Literal(50l)}); + auto predicate = PredicateBuilder::NotIn( + /*field_index=*/0, /*field_name=*/"f1", FieldType::BIGINT, + {Literal(int64_t{10}), Literal(int64_t{30}), Literal(int64_t{50})}); std::shared_ptr expected_array; auto array_status = arrow::ipc::internal::json::ChunkedArrayFromJSON(arrow_data_type, {R"([ [0, 20, "Lucy", 15.1, null], @@ -2247,7 +2247,7 @@ TEST_P(WriteAndReadInteWithIndexTest, TestWithAlterTable) { PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f4", FieldType::STRING, Literal(FieldType::STRING, "Alice", 5)); auto f1_predicate = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f1", - FieldType::BIGINT, Literal(30l)); + FieldType::BIGINT, Literal(int64_t{30})); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({f4_predicate, f1_predicate})); std::shared_ptr expected_array; auto array_status = arrow::ipc::internal::json::ChunkedArrayFromJSON(arrow_data_type, {R"([ @@ -2263,7 +2263,7 @@ TEST_P(WriteAndReadInteWithIndexTest, TestWithAlterTable) { PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f4", FieldType::STRING, Literal(FieldType::STRING, "Alice", 5)); auto f1_predicate = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f1", - FieldType::BIGINT, Literal(30l)); + FieldType::BIGINT, Literal(int64_t{30})); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::Or({f4_predicate, f1_predicate})); std::shared_ptr expected_array; auto array_status = arrow::ipc::internal::json::ChunkedArrayFromJSON(arrow_data_type, {R"([ @@ -2282,7 +2282,7 @@ TEST_P(WriteAndReadInteWithIndexTest, TestWithAlterTable) { PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f4", FieldType::STRING, Literal(FieldType::STRING, "Alice", 5)); auto f1_predicate = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f1", - FieldType::BIGINT, Literal(40l)); + FieldType::BIGINT, Literal(int64_t{40})); auto f5_predicate = PredicateBuilder::IsNotNull(/*field_index=*/3, /*field_name=*/"f5", FieldType::INT); ASSERT_OK_AND_ASSIGN(auto predicate, @@ -2292,12 +2292,12 @@ TEST_P(WriteAndReadInteWithIndexTest, TestWithAlterTable) { { // test non result auto predicate = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f1", - FieldType::BIGINT, Literal(40l)); + FieldType::BIGINT, Literal(int64_t{40})); CheckResult(path, {split}, predicate, /*expected_array=*/nullptr); } { auto predicate = PredicateBuilder::NotEqual(/*field_index=*/0, /*field_name=*/"f1", - FieldType::BIGINT, Literal(40l)); + FieldType::BIGINT, Literal(int64_t{40})); std::shared_ptr expected_array; auto array_status = arrow::ipc::internal::json::ChunkedArrayFromJSON(arrow_data_type, {R"([ [0, 10, "Alice", 11.1, null], @@ -2322,7 +2322,7 @@ TEST_P(WriteAndReadInteWithIndexTest, TestWithAlterTable) { // so its predicate cannot use the index and must be evaluated against the data. auto predicate = PredicateBuilder::In(/*field_index=*/0, /*field_name=*/"f1", FieldType::BIGINT, - {Literal(10l), Literal(int64_t{2147483648})}); + {Literal(int64_t{10}), Literal(int64_t{2147483648})}); ReadContextBuilder builder(path); builder.EnablePreciseFileIndexEvaluate(true) .EnablePredicateFilter(false) diff --git a/test/inte/write_inte_test.cpp b/test/inte/write_inte_test.cpp index 31a325ec1..490dfc673 100644 --- a/test/inte/write_inte_test.cpp +++ b/test/inte/write_inte_test.cpp @@ -2746,7 +2746,7 @@ TEST_P(WriteInteTest, TestPKTableWriteAndReadWithExternalPath) { std::string external_path = meta->external_path.value(); ASSERT_OK_AND_ASSIGN(bool file_exist, file_system_->Exists(external_path)); ASSERT_TRUE(file_exist); - ASSERT_TRUE(external_path.find("tmp") != std::string::npos); + ASSERT_NE(external_path.find(external_dir->Str() + "/"), std::string::npos); } TEST_P(WriteInteTest, TestAppendTableStreamWriteWithExternalPath) { From 8b8909a51c533f666e88383f597980efec04398d Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Mon, 9 Nov 2026 15:00:35 +0800 Subject: [PATCH 02/10] fix(lumina): enable Linux ARM64 artifacts and CI coverage --- ci/scripts/build_paimon.sh | 3 ++- cmake_modules/ThirdpartyToolchain.cmake | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/scripts/build_paimon.sh b/ci/scripts/build_paimon.sh index ba5237f94..06ea5d45a 100755 --- a/ci/scripts/build_paimon.sh +++ b/ci/scripts/build_paimon.sh @@ -145,7 +145,8 @@ if [[ "${enable_tsan}" == "true" ]]; then fi # CI always builds natively, so the host architecture is the target architecture. host_arch=$(uname -m) -if [[ "$(uname -s)" != "Linux" || "${host_arch}" != "x86_64" ]]; then +if [[ "$(uname -s)" != "Linux" || + ( "${host_arch}" != "x86_64" && "${host_arch}" != "aarch64" && "${host_arch}" != "arm64" ) ]]; then ENABLE_LUMINA="OFF" echo "=== Lumina disabled: no prebuilt artifacts for $(uname -s)-${host_arch} ===" fi diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake index 1b51ceede..54fd655c3 100644 --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake @@ -1360,7 +1360,7 @@ endmacro() macro(build_lumina) # The release tarball ships one prebuilt artifact tree per platform; keep this # list in sync with the platforms the pinned release actually contains. - set(LUMINA_SUPPORTED_PLATFORMS "linux-x86_64") + set(LUMINA_SUPPORTED_PLATFORMS "linux-x86_64" "linux-aarch64") if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND PAIMON_TARGET_CPU_FAMILY STREQUAL "x86") set(LUMINA_ARTIFACT_PLATFORM "linux-x86_64") elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND PAIMON_TARGET_CPU_FAMILY STREQUAL From 2211d0635a504b11c391dbbd336b02e5103fb5d3 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Mon, 9 Nov 2026 15:08:52 +0800 Subject: [PATCH 03/10] ci: expand ARM64 compiler coverage and enable Avro and ORC explicitly --- .github/workflows/build_and_test.yaml | 59 ++++++++++++++++++++++++--- ci/scripts/build_paimon.sh | 2 + 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 1f0df9b90..9f7c5aae2 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -51,12 +51,29 @@ jobs: run: ci/scripts/test_asan_symbolize.sh macos-build-and-test: - name: appleclang-release-arm64 + name: ${{ matrix.name }} runs-on: macos-15 timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + include: + - name: appleclang-release-arm64 + cc: /usr/bin/clang + cxx: /usr/bin/clang++ + build_type: Release + - name: appleclang-debug-arm64 + cc: /usr/bin/clang + cxx: /usr/bin/clang++ + build_type: Debug + - name: gcc15-release-macos-arm64 + cc: /opt/homebrew/bin/gcc-15 + cxx: /opt/homebrew/bin/g++-15 + build_type: Release + gcc_formula: gcc@15 env: - CC: /usr/bin/clang - CXX: /usr/bin/clang++ + CC: ${{ matrix.cc }} + CXX: ${{ matrix.cxx }} PAIMON_BUILD_JOBS: '3' CARGO_BUILD_JOBS: '3' steps: @@ -70,10 +87,21 @@ jobs: brew install cmake ninja protobuf openssl@3 curl echo "$(brew --prefix protobuf)/bin" >> "$GITHUB_PATH" echo "CMAKE_PREFIX_PATH=$(brew --prefix openssl@3):$(brew --prefix curl)" >> "$GITHUB_ENV" + - name: Install GCC + if: matrix.gcc_formula + shell: bash + env: + GCC_FORMULA: ${{ matrix.gcc_formula }} + run: brew install "${GCC_FORMULA}" + - name: Show compiler versions + shell: bash + run: | + "${CC}" --version + "${CXX}" --version - name: Setup ccache uses: ./.github/actions/setup-ccache with: - cache-key-prefix: ccache-appleclang-release-arm64 + cache-key-prefix: ccache-${{ matrix.name }} - name: Install Rust and protobuf tools shell: bash run: ci/scripts/setup_rust.sh @@ -85,13 +113,13 @@ jobs: run: | ci/scripts/build_paimon.sh \ --source_dir "$(pwd)" \ - --build_type Release \ + --build_type "${{ matrix.build_type }}" \ --install_smoke - name: Upload test logs if: failure() uses: actions/upload-artifact@v4 with: - name: appleclang-release-arm64-test-logs + name: ${{ matrix.name }}-test-logs path: | build/build/test-logs/ build/Testing/Temporary/ @@ -138,6 +166,12 @@ jobs: cc: gcc-14 cxx: g++-14 build_args: --build_type Release + - name: gcc13-release-aarch64 + runner: ubuntu-24.04-arm + cc: gcc-13 + cxx: g++-13 + gcc_version: '13.3.0' + build_args: --build_type Release - name: clang-debug-aarch64 runner: ubuntu-24.04-arm - name: asan-ubsan-aarch64 @@ -166,6 +200,19 @@ jobs: run: | sudo apt-get update sudo apt-get install -y gdb libcurl4-openssl-dev libssl-dev + - name: Install and verify regression compiler + if: matrix.gcc_version + shell: bash + env: + CC: ${{ matrix.cc }} + CXX: ${{ matrix.cxx }} + EXPECTED_GCC_VERSION: ${{ matrix.gcc_version }} + run: | + sudo apt-get install -y "${CC}" "${CXX}" + "${CC}" --version + "${CXX}" --version + test "$("${CC}" -dumpfullversion)" = "${EXPECTED_GCC_VERSION}" + test "$("${CXX}" -dumpfullversion)" = "${EXPECTED_GCC_VERSION}" - name: Install flatc and libclang (Vortex) shell: bash run: ci/scripts/setup_flatc.sh diff --git a/ci/scripts/build_paimon.sh b/ci/scripts/build_paimon.sh index 06ea5d45a..966e84521 100755 --- a/ci/scripts/build_paimon.sh +++ b/ci/scripts/build_paimon.sh @@ -159,6 +159,8 @@ CMAKE_ARGS=( "-G Ninja" "-DCMAKE_BUILD_TYPE=${build_type}" "-DPAIMON_BUILD_TESTS=ON" + "-DPAIMON_ENABLE_AVRO=ON" + "-DPAIMON_ENABLE_ORC=ON" "-DPAIMON_ENABLE_MOSAIC=ON" "-DPAIMON_ENABLE_LANCE=${ENABLE_LANCE}" "-DPAIMON_ENABLE_VORTEX=ON" From 6401537112bfba5300d19d44d8e5b0d2518d4fed Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Mon, 9 Nov 2026 15:18:52 +0800 Subject: [PATCH 04/10] docs: align platform and CI coverage with ARM64 validation --- CONTRIBUTING.md | 6 ++++ README.md | 2 ++ docs/source/building.rst | 61 ++++++++++++++++++++++++++++++++++------ 3 files changed, 60 insertions(+), 9 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2355f95af..2eb4a0b87 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -83,6 +83,12 @@ Before submitting, please verify: - **CMake** ≥ 3.22 - **Python 3** (for linting scripts and pre-commit) +See the [build guide](docs/source/building.rst) for platform requirements and +the CI compiler matrix, feature flags, and platform-specific exclusions. +The CI entry point is `ci/scripts/build_paimon.sh`; it enables the available +optional modules and runs CTest. It defaults to Debug; use +`--build_type Release` to reproduce a Release configuration. + --- ## Code Style diff --git a/README.md b/README.md index 276bcb6e0..040a2ce53 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,8 @@ Paimon C++ currently provides: Building requires a C++17-compatible compiler and CMake 3.22 or newer. Linux `x86_64` and `aarch64` builds are supported and verified in CI. +CI also includes macOS 15 ARM64 builds with AppleClang in Debug and Release, +and Homebrew GCC 15 in Release. See the [build guide](docs/source/building.rst) for platform support, system requirements, dependency configuration, optional components, and instructions for building and running tests. diff --git a/docs/source/building.rst b/docs/source/building.rst index b0fe95e44..c7088c200 100644 --- a/docs/source/building.rst +++ b/docs/source/building.rst @@ -67,9 +67,12 @@ Supported platforms - ``aarch64`` / ``arm64`` - Built and tested in CI. * - macOS - - ``x86_64``, ``arm64`` - - Architecture detection is wired up, but macOS as a whole is not an - officially supported platform and is not covered by CI. + - ``arm64`` + - CI includes macOS 15 builds with AppleClang in Debug and Release, + and Homebrew GCC 15 in Release. Lumina is unavailable on macOS. + * - macOS + - ``x86_64`` + - Architecture detection is available; no CI coverage. * - Windows - any - Not supported. @@ -91,18 +94,58 @@ global index is the same however Paimon was compiled. One optional component is architecture-restricted: ``-DPAIMON_ENABLE_LUMINA=ON`` consumes a prebuilt binary that is only -published for Linux ``x86_64``, so configuring it on any other platform fails -with an explicit error. +published for Linux ``x86_64`` and ``aarch64``, so configuring it on any other +platform fails with an explicit error. -Cross compilation is not wired up end to end: the bundled third-party builds are -handed only the C and C++ compilers, without a toolchain file, sysroot or find -root, so they can still pick up host libraries. Build natively on each target. +Cross compilation is not wired up end to end: not all bundled third-party builds +receive a complete toolchain, sysroot and find-root configuration, so they can +still pick up host libraries. Build natively on each target. ``build_and_package.sh`` therefore names its output after the host platform, for example ``output/paimon-cpp-linux-aarch64.tar.gz``, so artifacts built on several machines can be collected side by side. ``--platform