diff --git a/.github/actions/setup-ccache/action.yml b/.github/actions/setup-ccache/action.yml index 62de2837a..fa4da936e 100644 --- a/.github/actions/setup-ccache/action.yml +++ b/.github/actions/setup-ccache/action.yml @@ -27,7 +27,12 @@ runs: steps: - name: Install ccache shell: bash - run: sudo apt-get update && sudo apt-get install -y ccache + run: | + if [[ "${RUNNER_OS}" == "macOS" ]]; then + brew install ccache + else + sudo apt-get update && sudo apt-get install -y ccache + fi - name: Restore ccache uses: actions/cache@v4 diff --git a/.github/actions/setup-release-assets/action.yml b/.github/actions/setup-release-assets/action.yml new file mode 100644 index 000000000..9a32bbd97 --- /dev/null +++ b/.github/actions/setup-release-assets/action.yml @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Prepare release assets +description: Cache and verify pinned GitHub release archives before building +runs: + using: composite + steps: + - uses: actions/cache@v4 + with: + path: ${{ runner.temp }}/paimon-release-assets + key: release-assets-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('third_party/versions.txt', 'ci/scripts/setup_rust.sh') }} + - shell: bash + run: | + set -eu + source third_party/versions.txt + cache_dir="${RUNNER_TEMP}/paimon-release-assets" + for entry in "${DEPENDENCIES[@]}"; do + read -r var archive url <<< "${entry}" + case "${var}" in + PAIMON_ZLIB_URL|PAIMON_ZSTD_URL|PAIMON_PROTOBUF_URL|PAIMON_ARROW_URL) + checksum_var="${var%_URL}_BUILD_SHA256_CHECKSUM" + bash ci/scripts/download_release_asset.sh "${url}" "${cache_dir}/${archive}" "${!checksum_var}" + echo "${var}=${cache_dir}/${archive}" >> "${GITHUB_ENV}" + ;; + esac + done + echo "PAIMON_RELEASE_ASSET_DIR=${cache_dir}" >> "${GITHUB_ENV}" diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index f9548716d..b4f0fcfa1 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -43,6 +43,9 @@ jobs: - name: Run CMake module tests shell: bash run: ci/scripts/test_cmake_modules.sh + - name: Test release asset downloads + shell: bash + run: bash ci/scripts/test_download_release_asset.sh - name: Run packaging argument tests shell: bash run: ci/scripts/test_packaging_args.sh @@ -50,6 +53,86 @@ jobs: shell: bash run: ci/scripts/test_asan_symbolize.sh + macos-build-and-test: + name: ${{ matrix.name }} + runs-on: macos-15 + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + include: + - name: appleclang-release-arm64 + cc: /usr/bin/clang + cxx: /usr/bin/clang++ + build_type: Release + - name: appleclang-debug-arm64 + cc: /usr/bin/clang + cxx: /usr/bin/clang++ + build_type: Debug + - name: gcc15-release-macos-arm64 + cc: /opt/homebrew/bin/gcc-15 + cxx: /opt/homebrew/bin/g++-15 + build_type: Release + gcc_formula: gcc@15 + env: + CC: ${{ matrix.cc }} + CXX: ${{ matrix.cxx }} + PAIMON_BUILD_JOBS: '3' + CARGO_BUILD_JOBS: '3' + steps: + - name: Checkout paimon-cpp + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + lfs: true + - name: Install build dependencies + shell: bash + run: | + brew install cmake ninja protobuf openssl@3 curl + echo "$(brew --prefix protobuf)/bin" >> "$GITHUB_PATH" + echo "CMAKE_PREFIX_PATH=$(brew --prefix openssl@3):$(brew --prefix curl)" >> "$GITHUB_ENV" + - name: Install GCC + if: matrix.gcc_formula + shell: bash + env: + GCC_FORMULA: ${{ matrix.gcc_formula }} + run: brew install "${GCC_FORMULA}" + - name: Show compiler versions + shell: bash + run: | + "${CC}" --version + "${CXX}" --version + - name: Setup ccache + uses: ./.github/actions/setup-ccache + with: + cache-key-prefix: ccache-${{ matrix.name }} + - name: Prepare release assets + uses: ./.github/actions/setup-release-assets + - name: Install Rust and protobuf tools + shell: bash + run: ci/scripts/setup_rust.sh + - name: Install flatc and libclang (Vortex) + shell: bash + run: ci/scripts/setup_flatc.sh + - name: Build, test and verify installation + shell: bash + run: | + ci/scripts/build_paimon.sh \ + --source_dir "$(pwd)" \ + --build_type "${{ matrix.build_type }}" \ + --install_smoke + - name: Upload test logs + if: failure() + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.name }}-test-logs + path: | + build/build/test-logs/ + build/Testing/Temporary/ + build/CMakeFiles/CMakeConfigureLog.yaml + - name: Show ccache statistics + if: always() + run: ccache -s + build-and-test: name: ${{ matrix.name }} runs-on: ${{ matrix.runner || 'ubuntu-24.04' }} @@ -88,6 +171,12 @@ jobs: cc: gcc-14 cxx: g++-14 build_args: --build_type Release + - name: gcc13-release-aarch64 + runner: ubuntu-24.04-arm + cc: gcc-13 + cxx: g++-13 + gcc_version: '13.3.0' + build_args: --build_type Release - name: clang-debug-aarch64 runner: ubuntu-24.04-arm - name: asan-ubsan-aarch64 @@ -109,6 +198,8 @@ jobs: uses: ./.github/actions/setup-ccache with: cache-key-prefix: ccache-${{ matrix.name }} + - name: Prepare release assets + uses: ./.github/actions/setup-release-assets - name: Install Rust and protobuf tools shell: bash run: ci/scripts/setup_rust.sh @@ -116,6 +207,19 @@ jobs: run: | sudo apt-get update sudo apt-get install -y gdb libcurl4-openssl-dev libssl-dev + - name: Install and verify regression compiler + if: matrix.gcc_version + shell: bash + env: + CC: ${{ matrix.cc }} + CXX: ${{ matrix.cxx }} + EXPECTED_GCC_VERSION: ${{ matrix.gcc_version }} + run: | + sudo apt-get install -y "${CC}" "${CXX}" + "${CC}" --version + "${CXX}" --version + test "$("${CC}" -dumpfullversion)" = "${EXPECTED_GCC_VERSION}" + test "$("${CXX}" -dumpfullversion)" = "${EXPECTED_GCC_VERSION}" - name: Install flatc and libclang (Vortex) shell: bash run: ci/scripts/setup_flatc.sh diff --git a/.github/workflows/gcc8_test.yaml b/.github/workflows/gcc8_test.yaml index bdd6a039e..95ff853e0 100644 --- a/.github/workflows/gcc8_test.yaml +++ b/.github/workflows/gcc8_test.yaml @@ -60,6 +60,8 @@ jobs: ls -la - name: Checkout paimon-cpp uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Prepare release assets + uses: ./.github/actions/setup-release-assets - name: Install Rust and protobuf tools shell: bash run: ci/scripts/setup_rust.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index df1c51ea0..35a66f076 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -345,11 +345,46 @@ endif() if(PAIMON_ENABLE_LANCE) find_program(PAIMON_LANCE_PROTOC protoc REQUIRED) - corrosion_import_crate(MANIFEST_PATH crates/lance_ffi/Cargo.toml CRATES - paimon_lance_ffi) + set(PAIMON_LANCE_CORROSION_OPTIONS) + if(APPLE) + # Rust's host tools and Apple's native crypto dependencies require Apple's toolchain. + list(APPEND PAIMON_LANCE_CORROSION_OPTIONS NO_LINKER_OVERRIDE) + endif() + corrosion_import_crate(MANIFEST_PATH + crates/lance_ffi/Cargo.toml + CRATES + paimon_lance_ffi + ${PAIMON_LANCE_CORROSION_OPTIONS}) corrosion_experimental_cbindgen(TARGET paimon_lance_ffi HEADER_NAME paimon_lance_ffi.h) corrosion_set_env_vars(paimon_lance_ffi "PROTOC=${PAIMON_LANCE_PROTOC}") + if(APPLE) + target_link_libraries(paimon_lance_ffi INTERFACE "-framework Security" + "-framework CoreFoundation") + find_program(PAIMON_LANCE_CLANG clang + PATHS /usr/bin + NO_DEFAULT_PATH REQUIRED) + find_program(PAIMON_LANCE_CLANGXX clang++ + PATHS /usr/bin + NO_DEFAULT_PATH REQUIRED) + # cc-rs gives the hyphenated target variables priority over Corrosion's defaults. + # Preserve Apple's crypto-capable CPU baseline for these native dependencies. + corrosion_set_env_vars(paimon_lance_ffi + "CC_${Rust_CARGO_TARGET_CACHED}=${PAIMON_LANCE_CLANG}" + "CXX_${Rust_CARGO_TARGET_CACHED}=${PAIMON_LANCE_CLANGXX}" + "CFLAGS=" + "CXXFLAGS=" + "CFLAGS_${Rust_CARGO_TARGET_CACHED}=" + "CXXFLAGS_${Rust_CARGO_TARGET_CACHED}=") + if(CMAKE_OSX_SYSROOT) + corrosion_set_env_vars(paimon_lance_ffi "SDKROOT=${CMAKE_OSX_SYSROOT}") + endif() + if(CMAKE_OSX_DEPLOYMENT_TARGET) + corrosion_set_env_vars(paimon_lance_ffi + "MACOSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET}" + ) + endif() + endif() endif() if(PAIMON_ENABLE_LUCENE) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2355f95af..2eb4a0b87 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -83,6 +83,12 @@ Before submitting, please verify: - **CMake** ≥ 3.22 - **Python 3** (for linting scripts and pre-commit) +See the [build guide](docs/source/building.rst) for platform requirements and +the CI compiler matrix, feature flags, and platform-specific exclusions. +The CI entry point is `ci/scripts/build_paimon.sh`; it enables the available +optional modules and runs CTest. It defaults to Debug; use +`--build_type Release` to reproduce a Release configuration. + --- ## Code Style diff --git a/README.md b/README.md index 276bcb6e0..040a2ce53 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,8 @@ Paimon C++ currently provides: Building requires a C++17-compatible compiler and CMake 3.22 or newer. Linux `x86_64` and `aarch64` builds are supported and verified in CI. +CI also includes macOS 15 ARM64 builds with AppleClang in Debug and Release, +and Homebrew GCC 15 in Release. See the [build guide](docs/source/building.rst) for platform support, system requirements, dependency configuration, optional components, and instructions for building and running tests. diff --git a/build_support/asan_symbolize.py b/build_support/asan_symbolize.py index a119d9882..3b3323ad0 100755 --- a/build_support/asan_symbolize.py +++ b/build_support/asan_symbolize.py @@ -158,7 +158,8 @@ def open_atos(self): self.pipe = subprocess.Popen(cmdline, stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stderr=subprocess.PIPE, + encoding='utf-8', errors='surrogateescape') def symbolize(self, addr, binary, offset): """Overrides Symbolizer.symbolize.""" diff --git a/ci/scripts/build_paimon.sh b/ci/scripts/build_paimon.sh index eacaf7485..966e84521 100755 --- a/ci/scripts/build_paimon.sh +++ b/ci/scripts/build_paimon.sh @@ -145,9 +145,10 @@ if [[ "${enable_tsan}" == "true" ]]; then fi # CI always builds natively, so the host architecture is the target architecture. host_arch=$(uname -m) -if [[ "${host_arch}" != "x86_64" ]]; then +if [[ "$(uname -s)" != "Linux" || + ( "${host_arch}" != "x86_64" && "${host_arch}" != "aarch64" && "${host_arch}" != "arm64" ) ]]; then ENABLE_LUMINA="OFF" - echo "=== Lumina disabled: no prebuilt artifacts for ${host_arch} ===" + echo "=== Lumina disabled: no prebuilt artifacts for $(uname -s)-${host_arch} ===" fi if [[ "${verify_full_text_archives}" == "true" && "${ENABLE_FULL_TEXT}" != "ON" ]]; then echo "--verify_full_text_archives requires the full-text index, which this build disables" >&2 @@ -158,6 +159,8 @@ CMAKE_ARGS=( "-G Ninja" "-DCMAKE_BUILD_TYPE=${build_type}" "-DPAIMON_BUILD_TESTS=ON" + "-DPAIMON_ENABLE_AVRO=ON" + "-DPAIMON_ENABLE_ORC=ON" "-DPAIMON_ENABLE_MOSAIC=ON" "-DPAIMON_ENABLE_LANCE=${ENABLE_LANCE}" "-DPAIMON_ENABLE_VORTEX=ON" diff --git a/ci/scripts/download_release_asset.sh b/ci/scripts/download_release_asset.sh new file mode 100644 index 000000000..f8fb62a98 --- /dev/null +++ b/ci/scripts/download_release_asset.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Fetch a pinned GitHub release asset. Use the release API if the browser +# download endpoint times out, and publish a file only after verification. +set -euo pipefail + +url=$1 +output=$2 +sha256=$3 + +verify() { + if command -v sha256sum >/dev/null; then + echo "${sha256} $1" | sha256sum -c - + else + echo "${sha256} $1" | shasum -a 256 -c - + fi +} + +if [[ -f "${output}" ]] && verify "${output}"; then + exit 0 +fi + +mkdir -p "$(dirname "${output}")" +tmp=$(mktemp "${output}.XXXXXX") +trap 'rm -f "${tmp}"' EXIT +curl_args=(--proto '=https' --tlsv1.2 -fsSL --retry 3 --connect-timeout 15 --max-time 180) +if ! curl "${curl_args[@]}" "${url}" -o "${tmp}" || ! verify "${tmp}"; then + release=${url#https://github.com/} + repo=${release%%/releases/download/*} + asset=${release##*/} + tag=${release#*/releases/download/} + tag=${tag%/*} + asset_url=$(curl "${curl_args[@]}" "https://api.github.com/repos/${repo}/releases/tags/${tag}" | + python3 -c 'import json, sys +print(next(a["url"] for a in json.load(sys.stdin)["assets"] if a["name"] == sys.argv[1]))' "${asset}") + curl "${curl_args[@]}" -H 'Accept: application/octet-stream' "${asset_url}" -o "${tmp}" + verify "${tmp}" +fi +mv -f "${tmp}" "${output}" diff --git a/ci/scripts/setup_flatc.sh b/ci/scripts/setup_flatc.sh index ce8981f2f..dc93fd718 100755 --- a/ci/scripts/setup_flatc.sh +++ b/ci/scripts/setup_flatc.sh @@ -38,7 +38,23 @@ FLATC_INSTALL_DIR=${FLATC_INSTALL_DIR:-"${HOME}/.local/bin"} # Vortex's native dependencies invoke bindgen in their build scripts, which loads libclang at # build time. Images that only carry GCC (e.g. the gcc8 test container) may lack it. -if ! ldconfig -p 2>/dev/null | grep -q 'libclang'; then +if [[ "$(uname -s)" == "Darwin" ]]; then + # Both Xcode and Command Line Tools ship libclang alongside the selected clang. + clang_bin="$(xcrun --find clang)" + libclang_dir="$(dirname "${clang_bin}")/../lib" + if [[ ! -r "${libclang_dir}/libclang.dylib" ]]; then + libclang_dir="$(brew --prefix llvm)/lib" + if [[ ! -r "${libclang_dir}/libclang.dylib" ]]; then + brew install llvm + fi + fi + if [[ ! -r "${libclang_dir}/libclang.dylib" ]]; then + echo "Cannot find a readable libclang.dylib in ${libclang_dir}" >&2 + exit 1 + fi + export LIBCLANG_PATH="${libclang_dir}" + echo "LIBCLANG_PATH=${LIBCLANG_PATH}" >>"${GITHUB_ENV:-/dev/null}" +elif ! ldconfig -p 2>/dev/null | grep -q 'libclang'; then if [[ "$(id -u)" -ne 0 ]] && command -v sudo >/dev/null 2>&1; then apt_prefix="sudo" else @@ -63,7 +79,12 @@ git clone --depth 1 --branch "v${FLATBUFFERS_VERSION}" \ cmake -S "${workdir}/flatbuffers" -B "${workdir}/build" \ -DCMAKE_BUILD_TYPE=Release \ -DFLATBUFFERS_BUILD_TESTS=OFF -cmake --build "${workdir}/build" --target flatc -j "$(nproc)" +if command -v nproc >/dev/null 2>&1; then + build_jobs=$(nproc) +else + build_jobs=$(sysctl -n hw.ncpu) +fi +cmake --build "${workdir}/build" --target flatc -j "${build_jobs}" mkdir -p "${FLATC_INSTALL_DIR}" install -m 0755 "${workdir}/build/flatc" "${FLATC_INSTALL_DIR}/flatc" diff --git a/ci/scripts/setup_rust.sh b/ci/scripts/setup_rust.sh index a013e7f88..244f345bf 100755 --- a/ci/scripts/setup_rust.sh +++ b/ci/scripts/setup_rust.sh @@ -63,12 +63,11 @@ if ! command -v protoc >/dev/null 2>&1 || \ ;; esac protoc_home="${HOME}/.local/protoc-${PROTOC_VERSION}" - protoc_archive="/tmp/protoc-${PROTOC_VERSION}-${protoc_arch}.zip" + protoc_archive="${PAIMON_RELEASE_ASSET_DIR:-/tmp}/protoc-${PROTOC_VERSION}-${protoc_arch}.zip" if [[ ! -x "${protoc_home}/bin/protoc" ]]; then - curl --proto '=https' --tlsv1.2 -fL \ + bash "$(dirname "${BASH_SOURCE[0]}")/download_release_asset.sh" \ "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-${protoc_arch}.zip" \ - -o "${protoc_archive}" - echo "${protoc_sha256} ${protoc_archive}" | sha256sum --check + "${protoc_archive}" "${protoc_sha256}" mkdir -p "${protoc_home}" unzip -oq "${protoc_archive}" -d "${protoc_home}" fi diff --git a/ci/scripts/test_asan_symbolize.sh b/ci/scripts/test_asan_symbolize.sh index 0859559b4..2a40050cc 100755 --- a/ci/scripts/test_asan_symbolize.sh +++ b/ci/scripts/test_asan_symbolize.sh @@ -92,4 +92,34 @@ if [[ "${output}" != *"#0"* || "${output}" != *"tail"* || "${output}" != *" in"* status=1 fi +echo "=== Darwin symbolizer exchanges text with atos ===" +"${python}" - "${symbolizer}" <<'PY' || status=1 +import runpy +import subprocess +import sys +from unittest import mock + +symbolizer_path = sys.argv[1] +symbols = runpy.run_path(symbolizer_path) +sys.argv = [symbolizer_path] +real_popen = subprocess.Popen + + +def fake_atos(command, **kwargs): + assert command[0] == 'atos', command + return real_popen( + [sys.executable, '-c', + "assert input() == '0x123'; " + "print('foo(int) (in foo.dylib) (source.cc:42)')"], **kwargs) + + +symbolizer = symbols['DarwinSymbolizer']('0x123456789', '/blah/foo.dylib') +with mock.patch.object(subprocess, 'Popen', side_effect=fake_atos): + result = symbolizer.symbolize('0x123456789', '/blah/foo.dylib', '0x123') +assert result == ['0x123456789 in foo source.cc:42'], result +assert symbolizer.pipe.wait() == 0 +symbolizer.pipe.stdout.close() +symbolizer.pipe.stderr.close() +PY + exit "${status}" diff --git a/ci/scripts/test_download_release_asset.sh b/ci/scripts/test_download_release_asset.sh new file mode 100644 index 000000000..3bf636fe1 --- /dev/null +++ b/ci/scripts/test_download_release_asset.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -euo pipefail + +script="$(dirname "${BASH_SOURCE[0]}")/download_release_asset.sh" +work=$(mktemp -d) +trap 'rm -rf "${work}"' EXIT +export PATH="${work}:${PATH}" +export REQUEST_LOG="${work}/requests" +cat > "${work}/curl" <<'MOCK' +#!/usr/bin/env bash +set -eu +echo request >> "${REQUEST_LOG}" +case "$*" in + *github.com/org/repo/releases/download/*) exit 22 ;; + *api.github.com/repos/org/repo/releases/tags/v1*) + echo '{"assets":[{"name":"archive","url":"https://api.github.com/assets/123"}]}' ;; + *application/octet-stream*https://api.github.com/assets/123*) + printf '%s' "${ASSET_CONTENT:-abc}" > "${!#}" ;; + *) exit 1 ;; +esac +MOCK +chmod +x "${work}/curl" +url=https://github.com/org/repo/releases/download/v1/archive +checksum=ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad +fetch() { + bash "${script}" "${url}" "${work}/archive" "${checksum}" +} + +fetch +[[ $(cat "${work}/archive") == abc ]] +fetch +[[ $(wc -l < "${REQUEST_LOG}") -eq 3 ]] +echo corrupt > "${work}/archive" +fetch +[[ $(cat "${work}/archive") == abc ]] +rm "${work}/archive" +if ASSET_CONTENT=invalid fetch; then + echo "Checksum mismatch accepted" >&2 + exit 1 +fi +[[ ! -e "${work}/archive" ]] diff --git a/cmake_modules/BuildAwsAuth.cmake b/cmake_modules/BuildAwsAuth.cmake index 14b1f3ea9..a3275fc0f 100644 --- a/cmake_modules/BuildAwsAuth.cmake +++ b/cmake_modules/BuildAwsAuth.cmake @@ -79,6 +79,16 @@ function(paimon_add_aws_c_project endif() if(APPLE) list(APPEND AWS_PLATFORM_CMAKE_ARGS -DAWS_USE_SECITEM=ON) + if(CMAKE_C_COMPILER_ID STREQUAL "GNU") + # Apple's crypto and TLS backends require Clang's __builtin_available. + # These dependencies expose a C ABI, so Paimon can still use GCC. + find_program(AWS_C_CLANG_EXECUTABLE clang + PATHS /usr/bin + NO_DEFAULT_PATH REQUIRED) + list(APPEND AWS_PLATFORM_CMAKE_ARGS + -DCMAKE_C_COMPILER=${AWS_C_CLANG_EXECUTABLE} + -DCMAKE_OSX_SYSROOT=${CMAKE_OSX_SYSROOT}) + endif() endif() # aws-c-io's installed package unconditionally looks for s2n on Unix even # when it was built with Apple's native TLS backend. diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake index 8aa8f1183..9784170d7 100644 --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake @@ -881,12 +881,24 @@ macro(build_lucene) "-DZLIB_ROOT=${LUCENE_ZLIB_ROOT}" "-DCMAKE_INSTALL_PREFIX=${LUCENE_PREFIX}") + set(LUCENE_BUILD_COMMAND ${CMAKE_COMMAND} --build --config $) + if(CMAKE_VERSION VERSION_GREATER_EQUAL "4.0") + # Cotire invokes CMake in script mode without forwarding the parent's policy cache. + set(LUCENE_BUILD_COMMAND ${CMAKE_COMMAND} -E env CMAKE_POLICY_VERSION_MINIMUM=3.5 + ${LUCENE_BUILD_COMMAND}) + endif() + set(LUCENE_LIB "${LUCENE_PREFIX}/lib/liblucene++.a") + set(LUCENE_PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/lucene.diff") externalproject_add(lucene_ep ${EP_COMMON_OPTIONS} URL ${LUCENE_SOURCE_URL} URL_HASH "SHA256=${PAIMON_LUCENE_BUILD_SHA256_CHECKSUM}" + LOG_PATCH ON + PATCH_COMMAND ${CMAKE_COMMAND} -E chdir bash -c + "test -f .patched || (patch -s -N -p1 -i '${LUCENE_PATCH_FILE}' && touch .patched)" CMAKE_ARGS ${LUCENE_CMAKE_ARGS} + BUILD_COMMAND ${LUCENE_BUILD_COMMAND} BUILD_BYPRODUCTS ${LUCENE_LIB} DEPENDS zlib boost_date_time @@ -1297,6 +1309,25 @@ macro(build_jindosdk_c) set(JINDOSDK_C_LIB_DIR "${JINDOSDK_C_PREFIX}/lib/native") set(JINDOSDK_C_DYNAMIC_LIB "${JINDOSDK_C_LIB_DIR}/${JINDOSDK_C_DYNAMIC_LIB_FILE}") + set(JINDOSDK_C_INSTALL_EXTRA) + set(JINDOSDK_C_INSTALL_PATTERN "${JINDOSDK_C_DYNAMIC_LIB_FILE}*") + set(JINDOSDK_C_BYPRODUCTS "${JINDOSDK_C_DYNAMIC_LIB}") + if(APPLE) + # The SDK's install name uses the major version, but its packaged symlink is broken. + string(REGEX MATCH "^[0-9]+" JINDOSDK_C_MAJOR_VERSION + "${PAIMON_JINDOSDK_C_BUILD_VERSION}") + set(JINDOSDK_C_SONAME "libjindosdk_c.${JINDOSDK_C_MAJOR_VERSION}.dylib") + set(JINDOSDK_C_INSTALL_EXTRA + COMMAND + ${CMAKE_COMMAND} + -E + create_symlink + "${JINDOSDK_C_DYNAMIC_LIB_FILE}" + "${JINDOSDK_C_LIB_DIR}/${JINDOSDK_C_SONAME}") + set(JINDOSDK_C_INSTALL_PATTERN "libjindosdk_c*.dylib") + list(APPEND JINDOSDK_C_BYPRODUCTS "${JINDOSDK_C_LIB_DIR}/${JINDOSDK_C_SONAME}") + endif() + # Extract and install jindosdk from precompiled package externalproject_add(jindosdk_ep URL ${JINDOSDK_C_SOURCE_URL} @@ -1308,7 +1339,8 @@ macro(build_jindosdk_c) "cp -r /include/* ${JINDOSDK_C_INCLUDE_DIR}" COMMAND bash -c "cp -r /lib/native/${JINDOSDK_C_DYNAMIC_LIB_FILE}* ${JINDOSDK_C_LIB_DIR}" - BUILD_BYPRODUCTS "${JINDOSDK_C_DYNAMIC_LIB}") + ${JINDOSDK_C_INSTALL_EXTRA} + BUILD_BYPRODUCTS ${JINDOSDK_C_BYPRODUCTS}) # The include directory must exist before it is referenced by a target. file(MAKE_DIRECTORY "${JINDOSDK_C_INCLUDE_DIR}") @@ -1325,14 +1357,14 @@ macro(build_jindosdk_c) install(DIRECTORY "${JINDOSDK_C_LIB_DIR}/" DESTINATION ${CMAKE_INSTALL_LIBDIR} FILES_MATCHING - PATTERN "${JINDOSDK_C_DYNAMIC_LIB_FILE}*") + PATTERN "${JINDOSDK_C_INSTALL_PATTERN}") endmacro() macro(build_lumina) # The release tarball ships one prebuilt artifact tree per platform; keep this # list in sync with the platforms the pinned release actually contains. - set(LUMINA_SUPPORTED_PLATFORMS "linux-x86_64") + set(LUMINA_SUPPORTED_PLATFORMS "linux-x86_64" "linux-aarch64") if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND PAIMON_TARGET_CPU_FAMILY STREQUAL "x86") set(LUMINA_ARTIFACT_PLATFORM "linux-x86_64") elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND PAIMON_TARGET_CPU_FAMILY STREQUAL @@ -1508,6 +1540,18 @@ macro(build_vortex) file(MAKE_DIRECTORY "${VORTEX_INCLUDE_DIR}") file(MAKE_DIRECTORY "${VORTEX_LIB_DIR}") + set(VORTEX_PLATFORM_ENV) + if(APPLE) + list(APPEND VORTEX_PLATFORM_ENV "CFLAGS=${EP_C_FLAGS}" "CXXFLAGS=${EP_CXX_FLAGS}") + if(CMAKE_OSX_SYSROOT) + list(APPEND VORTEX_PLATFORM_ENV "SDKROOT=${CMAKE_OSX_SYSROOT}") + endif() + if(CMAKE_OSX_DEPLOYMENT_TARGET) + list(APPEND VORTEX_PLATFORM_ENV + "MACOSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET}") + endif() + endif() + # A local checkout (PAIMON_VORTEX_SOURCE_DIR) avoids re-downloading the full cargo workspace; # otherwise download the pinned source archive (with SHA256 when versions.txt provides one). if(DEFINED ENV{PAIMON_VORTEX_SOURCE_DIR}) @@ -1536,7 +1580,7 @@ macro(build_vortex) "/vortex-ffi/src/callback_io.rs" COMMAND ${CMAKE_COMMAND} -E env "FLATC=${VORTEX_FLATC}" "RUSTUP_TOOLCHAIN=stable" "CC=${CMAKE_C_COMPILER}" - "CXX=${CMAKE_CXX_COMPILER}" + "CXX=${CMAKE_CXX_COMPILER}" ${VORTEX_PLATFORM_ENV} "CARGO_TARGET_DIR=${VORTEX_CARGO_TARGET_DIR}" "RUSTFLAGS=-Crelocation-model=pic" ${PAIMON_CARGO_EXECUTABLE} rustc --locked --package @@ -1558,16 +1602,26 @@ macro(build_vortex) PROPERTIES IMPORTED_LOCATION "${VORTEX_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${VORTEX_INCLUDE_DIR}") - # System libraries required to link the Rust static archive on Linux (per Vortex's own - # vortex-ffi/cmake/SystemDependencies.cmake). - target_link_libraries(paimon_vortex_ffi - INTERFACE gcc_s - util - rt - Threads::Threads - m - ${CMAKE_DL_LIBS} - c) + if(APPLE) + target_link_libraries(paimon_vortex_ffi + INTERFACE iconv + "-framework Security" + "-framework CoreFoundation" + "-framework SystemConfiguration" + Threads::Threads + m + c + ${CMAKE_DL_LIBS}) + else() + target_link_libraries(paimon_vortex_ffi + INTERFACE gcc_s + util + rt + Threads::Threads + m + ${CMAKE_DL_LIBS} + c) + endif() add_dependencies(paimon_vortex_ffi vortex_ep) install(FILES "${VORTEX_STATIC_LIB}" DESTINATION ${CMAKE_INSTALL_LIBDIR}) @@ -2163,6 +2217,11 @@ macro(build_tbb) string(REPLACE "-Wdocumentation" "" TBB_CMAKE_CXX_FLAGS ${TBB_CMAKE_CXX_FLAGS}) string(REPLACE "-Wdocumentation" "" TBB_CMAKE_C_FLAGS ${TBB_CMAKE_C_FLAGS}) + if(APPLE AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + # TBB suppresses deprecated ucontext API warnings only for Clang on macOS. + string(APPEND TBB_CMAKE_CXX_FLAGS " -Wno-error=deprecated-declarations") + endif() + set(TBB_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/tbb_ep-install") if(LOWERCASE_BUILD_TYPE STREQUAL "debug") diff --git a/cmake_modules/lucene.diff b/cmake_modules/lucene.diff new file mode 100644 index 000000000..c7e8afb42 --- /dev/null +++ b/cmake_modules/lucene.diff @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- a/include/lucene++/Collection.h ++++ b/include/lucene++/Collection.h +@@ -169,3 +169,3 @@ + void swap(this_type& other) { +- container.swap(other->container); ++ container.swap(other.container); + } +--- a/include/lucene++/Set.h ++++ b/include/lucene++/Set.h +@@ -110,3 +110,3 @@ + void swap(this_type& other) { +- setContainer.swap(other->setContainer); ++ setContainer.swap(other.setContainer); + } diff --git a/docs/source/building.rst b/docs/source/building.rst index b0fe95e44..c7088c200 100644 --- a/docs/source/building.rst +++ b/docs/source/building.rst @@ -67,9 +67,12 @@ Supported platforms - ``aarch64`` / ``arm64`` - Built and tested in CI. * - macOS - - ``x86_64``, ``arm64`` - - Architecture detection is wired up, but macOS as a whole is not an - officially supported platform and is not covered by CI. + - ``arm64`` + - CI includes macOS 15 builds with AppleClang in Debug and Release, + and Homebrew GCC 15 in Release. Lumina is unavailable on macOS. + * - macOS + - ``x86_64`` + - Architecture detection is available; no CI coverage. * - Windows - any - Not supported. @@ -91,18 +94,58 @@ global index is the same however Paimon was compiled. One optional component is architecture-restricted: ``-DPAIMON_ENABLE_LUMINA=ON`` consumes a prebuilt binary that is only -published for Linux ``x86_64``, so configuring it on any other platform fails -with an explicit error. +published for Linux ``x86_64`` and ``aarch64``, so configuring it on any other +platform fails with an explicit error. -Cross compilation is not wired up end to end: the bundled third-party builds are -handed only the C and C++ compilers, without a toolchain file, sysroot or find -root, so they can still pick up host libraries. Build natively on each target. +Cross compilation is not wired up end to end: not all bundled third-party builds +receive a complete toolchain, sysroot and find-root configuration, so they can +still pick up host libraries. Build natively on each target. ``build_and_package.sh`` therefore names its output after the host platform, for example ``output/paimon-cpp-linux-aarch64.tar.gz``, so artifacts built on several machines can be collected side by side. ``--platform