From a01c9f1d1502c460ba1d47773a058a16dba95a91 Mon Sep 17 00:00:00 2001 From: "jinli.zjw" Date: Mon, 16 Nov 2026 17:50:00 +0800 Subject: [PATCH 1/6] build(deps): validate shared third-party patch state --- ci/scripts/test_cmake_modules.sh | 15 +- cmake_modules/ApplyThirdpartyPatch.cmake | 355 +++++++++++++ cmake_modules/ThirdpartyPatch.cmake | 201 ++++++++ cmake_modules/ThirdpartyToolchain.cmake | 144 ++++-- cmake_modules/tests/thirdparty_patch_test.py | 494 +++++++++++++++++++ docs/source/building.rst | 14 + 6 files changed, 1182 insertions(+), 41 deletions(-) create mode 100644 cmake_modules/ApplyThirdpartyPatch.cmake create mode 100644 cmake_modules/ThirdpartyPatch.cmake create mode 100644 cmake_modules/tests/thirdparty_patch_test.py diff --git a/ci/scripts/test_cmake_modules.sh b/ci/scripts/test_cmake_modules.sh index 31acb6737..cd0af1b06 100755 --- a/ci/scripts/test_cmake_modules.sh +++ b/ci/scripts/test_cmake_modules.sh @@ -15,9 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Runs the CMake module tests under cmake_modules/tests. These are script-mode -# tests: they need neither a toolchain nor a configured build tree, so they stay -# separate from ctest and run in seconds. +# Runs the CMake module tests under cmake_modules/tests. These tests need no +# compiler or configured build tree, so they stay separate from ctest. set -euo pipefail @@ -31,4 +30,14 @@ for test_script in "${source_dir}"/cmake_modules/tests/*_test.cmake; do fi done +for generator in "Unix Makefiles" "Ninja"; do + if [[ "${generator}" == "Ninja" ]] && ! command -v ninja >/dev/null; then + echo "Skipping Ninja patch lifecycle tests: ninja is not installed" + continue + fi + if ! python3 "${source_dir}/cmake_modules/tests/thirdparty_patch_test.py" --generator "${generator}"; then + status=1 + fi +done + exit "${status}" diff --git a/cmake_modules/ApplyThirdpartyPatch.cmake b/cmake_modules/ApplyThirdpartyPatch.cmake new file mode 100644 index 000000000..0f6d37d97 --- /dev/null +++ b/cmake_modules/ApplyThirdpartyPatch.cmake @@ -0,0 +1,355 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +function(paimon_patch_error) + string(CONCAT reason ${ARGV}) + set(cleanup "") + foreach(path IN LISTS PP_CLEAN) + string(APPEND cleanup "\n ${path}") + endforeach() + message(FATAL_ERROR "${PP_NAME}: ${reason}\n" + "Do not remove only .patched or a patch stamp. Remove these " + "build-owned paths, then re-run CMake and build:${cleanup}\n" + "Verified source archives may be retained. No files were deleted automatically." + ) +endfunction() + +function(paimon_patch_fingerprint output) + # The same ordered list is used here and by the application loop below. + set(identity + "paimon-patch-v1\n${PP_NAME}\n${PP_VERSION}\n${PP_ARCHIVE}\n${PP_SOURCE}\n${PP_BINARY}\n${PP_INSTALL}\n${PP_CONTEXT}\n" + ) + file(SHA256 "${CMAKE_CURRENT_FUNCTION_LIST_FILE}" runner_hash) + string(APPEND identity + "runner=${runner_hash}\npatch-options=-p1,--batch,--forward,--fuzz=0\n") + set(index 0) + foreach(input IN LISTS PP_PATCHES PP_INPUTS) + if(NOT EXISTS "${input}") + paimon_patch_error("Required patch/build input is missing: ${input}") + endif() + file(SHA256 "${input}" hash) + get_filename_component(name "${input}" NAME) + string(APPEND identity "${index}:${name}:${hash}\n") + math(EXPR index "${index} + 1") + endforeach() + list(LENGTH PP_PATCHES count) + string(APPEND identity "patch-count=${count}\n") + if(PP_VERSIONS_FILE) + if(NOT EXISTS "${PP_VERSIONS_FILE}") + paimon_patch_error("Version file is missing: ${PP_VERSIONS_FILE}") + endif() + foreach(variable IN LISTS PP_VERSION_VARIABLES) + file(STRINGS "${PP_VERSIONS_FILE}" entries REGEX "^${variable}=") + list(LENGTH entries count) + if(NOT count EQUAL 1) + paimon_patch_error("Expected one ${variable} entry in ${PP_VERSIONS_FILE}" + ) + endif() + string(APPEND identity "${entries}\n") + endforeach() + endif() + string(SHA256 fingerprint "${identity}") + set(${output} + "${fingerprint}" + PARENT_SCOPE) +endfunction() + +function(paimon_patch_write name content) + file(WRITE "${PP_STATE}/${name}.tmp" "${content}") + file(RENAME "${PP_STATE}/${name}.tmp" "${PP_STATE}/${name}") +endfunction() + +function(paimon_patch_check_paths) + file(REAL_PATH "${PP_ROOT}" root_real) + foreach(path IN LISTS PP_CLEAN) + file(REAL_PATH "${path}" path_real) + cmake_path(IS_PREFIX + root_real + "${path_real}" + NORMALIZE + owned) + if(NOT owned OR path_real STREQUAL root_real) + # Do not print removal advice for a path redirected outside the build. + message(FATAL_ERROR "${PP_NAME}: ${path} resolves outside its build-owned location. " + "Restore the directory layout; do not delete the external source." + ) + endif() + endforeach() +endfunction() + +function(paimon_patch_assert_empty directory) + file(GLOB_RECURSE contents + LIST_DIRECTORIES FALSE + "${directory}/*") + foreach(path IN LISTS contents) + cmake_path(IS_PREFIX + PP_STAMP + "${path}" + NORMALIZE + in_stamp) + cmake_path(IS_PREFIX + PP_TMP + "${path}" + NORMALIZE + in_tmp) + if(NOT in_stamp AND NOT in_tmp) + paimon_patch_error("Unrecognized existing source/build/install files in ${directory}" + ) + endif() + endforeach() +endfunction() + +function(paimon_patch_check_postimage) + if(NOT EXISTS "${PP_SOURCE}/.paimon-source-id" OR NOT EXISTS "${PP_STATE}/postimage") + paimon_patch_error("Patched source or its verification record is missing") + endif() + file(READ "${PP_SOURCE}/.paimon-source-id" source_id) + if(NOT source_id STREQUAL PP_EXPECTED) + paimon_patch_error("Source identity does not match its build state") + endif() + file(STRINGS "${PP_STATE}/postimage" entries) + foreach(entry IN LISTS entries) + if(NOT entry MATCHES "^([^ ]+) (.+)$") + paimon_patch_error("Invalid source verification record") + endif() + set(expected "${CMAKE_MATCH_1}") + set(relative "${CMAKE_MATCH_2}") + set(path "${PP_SOURCE}/${relative}") + if(expected STREQUAL "absent") + if(EXISTS "${path}" OR IS_SYMLINK "${path}") + paimon_patch_error("Deleted patch target reappeared: ${path}") + endif() + else() + if(NOT EXISTS "${path}" + OR IS_DIRECTORY "${path}" + OR IS_SYMLINK "${path}") + paimon_patch_error("Patch target is missing or replaced: ${path}") + endif() + file(SHA256 "${path}" actual) + if(NOT actual STREQUAL expected) + paimon_patch_error("Patched source was modified: ${path}") + endif() + endif() + endforeach() +endfunction() + +function(paimon_patch_guard) + paimon_patch_check_paths() + paimon_patch_fingerprint(actual) + if(NOT actual STREQUAL PP_EXPECTED) + paimon_patch_error("Inputs changed since configuration; re-run CMake. Configured " + "identity ${PP_EXPECTED}, current identity ${actual}") + endif() + if(EXISTS "${PP_STATE}/identity") + file(READ "${PP_STATE}/identity" previous) + if(NOT previous STREQUAL PP_EXPECTED) + paimon_patch_error("Source version, patches/order, or build inputs changed. " + "Previous identity ${previous}, requested identity ${PP_EXPECTED}" + ) + endif() + else() + # Generated EP helper scripts and empty include directories are harmless; + # an old source, install, compilation cache or success stamp is not. + paimon_patch_assert_empty("${PP_SOURCE}") + paimon_patch_assert_empty("${PP_BINARY}") + paimon_patch_assert_empty("${PP_INSTALL}") + foreach(step + download + update + update_disconnected + patch + patch_disconnected + configure + build + install + done + paimon-verify) + if(EXISTS "${PP_STAMP}/${PP_NAME}-${step}") + paimon_patch_error("Legacy ExternalProject stamp has no verified identity: ${step}" + ) + endif() + endforeach() + if(EXISTS "${PP_COMPLETE}") + paimon_patch_error("Legacy ExternalProject completion record has no verified identity" + ) + endif() + paimon_patch_write(identity "${PP_EXPECTED}") + endif() + if(EXISTS "${PP_STATE}/applying") + paimon_patch_error("Patch application failed or was interrupted; the source may be partially patched" + ) + endif() + if(EXISTS "${PP_STATE}/patched") + file(READ "${PP_STATE}/patched" patched) + if(NOT patched STREQUAL PP_EXPECTED) + paimon_patch_error("Patch completion record does not match the requested identity" + ) + endif() + paimon_patch_check_postimage() + # Re-extraction of the same archive can preserve timestamps of its files. + # Never reuse the old binary cache after a repeated download/update step. + foreach(step download update update_disconnected) + if(EXISTS "${PP_STAMP}/${PP_NAME}-${step}" + AND "${PP_STAMP}/${PP_NAME}-${step}" IS_NEWER_THAN "${PP_STATE}/patched") + paimon_patch_error("Source download/update ran after patch completion") + endif() + endforeach() + elseif(EXISTS "${PP_STAMP}/${PP_NAME}-patch" + OR EXISTS "${PP_STAMP}/${PP_NAME}-patch_disconnected" + OR EXISTS "${PP_STATE}/installed") + paimon_patch_error("A build stamp or install exists without verified patch completion" + ) + endif() + if(EXISTS "${PP_STATE}/installed") + file(READ "${PP_STATE}/installed" installed) + if(NOT installed STREQUAL PP_EXPECTED) + paimon_patch_error("Installation belongs to a different source/build identity" + ) + endif() + elseif(EXISTS "${PP_COMPLETE}") + paimon_patch_error("ExternalProject completed without a verified installation") + endif() +endfunction() + +function(paimon_patch_apply) + if(EXISTS "${PP_STATE}/patched") + return() + endif() + if(NOT EXISTS "${PP_STAMP}/${PP_NAME}-download") + paimon_patch_error("Cannot patch without a completed, checksum-verified download") + endif() + if(EXISTS "${PP_SOURCE}/.patched" OR EXISTS "${PP_SOURCE}/.paimon-vortex-patches") + paimon_patch_error("Legacy patch marker found in downloaded source") + endif() + # Reject unsupported patch paths before modifying anything. This mechanism + # handles textual -p1 diffs, including additions/deletions, not binary patches, + # renames, mode changes or paths through symlinks. + set(targets "") + foreach(patch IN LISTS PP_PATCHES) + file(STRINGS "${patch}" lines) + set(count 0) + foreach(line IN LISTS lines) + if(line MATCHES + "^(GIT binary patch|rename from |rename to |old mode |new mode )") + paimon_patch_error("Unsupported non-text patch operation in ${patch}") + endif() + if(line MATCHES "^(---|\\+\\+\\+) (.*)$") + set(path "${CMAKE_MATCH_2}") + if(path STREQUAL "/dev/null") + continue() + endif() + if(NOT path MATCHES "^[ab]/(.+)$") + paimon_patch_error("Unsupported patch path in ${patch}: ${path}") + endif() + set(relative "${CMAKE_MATCH_1}") + if(relative MATCHES "(^|/)\\.\\.(/|$)|[;\\\\\t]" OR IS_ABSOLUTE + "${relative}") + paimon_patch_error("Unsafe patch path in ${patch}: ${relative}") + endif() + file(REAL_PATH "${PP_SOURCE}" source_real) + file(REAL_PATH "${PP_SOURCE}/${relative}" target_real) + cmake_path(IS_PREFIX + source_real + "${target_real}" + NORMALIZE + contained) + if(NOT contained OR IS_SYMLINK "${PP_SOURCE}/${relative}") + paimon_patch_error("Patch target escapes its source tree: ${relative}" + ) + endif() + list(APPEND targets "${relative}") + math(EXPR count "${count} + 1") + endif() + endforeach() + if(count EQUAL 0) + paimon_patch_error("No text patch targets found in ${patch}") + endif() + endforeach() + paimon_patch_write(applying "${PP_EXPECTED}") + foreach(patch IN LISTS PP_PATCHES) + # Check each patch against the result of its predecessors. A failure + # leaves 'applying' in place; never guess which hunks succeeded. + foreach(check --dry-run apply) + set(options + --batch + --forward + --fuzz=0 + -p1 + -i + "${patch}") + if(check STREQUAL "--dry-run") + list(APPEND options --dry-run) + endif() + execute_process(COMMAND "${PP_PATCH_EXECUTABLE}" ${options} + WORKING_DIRECTORY "${PP_SOURCE}" + RESULT_VARIABLE result + OUTPUT_VARIABLE log + ERROR_VARIABLE error) + if(NOT result EQUAL 0) + paimon_patch_error("Cannot apply ${patch} (${check}): ${log}${error}") + endif() + endforeach() + endforeach() + list(REMOVE_DUPLICATES targets) + set(postimage "") + foreach(relative IN LISTS targets) + if(EXISTS "${PP_SOURCE}/${relative}") + file(SHA256 "${PP_SOURCE}/${relative}" hash) + else() + set(hash absent) + endif() + string(APPEND postimage "${hash} ${relative}\n") + endforeach() + paimon_patch_write(postimage "${postimage}") + file(WRITE "${PP_SOURCE}/.paimon-source-id" "${PP_EXPECTED}") + paimon_patch_write(patched "${PP_EXPECTED}") + # A crash between these operations is conservatively treated as interrupted. + file(REMOVE "${PP_STATE}/applying") +endfunction() + +if(CMAKE_SCRIPT_MODE_FILE STREQUAL CMAKE_CURRENT_LIST_FILE) + cmake_minimum_required(VERSION 3.22) + if(NOT PAIMON_PATCH_CONFIG) + message(FATAL_ERROR "PAIMON_PATCH_CONFIG is required") + endif() + include("${PAIMON_PATCH_CONFIG}") + file(LOCK "${PP_STATE}/lock" + GUARD PROCESS + TIMEOUT 0 + RESULT_VARIABLE locked) + if(NOT locked STREQUAL "0") + message(FATAL_ERROR "${PP_NAME}: another process is validating or patching this source. Retry after it finishes." + ) + endif() + paimon_patch_guard() + if(PAIMON_PATCH_MODE STREQUAL "apply") + paimon_patch_apply() + elseif(PAIMON_PATCH_MODE STREQUAL "complete" OR PAIMON_PATCH_MODE STREQUAL "consume") + if(NOT EXISTS "${PP_STATE}/patched") + paimon_patch_error("Cannot use outputs before patch completion") + endif() + if(PAIMON_PATCH_MODE STREQUAL "complete") + # This step runs only after ExternalProject's install command succeeds. + # Record that success for this identity, without sampling output files. + paimon_patch_write(installed "${PP_EXPECTED}") + elseif(NOT EXISTS "${PP_STATE}/installed") + paimon_patch_error("Build and installation have not completed") + endif() + elseif(PAIMON_PATCH_MODE) + message(FATAL_ERROR "Unknown patch mode: ${PAIMON_PATCH_MODE}") + endif() +endif() diff --git a/cmake_modules/ThirdpartyPatch.cmake b/cmake_modules/ThirdpartyPatch.cmake new file mode 100644 index 000000000..cd4b53979 --- /dev/null +++ b/cmake_modules/ThirdpartyPatch.cmake @@ -0,0 +1,201 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +include_guard(GLOBAL) +include(ExternalProject) + +# Call before ExternalProject_Add, then register the project after creating it. +function(paimon_patch_command name output) + set(${output} + "${CMAKE_COMMAND}" + "-DPAIMON_PATCH_MODE=apply" + "-DPAIMON_PATCH_CONFIG=${CMAKE_CURRENT_BINARY_DIR}/paimon-thirdparty/${name}/request.cmake" + -P + "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/ApplyThirdpartyPatch.cmake" + PARENT_SCOPE) +endfunction() + +# Read the previous generated request in a separate scope. Directory migrations +# must report old caches as well as the newly configured paths on failure. +function(paimon_patch_previous_cleanup config output) + set(PP_CLEAN "") + if(EXISTS "${config}") + include("${config}") + endif() + set(${output} + "${PP_CLEAN}" + PARENT_SCOPE) +endfunction() + +# Only SHA256-verified archives in build-owned directories are supported. PATCHES +# is ordered; an empty list still registers source/build identity validation. +function(paimon_register_patched_project name) + cmake_parse_arguments(P + "" + "VERSION;INSTALL_DIR" + "PATCHES;INPUTS;VERSION_VARIABLES" + ${ARGN}) + if(P_UNPARSED_ARGUMENTS + OR NOT P_VERSION + OR NOT P_INSTALL_DIR) + message(FATAL_ERROR "Invalid patch registration for ${name}") + endif() + if(CMAKE_CONFIGURATION_TYPES) + message(FATAL_ERROR "${name}: patch state validation requires a single-configuration generator" + ) + endif() + externalproject_get_property(${name} + SOURCE_DIR + BINARY_DIR + STAMP_DIR + TMP_DIR) + get_property(archive_hash + TARGET ${name} + PROPERTY _EP_URL_HASH) + if(NOT archive_hash MATCHES "^SHA256=[0-9a-fA-F]+$") + message(FATAL_ERROR "${name}: patched dependencies require a SHA256-verified archive" + ) + endif() + string(LENGTH "${archive_hash}" hash_length) + if(NOT hash_length EQUAL 71) + message(FATAL_ERROR "${name}: invalid archive SHA256") + endif() + + # Validate ownership even though this module never deletes directories. + # In particular, do not generate cleanup advice for a user checkout. + set(PP_ROOT "${CMAKE_BINARY_DIR}") + set(PP_NAME "${name}") + set(PP_SOURCE "${SOURCE_DIR}") + set(PP_BINARY "${BINARY_DIR}") + set(PP_STAMP "${STAMP_DIR}") + set(PP_TMP "${TMP_DIR}") + set(PP_INSTALL "${P_INSTALL_DIR}") + set(PP_STATE "${CMAKE_CURRENT_BINARY_DIR}/paimon-thirdparty/${name}") + set(PP_COMPLETE "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${name}-complete") + paimon_patch_previous_cleanup("${PP_STATE}/request.cmake" previous_cleanup) + set(PP_CLEAN + "${SOURCE_DIR};${BINARY_DIR};${STAMP_DIR};${TMP_DIR};${P_INSTALL_DIR};${PP_COMPLETE};${PP_STATE};${previous_cleanup}" + ) + list(FILTER + PP_CLEAN + EXCLUDE + REGEX + "^$") + list(REMOVE_DUPLICATES PP_CLEAN) + file(REAL_PATH "${PP_ROOT}" root_real) + foreach(path IN LISTS PP_CLEAN) + file(REAL_PATH "${path}" path_real) + cmake_path(IS_PREFIX + root_real + "${path_real}" + NORMALIZE + owned) + if(NOT owned OR path_real STREQUAL root_real) + message(FATAL_ERROR "${name}: ${path} is not a build-owned directory. " + "Local source overrides are not supported by the patch mechanism." + ) + endif() + endforeach() + + set(PP_VERSION "${P_VERSION}") + string(TOLOWER "${archive_hash}" PP_ARCHIVE) + set(PP_PATCHES "${P_PATCHES}") + set(PP_INPUTS "${P_INPUTS}") + # Resolve relative to this module, including when Paimon is a subproject. + get_filename_component(PP_VERSIONS_FILE + "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/../third_party/versions.txt" + ABSOLUTE) + set(PP_VERSION_VARIABLES "${P_VERSION_VARIABLES}") + set(PP_CONTEXT + "${CMAKE_GENERATOR}|${CMAKE_BUILD_TYPE}|${CMAKE_C_COMPILER}|${CMAKE_CXX_COMPILER}" + ) + foreach(property CMAKE_ARGS CONFIGURE_COMMAND BUILD_COMMAND INSTALL_COMMAND) + get_property(value + TARGET ${name} + PROPERTY _EP_${property}) + string(APPEND PP_CONTEXT "\n${property}=${value}") + endforeach() + set(PP_RUNNER "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/ApplyThirdpartyPatch.cmake") + find_program(PP_PATCH_EXECUTABLE patch REQUIRED) + + set_property(DIRECTORY + APPEND + PROPERTY CMAKE_CONFIGURE_DEPENDS + ${PP_PATCHES} + ${PP_INPUTS} + ${PP_VERSIONS_FILE} + "${PP_RUNNER}") + # Including the runner only defines helpers; script mode executes them. + include("${PP_RUNNER}") + paimon_patch_fingerprint(PP_EXPECTED) + file(MAKE_DIRECTORY "${PP_STATE}") + set(request "") + foreach(variable + ROOT + NAME + SOURCE + BINARY + STAMP + TMP + INSTALL + STATE + COMPLETE + CLEAN + VERSION + ARCHIVE + PATCHES + INPUTS + VERSIONS_FILE + VERSION_VARIABLES + CONTEXT + PATCH_EXECUTABLE + EXPECTED) + # Bracket arguments preserve spaces, semicolons, dollar signs and backslashes. + set(delimiter "=") + while("${PP_${variable}}" MATCHES "]${delimiter}]") + string(APPEND delimiter "=") + endwhile() + string(APPEND request + "set(PP_${variable} [${delimiter}[${PP_${variable}}]${delimiter}])\n") + endforeach() + # Do not update timestamps on ordinary reconfiguration. + set(config "${PP_STATE}/request.cmake") + file(WRITE "${config}.tmp" "${request}") + configure_file("${config}.tmp" "${config}" COPYONLY) + file(REMOVE "${config}.tmp") + set(command "${CMAKE_COMMAND}" "-DPAIMON_PATCH_CONFIG=${config}" -P "${PP_RUNNER}") + execute_process(COMMAND ${command} RESULT_VARIABLE result) + if(NOT result EQUAL 0) + message(FATAL_ERROR "${name}: third-party patch state validation failed") + endif() + + add_custom_target(${name}-patch-guard + COMMAND ${command} + VERBATIM) + # A target dependency orders validation without making the download stamp + # perpetually out of date, unlike an ALWAYS step in the EP step chain. + externalproject_add_stepdependencies(${name} download ${name}-patch-guard) + externalproject_add_step(${name} paimon-verify + COMMAND "${CMAKE_COMMAND}" "-DPAIMON_PATCH_MODE=complete" + "-DPAIMON_PATCH_CONFIG=${config}" -P "${PP_RUNNER}" + DEPENDEES install) # codespell:ignore dependees + # cmake --install does not build first. Refuse to export stale artifacts. + install(CODE "execute_process(COMMAND \"${CMAKE_COMMAND}\" \"-DPAIMON_PATCH_MODE=consume\" \"-DPAIMON_PATCH_CONFIG=${config}\" -P \"${PP_RUNNER}\" RESULT_VARIABLE patch_result) +if(NOT patch_result EQUAL 0) + message(FATAL_ERROR \"${name}: build the dependency successfully before installing Paimon\") +endif()") +endfunction() diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake index dffebe650..4536e8d65 100644 --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. +include(ThirdpartyPatch) + set(THIRDPARTY_LOG_OPTIONS LOG_CONFIGURE 1 @@ -43,6 +45,9 @@ endmacro() set(THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/third_party") # Read toolchain versions from third_party/versions.txt +set_property(DIRECTORY + APPEND + PROPERTY CMAKE_CONFIGURE_DEPENDS "${THIRDPARTY_DIR}/versions.txt") file(STRINGS "${THIRDPARTY_DIR}/versions.txt" TOOLCHAIN_VERSIONS_TXT) foreach(_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT}) # Exclude comments @@ -946,18 +951,31 @@ macro(build_jieba) "-DCMAKE_INSTALL_PREFIX=${JIEBA_INSTALL}") set(PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/jieba.diff") + paimon_patch_command(jieba_ep PATCH_COMMAND) externalproject_add(jieba_ep ${EP_COMMON_OPTIONS} URL ${JIEBA_SOURCE_URL} URL_HASH "SHA256=${PAIMON_JIEBA_BUILD_SHA256_CHECKSUM}" CMAKE_ARGS ${JIEBA_CMAKE_ARGS} LOG_PATCH ON - PATCH_COMMAND ${CMAKE_COMMAND} -E chdir bash -c - "[ -f .patched ] && echo ' patch already applied, ignore...' || patch -s -N -p1 -i '${PATCH_FILE}' && touch .patched" + PATCH_COMMAND ${PATCH_COMMAND} INSTALL_COMMAND bash -c "cp -r ${JIEBA_PREFIX}/src/jieba_ep/include/* ${JIEBA_INSTALL}/include/ && cp -r ${JIEBA_PREFIX}/src/jieba_ep/dict/* ${JIEBA_INSTALL}/dict/ && cp -r ${LIMONP_PREFIX}/src/limonp_ep/include/* ${JIEBA_INSTALL}/include/" ) + paimon_register_patched_project(jieba_ep + VERSION + "${PAIMON_JIEBA_BUILD_VERSION}" + PATCHES + "${PATCH_FILE}" + INSTALL_DIR + "${JIEBA_INSTALL}" + VERSION_VARIABLES + PAIMON_JIEBA_BUILD_VERSION + PAIMON_JIEBA_BUILD_SHA256_CHECKSUM + PAIMON_LIMONP_BUILD_VERSION + PAIMON_LIMONP_BUILD_SHA256_CHECKSUM) + # The include directory must exist before it is referenced by a target. include_directories(SYSTEM ${JIEBA_INCLUDE_DIR} ${JIEBA_DICT_DIR}) add_library(jieba INTERFACE IMPORTED) @@ -1495,7 +1513,8 @@ macro(build_vortex) set(VORTEX_STATIC_LIB "${VORTEX_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}vortex_ffi${CMAKE_STATIC_LIBRARY_SUFFIX}" ) - set(VORTEX_CARGO_TARGET_DIR "${CMAKE_CURRENT_BINARY_DIR}/vortex_ep-cargo") + set(VORTEX_CARGO_TARGET_DIR + "${CMAKE_CURRENT_BINARY_DIR}/vortex_ep-prefix/src/vortex_ep-build") set(VORTEX_CARGO_STATIC_LIB "${VORTEX_CARGO_TARGET_DIR}/release/${CMAKE_STATIC_LIBRARY_PREFIX}vortex_ffi${CMAKE_STATIC_LIBRARY_SUFFIX}" ) @@ -1504,42 +1523,29 @@ macro(build_vortex) # backed by host callbacks; it is maintained in this repository and injected into the Vortex # source tree. The patch declares the module. set(VORTEX_PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/vortex.diff") + paimon_patch_command(vortex_ep PATCH_COMMAND) set(VORTEX_CALLBACK_IO_SRC "${CMAKE_SOURCE_DIR}/crates/vortex_callback_io/callback_io.rs") file(MAKE_DIRECTORY "${VORTEX_INCLUDE_DIR}") file(MAKE_DIRECTORY "${VORTEX_LIB_DIR}") - # A local checkout (PAIMON_VORTEX_SOURCE_DIR) avoids re-downloading the full cargo workspace; - # otherwise download the pinned source archive (with SHA256 when versions.txt provides one). - if(DEFINED ENV{PAIMON_VORTEX_SOURCE_DIR}) - set(VORTEX_EP_DOWNLOAD SOURCE_DIR "$ENV{PAIMON_VORTEX_SOURCE_DIR}" - DOWNLOAD_COMMAND "") - elseif(PAIMON_VORTEX_BUILD_SHA256_CHECKSUM) - set(VORTEX_EP_DOWNLOAD URL ${VORTEX_SOURCE_URL} URL_HASH - "SHA256=${PAIMON_VORTEX_BUILD_SHA256_CHECKSUM}") - else() - set(VORTEX_EP_DOWNLOAD URL ${VORTEX_SOURCE_URL}) - endif() - externalproject_add(vortex_ep - ${VORTEX_EP_DOWNLOAD} ${THIRDPARTY_LOG_OPTIONS} - BUILD_IN_SOURCE TRUE + URL ${VORTEX_SOURCE_URL} + URL_HASH "SHA256=${PAIMON_VORTEX_BUILD_SHA256_CHECKSUM}" + ${THIRDPARTY_LOG_OPTIONS} + BINARY_DIR "${VORTEX_CARGO_TARGET_DIR}" CONFIGURE_COMMAND "" LOG_PATCH ON - PATCH_COMMAND ${CMAKE_COMMAND} -E chdir bash -c - "[ -f .patched ] && echo ' patch already applied, ignore...' || patch -s -N -p1 -i '${VORTEX_PATCH_FILE}' && touch .patched" - # Rebuild every time so edits to callback_io.rs are picked up; cargo - # returns immediately when nothing changed, and copy_if_different keeps - # the archive's timestamp stable so dependents do not relink. - BUILD_ALWAYS TRUE + PATCH_COMMAND ${PATCH_COMMAND} + # The patch guard tracks callback_io.rs as a build input. BUILD_COMMAND ${CMAKE_COMMAND} -E copy_if_different "${VORTEX_CALLBACK_IO_SRC}" "/vortex-ffi/src/callback_io.rs" - COMMAND ${CMAKE_COMMAND} -E env "FLATC=${VORTEX_FLATC}" - "RUSTUP_TOOLCHAIN=stable" "CC=${CMAKE_C_COMPILER}" - "CXX=${CMAKE_CXX_COMPILER}" - "CARGO_TARGET_DIR=${VORTEX_CARGO_TARGET_DIR}" + COMMAND ${CMAKE_COMMAND} -E chdir ${CMAKE_COMMAND} -E + env "FLATC=${VORTEX_FLATC}" "RUSTUP_TOOLCHAIN=stable" + "CC=${CMAKE_C_COMPILER}" "CXX=${CMAKE_CXX_COMPILER}" + "CARGO_TARGET_DIR=" "RUSTFLAGS=-Crelocation-model=pic" ${PAIMON_CARGO_EXECUTABLE} rustc --locked --package vortex-ffi --lib --crate-type=staticlib --release @@ -1551,6 +1557,19 @@ macro(build_vortex) INSTALL_COMMAND "" BUILD_BYPRODUCTS "${VORTEX_STATIC_LIB}") + paimon_register_patched_project(vortex_ep + VERSION + "${PAIMON_VORTEX_BUILD_VERSION}" + PATCHES + "${VORTEX_PATCH_FILE}" + INPUTS + "${VORTEX_CALLBACK_IO_SRC}" + INSTALL_DIR + "${VORTEX_PREFIX}" + VERSION_VARIABLES + PAIMON_VORTEX_BUILD_VERSION + PAIMON_VORTEX_BUILD_SHA256_CHECKSUM) + if(NOT TARGET Threads::Threads) set(THREADS_PREFER_PTHREAD_FLAG TRUE) find_package(Threads REQUIRED) @@ -1686,16 +1705,27 @@ macro(build_oss_sdk_v2) # Avoid copying the metadata variant, which triggers GCC 12's # -Wmaybe-uninitialized warning in optimized builds. set(OSS_SDK_V2_PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/oss_sdk_v2.diff") + paimon_patch_command(oss_sdk_v2_ep PATCH_COMMAND) externalproject_add(oss_sdk_v2_ep ${EP_COMMON_OPTIONS} URL ${OSS_SDK_V2_SOURCE_URL} URL_HASH "SHA256=${PAIMON_OSS_SDK_V2_BUILD_SHA256_CHECKSUM}" LOG_PATCH ON - PATCH_COMMAND ${CMAKE_COMMAND} -E chdir bash -c - "[ -f .patched ] && echo ' patch already applied, ignore...' || patch -s -N -p1 -i '${OSS_SDK_V2_PATCH_FILE}' && touch .patched" + PATCH_COMMAND ${PATCH_COMMAND} CMAKE_ARGS ${OSS_SDK_V2_CMAKE_ARGS} ${THIRDPARTY_LOG_OPTIONS} BUILD_BYPRODUCTS "${OSS_SDK_V2_STATIC_LIB}") + paimon_register_patched_project(oss_sdk_v2_ep + VERSION + "${PAIMON_OSS_SDK_V2_BUILD_VERSION}" + PATCHES + "${OSS_SDK_V2_PATCH_FILE}" + INSTALL_DIR + "${OSS_SDK_V2_PREFIX}" + VERSION_VARIABLES + PAIMON_OSS_SDK_V2_BUILD_VERSION + PAIMON_OSS_SDK_V2_BUILD_SHA256_CHECKSUM) + file(MAKE_DIRECTORY "${OSS_SDK_V2_INCLUDE_DIR}") file(MAKE_DIRECTORY "${OSS_SDK_V2_LIB_DIR}") @@ -1873,10 +1903,8 @@ macro(build_orc) "-DCMAKE_MODULE_LINKER_FLAGS=-Wl,-rpath=${ORC_RPATH}") endif() - set(ORC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/orc_ep-prefix") + set(ORC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/orc_ep-install") set(ORC_INCLUDE_DIR "${ORC_PREFIX}/include") - set(ORC_SOURCE_DIR "${ORC_PREFIX}/cpp") - set(ORC_BUILD_DIR "${CMAKE_BINARY_DIR}/build/orc") set(ORC_STATIC_LIB "${ORC_PREFIX}/lib/liborc.a") @@ -1896,15 +1924,13 @@ macro(build_orc) -DBUILD_CPP_ENABLE_METRICS=ON) set(PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/orc.diff") + paimon_patch_command(orc_ep PATCH_COMMAND) externalproject_add(orc_ep URL ${ORC_SOURCE_URL} URL_HASH "SHA256=${PAIMON_ORC_BUILD_SHA256_CHECKSUM}" - SOURCE_DIR ${ORC_SOURCE_DIR} - BINARY_DIR ${ORC_BUILD_DIR} CMAKE_ARGS ${ORC_CMAKE_ARGS} LOG_PATCH ON - PATCH_COMMAND ${CMAKE_COMMAND} -E chdir bash -c - "[ -f .patched ] && echo ' patch already applied, ignore...' || patch -s -N -p1 -i '${PATCH_FILE}' && touch .patched" + PATCH_COMMAND ${PATCH_COMMAND} UPDATE_DISCONNECTED 1 BUILD_BYPRODUCTS ${ORC_STATIC_LIB} DEPENDS zstd @@ -1913,6 +1939,27 @@ macro(build_orc) zlib libprotobuf) + paimon_register_patched_project(orc_ep + VERSION + "${PAIMON_ORC_BUILD_VERSION}" + PATCHES + "${PATCH_FILE}" + INSTALL_DIR + "${ORC_PREFIX}" + VERSION_VARIABLES + PAIMON_ORC_BUILD_VERSION + PAIMON_ORC_BUILD_SHA256_CHECKSUM + PAIMON_PROTOBUF_BUILD_VERSION + PAIMON_PROTOBUF_BUILD_SHA256_CHECKSUM + PAIMON_ZSTD_BUILD_VERSION + PAIMON_ZSTD_BUILD_SHA256_CHECKSUM + PAIMON_SNAPPY_BUILD_VERSION + PAIMON_SNAPPY_BUILD_SHA256_CHECKSUM + PAIMON_LZ4_BUILD_VERSION + PAIMON_LZ4_BUILD_SHA256_CHECKSUM + PAIMON_ZLIB_BUILD_VERSION + PAIMON_ZLIB_BUILD_SHA256_CHECKSUM) + # The include directory must exist before it is referenced by a target. file(MAKE_DIRECTORY "${ORC_INCLUDE_DIR}") @@ -2022,12 +2069,12 @@ macro(build_arrow) set(ARROW_CONFIGURE SOURCE_SUBDIR "cpp" CMAKE_ARGS ${ARROW_CMAKE_ARGS}) set(PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/arrow.diff") + paimon_patch_command(arrow_ep PATCH_COMMAND) externalproject_add(arrow_ep URL ${ARROW_SOURCE_URL} URL_HASH "SHA256=${PAIMON_ARROW_BUILD_SHA256_CHECKSUM}" LOG_PATCH ON - PATCH_COMMAND ${CMAKE_COMMAND} -E chdir bash -c - "[ -f .patched ] && echo ' patch already applied, ignore...' || patch -s -N -p1 -i '${PATCH_FILE}' && touch .patched" + PATCH_COMMAND ${PATCH_COMMAND} GIT_SUBMODULES "" GIT_SUBMODULES_RECURSE FALSE ${ARROW_CONFIGURE} UPDATE_DISCONNECTED 1 BUILD_BYPRODUCTS "${ARROW_STATIC_LIB}" @@ -2041,6 +2088,27 @@ macro(build_arrow) zlib re2::re2) + paimon_register_patched_project(arrow_ep + VERSION + "${PAIMON_ARROW_BUILD_VERSION}" + PATCHES + "${PATCH_FILE}" + INSTALL_DIR + "${ARROW_PREFIX}" + VERSION_VARIABLES + PAIMON_ARROW_BUILD_VERSION + PAIMON_ARROW_BUILD_SHA256_CHECKSUM + PAIMON_ZSTD_BUILD_VERSION + PAIMON_ZSTD_BUILD_SHA256_CHECKSUM + PAIMON_SNAPPY_BUILD_VERSION + PAIMON_SNAPPY_BUILD_SHA256_CHECKSUM + PAIMON_LZ4_BUILD_VERSION + PAIMON_LZ4_BUILD_SHA256_CHECKSUM + PAIMON_ZLIB_BUILD_VERSION + PAIMON_ZLIB_BUILD_SHA256_CHECKSUM + PAIMON_RE2_BUILD_VERSION + PAIMON_RE2_BUILD_SHA256_CHECKSUM) + add_library(arrow STATIC IMPORTED) set_target_properties(arrow PROPERTIES IMPORTED_LOCATION "${ARROW_PREFIX}/lib/libarrow.a" diff --git a/cmake_modules/tests/thirdparty_patch_test.py b/cmake_modules/tests/thirdparty_patch_test.py new file mode 100644 index 000000000..210ebecf1 --- /dev/null +++ b/cmake_modules/tests/thirdparty_patch_test.py @@ -0,0 +1,494 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Offline ExternalProject lifecycle tests. No compiler or downloaded packages needed. + +python3 cmake_modules/tests/thirdparty_patch_test.py --generator Ninja +python3 cmake_modules/tests/thirdparty_patch_test.py --generator 'Unix Makefiles' +Use --cmake to also test the minimum supported CMake version. +""" + +import argparse +import hashlib +import os +from pathlib import Path +import shutil +import subprocess +import tarfile +import tempfile +import unittest + + +MODULES = Path(__file__).resolve().parents[1] +PARSER = argparse.ArgumentParser() +PARSER.add_argument("--generator", default="Ninja") +PARSER.add_argument("--cmake", default="cmake") +ARGS, UNITTEST_ARGS = PARSER.parse_known_args() + + +def patch(before, after, filename="value"): + return ("diff --git a/{0} b/{0}\n--- a/{0}\n+++ b/{0}\n" + "@@ -1 +1 @@\n-{1}\n+{2}\n").format(filename, before, after) + + +class ThirdpartyPatchTest(unittest.TestCase): + def setUp(self): + self.create_project() + + def create_project(self): + self.temp = tempfile.TemporaryDirectory(prefix="paimon patch ") + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.project = self.root / "project" + self.project.mkdir() + self.modules = self.project / "cmake_modules" + self.modules.mkdir() + for name in ["ThirdpartyPatch.cmake", "ApplyThirdpartyPatch.cmake"]: + shutil.copyfile(str(MODULES / name), str(self.modules / name)) + (self.project / "third_party").mkdir() + # Exercise git parent discovery: source archives live under this repo. + self.command(["git", "init", "-q", str(self.project)]) + self.build = self.project / "build" + self.source = self.build / "dep-prefix/src/dep" + self.install = self.build / "dep-install" + self.stamp = self.build / "dep-prefix/src/dep-stamp" + self.package = self.root / "archive-source" + self.package.mkdir() + (self.package / "value").write_text("alpha\n") + (self.package / "other").write_text("unchanged\n") + self.archive = self.project / "dep.tar.gz" + self.make_archive() + self.write_versions() + (self.project / "one.diff").write_text(patch("alpha", "beta")) + (self.project / "two.diff").write_text(patch("beta", "gamma")) + (self.project / "input.rs").write_text("input\n") + self.write_project() + + def command(self, command, success=True): + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + universal_newlines=True) + if success: + self.assertEqual(result.returncode, 0, result.stdout) + else: + self.assertNotEqual(result.returncode, 0, result.stdout) + return result.stdout + + def make_archive(self): + with tarfile.open(str(self.archive), "w:gz") as archive: + archive.add(str(self.package), arcname="dep") + self.sha = hashlib.sha256(self.archive.read_bytes()).hexdigest() + + def write_versions(self, version="1", checksum=None, dependency="1"): + (self.project / "third_party/versions.txt").write_text( + "DEP_VERSION={}\nDEP_SHA={}\nOTHER_VERSION={}\n".format( + version, checksum or self.sha, dependency)) + + def write_project(self, patches="one.diff two.diff", extra="", layout=""): + (self.project / "record.cmake").write_text(''' +file(APPEND "${CMAKE_CURRENT_LIST_DIR}/events" "${PHASE}\\n") +if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/fail-${PHASE}") + message(FATAL_ERROR "injected ${PHASE} failure") +endif() +if(PHASE STREQUAL "build") + file(READ "${SOURCE}/value" value) + file(WRITE "${BINARY}/built-value" "${value}") +elseif(PHASE STREQUAL "install") + file(MAKE_DIRECTORY "${DEST}") + file(COPY "${BINARY}/built-value" DESTINATION "${DEST}") +endif() +''') + text = '''cmake_minimum_required(VERSION 3.22) +project(PatchTest NONE) +if(POLICY CMP0135) + cmake_policy(SET CMP0135 NEW) +endif() +include("@MODULES@/ThirdpartyPatch.cmake") +file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/third_party/versions.txt" versions) +foreach(entry IN LISTS versions) + string(REPLACE "=" ";" fields "${entry}") + list(GET fields 0 key) + list(GET fields 1 value) + set(${key} "${value}") +endforeach() +set(patches @PATCHES@) +list(TRANSFORM patches PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/") +set(dest "${CMAKE_CURRENT_BINARY_DIR}/dep-install") +set(extra_options "") +@LAYOUT@ +paimon_patch_command(dep patch_command) +externalproject_add(dep + URL "${CMAKE_CURRENT_SOURCE_DIR}/dep.tar.gz" + URL_HASH "SHA256=${DEP_SHA}" + ${extra_options} + PATCH_COMMAND ${patch_command} + CONFIGURE_COMMAND "" + BUILD_COMMAND "${CMAKE_COMMAND}" -DPHASE=build "-DSOURCE=" + "-DBINARY=" "-DDEST=${dest}" + -P "${CMAKE_CURRENT_SOURCE_DIR}/record.cmake" + INSTALL_COMMAND "${CMAKE_COMMAND}" -DPHASE=install "-DSOURCE=" + "-DBINARY=" "-DDEST=${dest}" + -P "${CMAKE_CURRENT_SOURCE_DIR}/record.cmake" + BUILD_BYPRODUCTS "${dest}/built-value") +paimon_register_patched_project(dep + VERSION "${DEP_VERSION}" + PATCHES ${patches} + INPUTS "${CMAKE_CURRENT_SOURCE_DIR}/input.rs" + INSTALL_DIR "${dest}" + VERSION_VARIABLES DEP_VERSION DEP_SHA OTHER_VERSION) +@EXTRA@ +''' + for key, value in [("MODULES", str(self.modules)), ("PATCHES", patches), + ("EXTRA", extra), ("LAYOUT", layout)]: + text = text.replace("@{}@".format(key), value) + (self.project / "CMakeLists.txt").write_text(text) + + def configure(self, success=True, options=()): + return self.command([ARGS.cmake, "-S", str(self.project), "-B", str(self.build), + "-G", ARGS.generator] + list(options), success) + + def compile(self, success=True, target="dep"): + return self.command([ARGS.cmake, "--build", str(self.build), "--target", target, + "--parallel", "4"], success) + + def built(self): + self.configure() + self.compile() + self.assertEqual((self.install / "built-value").read_text(), "gamma\n") + + def assert_cleanup(self, output): + self.assertIn("Remove these build-owned paths", " ".join(output.split())) + self.assertIn(str(self.source), output) + self.assertIn(str(self.install), output) + self.assertIn(str(self.stamp), output) + + def assert_reuse_rejected(self, targets=("dep",)): + for target in targets: + with self.subTest(target=target): + self.assert_cleanup(self.compile(False, target=target)) + self.assert_cleanup(self.command([ARGS.cmake, "--install", str(self.build)], False)) + self.assertEqual((self.project / "events").read_text(), "build\ninstall\n") + self.assertEqual((self.install / "built-value").read_text(), "gamma\n") + + def clean_reported_paths(self, output): + paths = {Path(line.strip()) for line in output.splitlines() + if line.strip().startswith(str(self.build) + os.sep)} + self.assertTrue(paths, output) + for path in paths: + self.assertIn(self.build.resolve(), path.resolve().parents) + if path.is_dir(): + shutil.rmtree(str(path)) + elif path.exists(): + path.unlink() + + def test_ordered_patches_and_repeat_build(self): + self.built() + artifact = self.install / "built-value" + timestamp = artifact.stat().st_mtime_ns + self.compile() + self.configure() + self.compile() + self.assertEqual((self.project / "events").read_text(), "build\ninstall\n") + self.assertEqual(artifact.stat().st_mtime_ns, timestamp) + + def test_changed_patch_detected_even_with_preserved_timestamp(self): + self.built() + path = self.project / "two.diff" + stat = path.stat() + path.write_text(patch("beta", "delta")) + os.utime(str(path), ns=(stat.st_atime_ns, stat.st_mtime_ns)) + self.assert_reuse_rejected() + + def test_patch_list_changes_require_cleanup(self): + self.built() + (self.project / "three.diff").write_text(patch("gamma", "delta")) + for patches in ["two.diff one.diff", "", "one.diff two.diff three.diff"]: + with self.subTest(patches=patches): + self.write_project(patches=patches) + self.assert_cleanup(self.configure(False)) + self.assertEqual((self.install / "built-value").read_text(), "gamma\n") + + def test_source_and_build_input_changes_require_cleanup(self): + cases = { + "version": lambda: self.write_versions(version="2"), + "checksum": lambda: self.write_versions(checksum="0" * 64), + "dependency": lambda: self.write_versions(dependency="2"), + "additional input": lambda: (self.project / "input.rs").write_text("changed\n"), + "missing patch": lambda: (self.project / "one.diff").unlink(), + "build type": lambda: self.write_project(layout='set(CMAKE_BUILD_TYPE Release)'), + } + for name, change in cases.items(): + with self.subTest(change=name): + self.create_project() + self.built() + change() + self.assert_reuse_rejected() + + def test_failed_second_patch_blocks_retry(self): + (self.project / "two.diff").write_text(patch("wrong", "gamma")) + self.configure() + self.assert_cleanup(self.compile(False)) + self.assertEqual((self.source / "value").read_text(), "beta\n") + self.command([ARGS.cmake, "--install", str(self.build)], False) + output = self.compile(False) + self.assertIn("interrupted", output) + self.assertFalse((self.install / "built-value").exists()) + + def test_interrupted_patch_blocks_retry(self): + real_patch = shutil.which("patch") + wrapper = self.project / "patch" + wrapper.write_text('''#!/usr/bin/env python3 +import os, signal, subprocess, sys +result = subprocess.call([@PATCH@] + sys.argv[1:]) +if result == 0 and '--dry-run' not in sys.argv: + os.kill(os.getppid(), signal.SIGKILL) +sys.exit(result) +'''.replace("@PATCH@", repr(real_patch))) + wrapper.chmod(0o755) + self.configure(options=["-DCMAKE_PROGRAM_PATH=" + str(self.project)]) + self.compile(False) + self.assertEqual((self.source / "value").read_text(), "beta\n") + self.assertIn("interrupted", self.compile(False)) + self.command([ARGS.cmake, "--install", str(self.build)], False) + + def test_build_and_install_failures_can_resume(self): + for phase in ["build", "install"]: + with self.subTest(phase=phase): + self.create_project() + failure = self.project / ("fail-" + phase) + failure.touch() + self.configure() + self.compile(False) + self.command([ARGS.cmake, "--install", str(self.build)], False) + self.assertFalse((self.install / "built-value").exists()) + failure.unlink() + self.compile() + self.command([ARGS.cmake, "--install", str(self.build)]) + self.assertEqual((self.install / "built-value").read_text(), "gamma\n") + + def test_source_damage_cannot_bypass_guard(self): + self.write_project(extra="externalproject_add_steptargets(dep build)") + self.built() + source = self.source / "value" + for content in ["alpha\n", None]: + with self.subTest(content=content): + if content is None: + source.unlink() + else: + source.write_text(content) + try: + self.assert_reuse_rejected(targets=("dep", "dep-build")) + finally: + source.write_text("gamma\n") + self.compile() + + def test_install_completion_does_not_check_individual_outputs(self): + self.built() + (self.install / "built-value").unlink() + # A regular build tool can rebuild declared byproducts. The patch guard + # itself accepts a completed identity without inspecting installed files. + self.configure() + self.command([ARGS.cmake, "--install", str(self.build)]) + self.assertEqual((self.project / "events").read_text(), "build\ninstall\n") + self.assertFalse((self.install / "built-value").exists()) + + def test_damaged_completion_state_is_not_success(self): + self.built() + # Fault injection deliberately knows the state filenames. Verify the + # public build/install behavior, not the contents of a valid marker. + state = self.build / "paimon-thirdparty/dep" + for name, replacement in [("installed", None), ("installed", "outdated identity"), + ("patched", None)]: + with self.subTest(marker=name, replacement=replacement): + marker = state / name + original = marker.read_bytes() + if replacement is None: + marker.unlink() + else: + marker.write_text(replacement) + try: + self.assert_reuse_rejected() + finally: + marker.write_bytes(original) + + def test_legacy_artifacts_are_not_adopted_or_deleted(self): + for kind in ["source", "install", "stamp"]: + with self.subTest(kind=kind): + self.create_project() + artifact = {"source": self.source / "value", + "install": self.install / "obsolete.h", + "stamp": self.stamp / "dep-install"}[kind] + artifact.parent.mkdir(parents=True) + artifact.write_text("old data\n") + if kind == "source": + (self.source / ".patched").touch() + self.assert_cleanup(self.configure(False)) + self.assertEqual(artifact.read_text(), "old data\n") + + def test_file_addition_and_deletion(self): + (self.project / "two.diff").write_text('''diff --git a/other b/other +--- a/other ++++ /dev/null +@@ -1 +0,0 @@ +-unchanged +diff --git a/added b/added +--- /dev/null ++++ b/added +@@ -0,0 +1 @@ ++created +''') + self.configure() + self.compile() + self.compile() + self.assertFalse((self.source / "other").exists()) + self.assertEqual((self.source / "added").read_text(), "created\n") + (self.source / "other").write_text("old\n") + self.assertIn("Deleted patch target reappeared", self.compile(False)) + + def test_layout_migration_reports_old_paths(self): + layouts = { + "orc": '''set(dest "${CMAKE_CURRENT_BINARY_DIR}/dep-prefix") +set(extra_options SOURCE_DIR "${dest}/cpp" BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/build/dep" + UPDATE_DISCONNECTED TRUE)''', + "cargo": '''set(extra_options BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/dep-cargo")''', + } + for name, layout in layouts.items(): + with self.subTest(layout=name): + self.create_project() + self.write_project(layout=layout) + if name == "orc": + self.install = self.build / "dep-prefix" + self.source = self.install / "cpp" + old_binary = self.build / ("build/dep" if name == "orc" else "dep-cargo") + self.built() + old_source, old_install = self.source, self.install + self.write_project() + self.source = self.build / "dep-prefix/src/dep" + self.install = self.build / "dep-install" + # Repeated failed configuration must retain the original advice. + for _ in range(2): + output = self.configure(False) + self.assert_cleanup(output) + for path in [old_source, old_install, old_binary]: + self.assertIn(str(path), output) + self.assertEqual((old_install / "built-value").read_text(), "gamma\n") + self.clean_reported_paths(output) + self.built() + + def test_version_file_is_relative_to_module_in_subproject(self): + parent = self.root / "parent" + parent.mkdir() + (parent / "third_party").mkdir() + (parent / "third_party/versions.txt").write_text("unrelated parent versions\n") + (parent / "CMakeLists.txt").write_text('''cmake_minimum_required(VERSION 3.22) +project(Parent NONE) +add_subdirectory("{}" dependency) +'''.format(self.project)) + self.command([ARGS.cmake, "-S", str(parent), "-B", str(self.build), + "-G", ARGS.generator]) + self.compile() + self.install = self.build / "dependency/dep-install" + self.source = self.build / "dependency/dep-prefix/src/dep" + self.stamp = self.build / "dependency/dep-prefix/src/dep-stamp" + self.assertEqual((self.install / "built-value").read_text(), "gamma\n") + self.write_versions(version="2") + self.assert_cleanup(self.compile(False)) + + def test_vortex_build_uses_standard_directory_and_source_working_directory(self): + # Exercise the production adapter with Cargo simulated, without requiring + # a Rust toolchain or building the Vortex workspace in this offline suite. + toolchain = (MODULES / "ThirdpartyToolchain.cmake").read_text() + macro = "macro(build_vortex)" + toolchain.split("macro(build_vortex)", 1)[1] + macro = macro.split("endmacro()", 1)[0] + "endmacro()\n" + (self.project / "vortex-adapter.cmake").write_text(macro) + (self.package / "vortex-ffi/cinclude").mkdir(parents=True) + (self.package / "vortex-ffi/src").mkdir() + (self.package / "vortex-ffi/cinclude/vortex.h").write_text("header\n") + (self.package / ".cargo").mkdir() + (self.package / ".cargo/config.toml").write_text("# workspace config\n") + self.make_archive() + (self.project / "third_party/versions.txt").write_text( + "PAIMON_VORTEX_BUILD_VERSION=1\nPAIMON_VORTEX_BUILD_SHA256_CHECKSUM={}\n".format(self.sha)) + shutil.copyfile(str(self.project / "one.diff"), str(self.project / "vortex.diff")) + callback_dir = self.project / "crates/vortex_callback_io" + callback_dir.mkdir(parents=True) + (callback_dir / "callback_io.rs").write_text("callback\n") + cargo = self.project / "cargo" + cargo.write_text('''#!/usr/bin/env python3 +import os +from pathlib import Path +assert Path('.cargo/config.toml').is_file() +assert Path('value').read_text() == 'beta\\n' +assert Path('vortex-ffi/src/callback_io.rs').read_text() == 'callback\\n' +target = Path(os.environ['CARGO_TARGET_DIR']) +(target / 'release').mkdir(parents=True, exist_ok=True) +(target / 'release/libvortex_ffi.a').write_text('simulated library\\n') +(target / 'working-directory').write_text(str(Path.cwd())) +''') + cargo.chmod(0o755) + (self.project / "CMakeLists.txt").write_text('''cmake_minimum_required(VERSION 3.22) +project(VortexAdapter NONE) +if(POLICY CMP0135) + cmake_policy(SET CMP0135 NEW) +endif() +include("${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules/ThirdpartyPatch.cmake") +include("${CMAKE_CURRENT_SOURCE_DIR}/vortex-adapter.cmake") +set(PAIMON_CARGO_EXECUTABLE "${CMAKE_CURRENT_SOURCE_DIR}/cargo") +set(VORTEX_FLATC "${CMAKE_COMMAND}") +set(PAIMON_VORTEX_BUILD_VERSION 1) +file(SHA256 "${CMAKE_CURRENT_SOURCE_DIR}/dep.tar.gz" PAIMON_VORTEX_BUILD_SHA256_CHECKSUM) +set(VORTEX_SOURCE_URL "${CMAKE_CURRENT_SOURCE_DIR}/dep.tar.gz") +set(CMAKE_STATIC_LIBRARY_PREFIX lib) +set(CMAKE_STATIC_LIBRARY_SUFFIX .a) +set(CMAKE_INSTALL_LIBDIR lib) +add_library(Threads::Threads INTERFACE IMPORTED) +build_vortex() +''') + self.configure() + self.compile(target="vortex_ep") + binary = self.build / "vortex_ep-prefix/src/vortex_ep-build" + source = self.build / "vortex_ep-prefix/src/vortex_ep" + self.assertEqual((binary / "working-directory").read_text(), str(source)) + installed = self.build / "vortex_ep-install/lib/libvortex_ffi.a" + self.assertEqual(installed.read_text(), "simulated library\n") + timestamp = installed.stat().st_mtime_ns + self.compile(target="vortex_ep") + self.assertEqual(installed.stat().st_mtime_ns, timestamp) + self.assertFalse((self.build / "vortex_ep-cargo").exists()) + + def test_clean_rebuild_after_version_change(self): + self.built() + self.write_versions(version="2") + output = self.configure(False) + self.assert_cleanup(output) + self.clean_reported_paths(output) + self.built() + + def test_empty_patch_list(self): + self.write_project(patches="") + self.configure() + self.compile() + self.compile() + self.assertEqual((self.install / "built-value").read_text(), "alpha\n") + + def test_escaping_patch_path(self): + (self.project / "one.diff").write_text(patch("alpha", "beta", "../outside")) + self.configure() + self.assertIn("Unsafe patch path", self.compile(False)) + + +if __name__ == "__main__": + unittest.main(argv=[__file__] + UNITTEST_ARGS, verbosity=2) diff --git a/docs/source/building.rst b/docs/source/building.rst index 14973d13f..3d3e6e28f 100644 --- a/docs/source/building.rst +++ b/docs/source/building.rst @@ -262,6 +262,20 @@ All other enabled dependencies must be available as system packages or under the specified prefix. When ORC support is disabled, the ``ORC_SOURCE`` override can be omitted. +Some third-party dependencies require project-specific patches. +Repeated builds reuse these dependencies when their versions, patches, and build +inputs are unchanged. + +If an existing build cannot be safely reused after an update or interrupted +patching, the build stops and lists the paths to clean. Remove all listed paths, +then re-run CMake and build. No files are deleted automatically. Older build +directories may also require this cleanup when first using the patch checks. + +Compilation or installation failures after successful patching can be retried +without cleanup if the inputs are unchanged. Avoid manually modifying dependency +sources or installed files in the build directory; clean and rebuild the affected +dependency if its files are damaged. + Package-manager-specific modes are intentionally out of scope for this first dependency source interface. They can still be used through standard CMake mechanisms such as ``CMAKE_PREFIX_PATH`` or ``CMAKE_TOOLCHAIN_FILE``, while From 3240553dc73cf68c90db33c163e609703a6e4327 Mon Sep 17 00:00:00 2001 From: "jinli.zjw" Date: Mon, 16 Nov 2026 19:16:41 +0800 Subject: [PATCH 2/6] fix(build): harden third-party patch validation --- CMakeLists.txt | 6 - cmake_modules/ApplyThirdpartyPatch.cmake | 98 ++++++++----- cmake_modules/BuildUtils.cmake | 4 +- cmake_modules/ThirdpartyPatch.cmake | 24 +++ cmake_modules/tests/thirdparty_patch_test.py | 147 ++++++++++++++++++- docs/source/building.rst | 7 +- 6 files changed, 232 insertions(+), 54 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index df1c51ea0..149d689a0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -149,12 +149,6 @@ include(GNUInstallDirs) set(PAIMON_CMAKE_INSTALL_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/Paimon") include(DefineOptions) -if(PAIMON_OPTIONAL_INSTALL) - # Don't make the "install" target depend on the "all" target - set(CMAKE_SKIP_INSTALL_ALL_DEPENDENCY true) - set(INSTALL_IS_OPTIONAL OPTIONAL) -endif() - if(PAIMON_EXTRA_ERROR_CONTEXT) add_definitions(-DPAIMON_EXTRA_ERROR_CONTEXT) endif() diff --git a/cmake_modules/ApplyThirdpartyPatch.cmake b/cmake_modules/ApplyThirdpartyPatch.cmake index 0f6d37d97..b45419690 100644 --- a/cmake_modules/ApplyThirdpartyPatch.cmake +++ b/cmake_modules/ApplyThirdpartyPatch.cmake @@ -29,9 +29,15 @@ function(paimon_patch_error) endfunction() function(paimon_patch_fingerprint output) + foreach(url IN LISTS PP_URLS) + if(IS_DIRECTORY "${url}") + paimon_patch_error("Local source directories are not supported; use a SHA256-verified archive: ${url}" + ) + endif() + endforeach() # The same ordered list is used here and by the application loop below. set(identity - "paimon-patch-v1\n${PP_NAME}\n${PP_VERSION}\n${PP_ARCHIVE}\n${PP_SOURCE}\n${PP_BINARY}\n${PP_INSTALL}\n${PP_CONTEXT}\n" + "paimon-patch-v2\n${PP_NAME}\n${PP_VERSION}\n${PP_ARCHIVE}\n${PP_URLS}\n${PP_SOURCE}\n${PP_BINARY}\n${PP_INSTALL}\n${PP_CONTEXT}\n" ) file(SHA256 "${CMAKE_CURRENT_FUNCTION_LIST_FILE}" runner_hash) string(APPEND identity @@ -121,6 +127,12 @@ function(paimon_patch_check_postimage) if(NOT source_id STREQUAL PP_EXPECTED) paimon_patch_error("Source identity does not match its build state") endif() + file(SHA256 "${PP_STATE}/postimage" postimage_hash) + file(READ "${PP_STATE}/patched" patched) + if(NOT patched STREQUAL "${PP_EXPECTED}\n${postimage_hash}\n") + paimon_patch_error("Patch completion or source verification record is inconsistent" + ) + endif() file(STRINGS "${PP_STATE}/postimage" entries) foreach(entry IN LISTS entries) if(NOT entry MATCHES "^([^ ]+) (.+)$") @@ -194,9 +206,10 @@ function(paimon_patch_guard) ) endif() if(EXISTS "${PP_STATE}/patched") - file(READ "${PP_STATE}/patched" patched) - if(NOT patched STREQUAL PP_EXPECTED) - paimon_patch_error("Patch completion record does not match the requested identity" + # Clean targets remove ExternalProject stamps but retain the source and + # our records. Stop before download can overwrite that patched source. + if(NOT EXISTS "${PP_STAMP}/${PP_NAME}-download") + paimon_patch_error("Download completion record is missing; refusing to overwrite patched source" ) endif() paimon_patch_check_postimage() @@ -240,44 +253,48 @@ function(paimon_patch_apply) # renames, mode changes or paths through symlinks. set(targets "") foreach(patch IN LISTS PP_PATCHES) - file(STRINGS "${patch}" lines) - set(count 0) - foreach(line IN LISTS lines) - if(line MATCHES - "^(GIT binary patch|rename from |rename to |old mode |new mode )") - paimon_patch_error("Unsupported non-text patch operation in ${patch}") + file(READ "${patch}" contents) + if(contents MATCHES + "(^|\n)(GIT binary patch|rename from |rename to |old mode |new mode )") + paimon_patch_error("Unsupported non-text patch operation in ${patch}") + endif() + execute_process(COMMAND "${PP_LSDIFF_EXECUTABLE}" -- "${patch}" + RESULT_VARIABLE result + OUTPUT_VARIABLE paths + ERROR_VARIABLE error) + if(NOT result STREQUAL "0") + paimon_patch_error("Cannot list patch targets in ${patch}: ${error}") + endif() + if(paths STREQUAL "") + paimon_patch_error("No text patch targets found in ${patch}") + endif() + # Only convert the filename output to a list, never the diff body. + # Reject filenames that cannot be represented by our verification record. + if(paths MATCHES "[;\\\t]|\\[|\\]") + paimon_patch_error("Unsafe patch path in ${patch}: ${paths}") + endif() + string(REGEX REPLACE "\n+$" "" paths "${paths}") + string(REPLACE "\n" ";" paths "${paths}") + foreach(path IN LISTS paths) + if(NOT path MATCHES "^[ab]/(.+)$") + paimon_patch_error("Unsupported patch path in ${patch}: ${path}") endif() - if(line MATCHES "^(---|\\+\\+\\+) (.*)$") - set(path "${CMAKE_MATCH_2}") - if(path STREQUAL "/dev/null") - continue() - endif() - if(NOT path MATCHES "^[ab]/(.+)$") - paimon_patch_error("Unsupported patch path in ${patch}: ${path}") - endif() - set(relative "${CMAKE_MATCH_1}") - if(relative MATCHES "(^|/)\\.\\.(/|$)|[;\\\\\t]" OR IS_ABSOLUTE - "${relative}") - paimon_patch_error("Unsafe patch path in ${patch}: ${relative}") - endif() - file(REAL_PATH "${PP_SOURCE}" source_real) - file(REAL_PATH "${PP_SOURCE}/${relative}" target_real) - cmake_path(IS_PREFIX - source_real - "${target_real}" - NORMALIZE - contained) - if(NOT contained OR IS_SYMLINK "${PP_SOURCE}/${relative}") - paimon_patch_error("Patch target escapes its source tree: ${relative}" - ) - endif() - list(APPEND targets "${relative}") - math(EXPR count "${count} + 1") + set(relative "${CMAKE_MATCH_1}") + if(relative MATCHES "(^|/)\\.\\.(/|$)" OR IS_ABSOLUTE "${relative}") + paimon_patch_error("Unsafe patch path in ${patch}: ${relative}") endif() + file(REAL_PATH "${PP_SOURCE}" source_real) + file(REAL_PATH "${PP_SOURCE}/${relative}" target_real) + cmake_path(IS_PREFIX + source_real + "${target_real}" + NORMALIZE + contained) + if(NOT contained OR IS_SYMLINK "${PP_SOURCE}/${relative}") + paimon_patch_error("Patch target escapes its source tree: ${relative}") + endif() + list(APPEND targets "${relative}") endforeach() - if(count EQUAL 0) - paimon_patch_error("No text patch targets found in ${patch}") - endif() endforeach() paimon_patch_write(applying "${PP_EXPECTED}") foreach(patch IN LISTS PP_PATCHES) @@ -316,7 +333,8 @@ function(paimon_patch_apply) endforeach() paimon_patch_write(postimage "${postimage}") file(WRITE "${PP_SOURCE}/.paimon-source-id" "${PP_EXPECTED}") - paimon_patch_write(patched "${PP_EXPECTED}") + string(SHA256 postimage_hash "${postimage}") + paimon_patch_write(patched "${PP_EXPECTED}\n${postimage_hash}\n") # A crash between these operations is conservatively treated as interrupted. file(REMOVE "${PP_STATE}/applying") endfunction() diff --git a/cmake_modules/BuildUtils.cmake b/cmake_modules/BuildUtils.cmake index c0a9e14ff..a7d207a6c 100644 --- a/cmake_modules/BuildUtils.cmake +++ b/cmake_modules/BuildUtils.cmake @@ -198,7 +198,7 @@ function(add_paimon_lib LIB_NAME) target_link_options(${LIB_NAME}_shared PRIVATE ${SHARED_LINK_OPTIONS}) endif() - install(TARGETS ${LIB_NAME}_shared ${INSTALL_IS_OPTIONAL} + install(TARGETS ${LIB_NAME}_shared EXPORT PaimonTargets RUNTIME DESTINATION ${RUNTIME_INSTALL_DIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} @@ -245,7 +245,7 @@ function(add_paimon_lib LIB_NAME) target_link_libraries(${LIB_NAME}_static PUBLIC "$") - install(TARGETS ${LIB_NAME}_static ${INSTALL_IS_OPTIONAL} + install(TARGETS ${LIB_NAME}_static EXPORT PaimonTargets RUNTIME DESTINATION ${RUNTIME_INSTALL_DIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} diff --git a/cmake_modules/ThirdpartyPatch.cmake b/cmake_modules/ThirdpartyPatch.cmake index cd4b53979..01ff9454c 100644 --- a/cmake_modules/ThirdpartyPatch.cmake +++ b/cmake_modules/ThirdpartyPatch.cmake @@ -74,6 +74,23 @@ function(paimon_register_patched_project name) if(NOT hash_length EQUAL 71) message(FATAL_ERROR "${name}: invalid archive SHA256") endif() + get_property(urls + TARGET ${name} + PROPERTY _EP_URL) + set(PP_URLS "") + foreach(url IN LISTS urls) + # Match ExternalProject's local file URL handling and resolve relative + # paths now, before the guard runs from a different working directory. + string(REGEX REPLACE "^file://" "" url "${url}") + if(NOT url MATCHES "^[a-zA-Z][a-zA-Z0-9+.-]*://") + get_filename_component(url + "${url}" + ABSOLUTE + BASE_DIR + "${CMAKE_CURRENT_SOURCE_DIR}") + endif() + list(APPEND PP_URLS "${url}") + endforeach() # Validate ownership even though this module never deletes directories. # In particular, do not generate cleanup advice for a user checkout. @@ -131,6 +148,11 @@ function(paimon_register_patched_project name) endforeach() set(PP_RUNNER "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/ApplyThirdpartyPatch.cmake") find_program(PP_PATCH_EXECUTABLE patch REQUIRED) + find_program(PP_LSDIFF_EXECUTABLE lsdiff) + if(NOT PP_LSDIFF_EXECUTABLE) + message(FATAL_ERROR "${name}: lsdiff is required to inspect dependency patches. Install patchutils and re-run CMake." + ) + endif() set_property(DIRECTORY APPEND @@ -157,12 +179,14 @@ function(paimon_register_patched_project name) CLEAN VERSION ARCHIVE + URLS PATCHES INPUTS VERSIONS_FILE VERSION_VARIABLES CONTEXT PATCH_EXECUTABLE + LSDIFF_EXECUTABLE EXPECTED) # Bracket arguments preserve spaces, semicolons, dollar signs and backslashes. set(delimiter "=") diff --git a/cmake_modules/tests/thirdparty_patch_test.py b/cmake_modules/tests/thirdparty_patch_test.py index 210ebecf1..eae60c017 100644 --- a/cmake_modules/tests/thirdparty_patch_test.py +++ b/cmake_modules/tests/thirdparty_patch_test.py @@ -15,7 +15,9 @@ # specific language governing permissions and limitations # under the License. -"""Offline ExternalProject lifecycle tests. No compiler or downloaded packages needed. +"""Offline ExternalProject lifecycle tests. Requires patch and lsdiff (patchutils). + +No compiler or downloaded packages needed. python3 cmake_modules/tests/thirdparty_patch_test.py --generator Ninja python3 cmake_modules/tests/thirdparty_patch_test.py --generator 'Unix Makefiles' @@ -212,6 +214,140 @@ def test_changed_patch_detected_even_with_preserved_timestamp(self): os.utime(str(path), ns=(stat.st_atime_ns, stat.st_mtime_ns)) self.assert_reuse_rejected() + def test_multifile_patch_with_brackets_and_header_like_content(self): + (self.package / "brackets").write_text("unchanged\n") + (self.package / "other").write_text("-- a/phantom\n") + self.make_archive() + self.write_versions() + (self.project / "one.diff").write_text( + patch("unchanged", "[;", "brackets") + + patch("-- a/phantom", "++ b/phantom", "other") + patch("alpha", "gamma")) + self.write_project(patches="one.diff") + self.built() + for name, expected in [("brackets", "[;\n"), ("other", "++ b/phantom\n"), + ("value", "gamma\n")]: + source = self.source / name + self.assertEqual(source.read_text(), expected) + for replacement in ["corrupted\n", None]: + with self.subTest(file=name, replacement=replacement): + if replacement is None: + source.unlink() + else: + source.write_text(replacement) + try: + self.assert_reuse_rejected() + finally: + source.write_text(expected) + self.compile() + + def test_missing_lsdiff_reports_required_package(self): + output = self.configure(False, options=[ + "-DCMAKE_FIND_USE_CMAKE_SYSTEM_PATH=OFF", + "-DCMAKE_FIND_USE_SYSTEM_ENVIRONMENT_PATH=OFF", + "-DCMAKE_FIND_USE_CMAKE_ENVIRONMENT_PATH=OFF", + "-DCMAKE_MAKE_PROGRAM=" + shutil.which( + "ninja" if ARGS.generator == "Ninja" else "make"), + "-DPP_PATCH_EXECUTABLE=" + shutil.which("patch"), + ]) + self.assertIn("lsdiff is required", output) + self.assertIn("Install patchutils", output) + self.assertFalse((self.source / "value").exists()) + + def test_lsdiff_failure_or_empty_output_does_not_apply_patches(self): + for exit_code, message in [(1, "Cannot list patch targets"), + (0, "No text patch targets found")]: + with self.subTest(exit_code=exit_code): + self.create_project() + wrapper = self.project / "lsdiff" + wrapper.write_text("#!/bin/sh\nexit {}\n".format(exit_code)) + wrapper.chmod(0o755) + self.configure(options=["-DCMAKE_PROGRAM_PATH=" + str(self.project)]) + self.assertIn(message, self.compile(False)) + self.assertEqual((self.source / "value").read_text(), "alpha\n") + self.assertFalse((self.install / "built-value").exists()) + + def test_local_directory_urls_are_rejected(self): + for prefix in ["", "file://"]: + with self.subTest(prefix=prefix): + self.create_project() + cmake = self.project / "CMakeLists.txt" + cmake.write_text(cmake.read_text().replace( + '${CMAKE_CURRENT_SOURCE_DIR}/dep.tar.gz', prefix + str(self.package))) + self.write_versions(checksum="0" * 64) + output = self.configure(False) + self.assertIn("Local source directories are not supported", output) + self.assertFalse((self.source / "value").exists()) + self.assertFalse((self.stamp / "dep-download").exists()) + self.assertEqual((self.package / "value").read_text(), "alpha\n") + + def test_local_archive_replaced_by_directory_is_rejected(self): + self.configure() + self.archive.unlink() + shutil.copytree(str(self.package), str(self.archive)) + output = self.compile(False) + self.assertIn("Local source directories are not supported", output) + self.assertFalse((self.source / "value").exists()) + + def test_local_archive_urls_still_require_matching_checksum(self): + for prefix in ["", "file://"]: + for valid in [True, False]: + with self.subTest(prefix=prefix, valid_checksum=valid): + self.create_project() + cmake = self.project / "CMakeLists.txt" + cmake.write_text(cmake.read_text().replace( + '${CMAKE_CURRENT_SOURCE_DIR}/dep.tar.gz', prefix + str(self.archive))) + if not valid: + self.write_versions(checksum="0" * 64) + self.configure() + if valid: + self.compile() + self.assertEqual((self.install / "built-value").read_text(), "gamma\n") + else: + self.compile(False) + self.assertFalse((self.source / "value").exists()) + self.assertFalse((self.install / "built-value").exists()) + + def test_clean_first_stops_before_source_is_overwritten(self): + self.built() + output = self.command([ARGS.cmake, "--build", str(self.build), "--target", "dep", + "--clean-first"], False) + self.assertIn("Download completion record is missing", output) + self.assert_cleanup(output) + self.assertEqual((self.source / "value").read_text(), "gamma\n") + self.assertEqual((self.project / "events").read_text(), "build\ninstall\n") + self.clean_reported_paths(output) + self.built() + + def test_same_archive_at_new_url_requires_cleanup(self): + self.built() + mirror = self.project / "mirror.tar.gz" + shutil.copyfile(str(self.archive), str(mirror)) + cmake = self.project / "CMakeLists.txt" + cmake.write_text(cmake.read_text().replace("dep.tar.gz", "mirror.tar.gz")) + output = self.configure(False) + self.assert_cleanup(output) + self.assertEqual((self.source / "value").read_text(), "gamma\n") + self.assert_reuse_rejected() + self.clean_reported_paths(output) + self.built() + + def test_truncated_postimage_is_rejected(self): + (self.project / "two.diff").write_text( + patch("beta", "gamma") + patch("unchanged", "updated", "other")) + self.built() + record = self.build / "paimon-thirdparty/dep/postimage" + original = record.read_text() + for replacement in [original.splitlines()[0] + "\n", ""]: + with self.subTest(postimage=replacement): + record.write_text(replacement) + (self.source / "other").write_text("corrupted\n") + try: + self.assert_reuse_rejected() + finally: + record.write_text(original) + (self.source / "other").write_text("updated\n") + self.compile() + def test_patch_list_changes_require_cleanup(self): self.built() (self.project / "three.diff").write_text(patch("gamma", "delta")) @@ -485,9 +621,12 @@ def test_empty_patch_list(self): self.assertEqual((self.install / "built-value").read_text(), "alpha\n") def test_escaping_patch_path(self): - (self.project / "one.diff").write_text(patch("alpha", "beta", "../outside")) - self.configure() - self.assertIn("Unsafe patch path", self.compile(False)) + for name in ["../outside", "file[", "semi;colon", "back\\slash"]: + with self.subTest(path=name): + self.create_project() + (self.project / "one.diff").write_text(patch("alpha", "beta", name)) + self.configure() + self.assertIn("Unsafe patch path", self.compile(False)) if __name__ == "__main__": diff --git a/docs/source/building.rst b/docs/source/building.rst index 3d3e6e28f..0cdd8bc27 100644 --- a/docs/source/building.rst +++ b/docs/source/building.rst @@ -34,6 +34,7 @@ Building requires: * A C++17-enabled compiler. On Linux, gcc 8 and higher should be sufficient. Windows is not supported for now. +* ``patch`` and ``lsdiff`` (provided by ``patchutils``) for patched dependencies. * At least 2GB of RAM for a minimal build, 8GB for a minimal debug build with tests and 16GB for a full build. @@ -43,7 +44,9 @@ On Ubuntu/Debian you can install the requirements with: sudo apt-get install \ build-essential \ - cmake + cmake \ + patch \ + patchutils We also provide a docker template to help you get started quickly. See in ``.devcontainer`` folder for more details. @@ -266,7 +269,7 @@ Some third-party dependencies require project-specific patches. Repeated builds reuse these dependencies when their versions, patches, and build inputs are unchanged. -If an existing build cannot be safely reused after an update or interrupted +If an existing build cannot be safely reused after an update, cleanup, or interrupted patching, the build stops and lists the paths to clean. Remove all listed paths, then re-run CMake and build. No files are deleted automatically. Older build directories may also require this cleanup when first using the patch checks. From f2849e6b6f74559c9211382fd7ff4069544b4379 Mon Sep 17 00:00:00 2001 From: "jinli.zjw" Date: Tue, 17 Nov 2026 10:20:23 +0800 Subject: [PATCH 3/6] ci: install patchutils in build and development environments --- .devcontainer/Dockerfile.template | 1 + .devcontainer/centos7/Dockerfile | 1 + .github/workflows/build_and_test.yaml | 6 +++++- .github/workflows/gcc8_test.yaml | 2 +- .github/workflows/release_candidate.yaml | 4 ++-- 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.devcontainer/Dockerfile.template b/.devcontainer/Dockerfile.template index 0d2e3816f..c93529630 100644 --- a/.devcontainer/Dockerfile.template +++ b/.devcontainer/Dockerfile.template @@ -65,6 +65,7 @@ RUN apt update && \ lsb-release \ meson \ ninja-build \ + patchutils \ pkg-config \ python3 \ python3-pip \ diff --git a/.devcontainer/centos7/Dockerfile b/.devcontainer/centos7/Dockerfile index fa0f9c1cd..6119a3587 100644 --- a/.devcontainer/centos7/Dockerfile +++ b/.devcontainer/centos7/Dockerfile @@ -133,6 +133,7 @@ enabled=1\n' > /etc/yum.repos.d/CentOS-SCLo-scl.repo \ ninja-build \ make \ patch \ + patchutils \ curl \ wget \ unzip \ diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 7e32fffe3..50e9c05a5 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -40,6 +40,10 @@ jobs: steps: - name: Checkout paimon-cpp uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Install patch inspection tools + run: | + sudo apt-get update + sudo apt-get install -y patchutils - name: Run CMake module tests shell: bash run: ci/scripts/test_cmake_modules.sh @@ -126,7 +130,7 @@ jobs: - name: Install build and debugging dependencies run: | sudo apt-get update - sudo apt-get install -y gdb libcurl4-openssl-dev libssl-dev + sudo apt-get install -y gdb libcurl4-openssl-dev libssl-dev patchutils - name: Install and verify regression compiler if: matrix.gcc_version shell: bash diff --git a/.github/workflows/gcc8_test.yaml b/.github/workflows/gcc8_test.yaml index 045c43669..ca18360ea 100644 --- a/.github/workflows/gcc8_test.yaml +++ b/.github/workflows/gcc8_test.yaml @@ -52,7 +52,7 @@ jobs: printf '%s\n' 'Acquire::https::CaInfo "/etc/apt/runner-ca-certificates.crt";' \ 'Acquire::Retries "3";' > /etc/apt/apt.conf.d/80-ci-downloads apt-get update - DEBIAN_FRONTEND=noninteractive apt-get install -y gcc-8 g++-8 ninja-build git tar curl libcurl4-openssl-dev libssl-dev tzdata zip unzip pkg-config build-essential python3-dev gdb sudo + DEBIAN_FRONTEND=noninteractive apt-get install -y gcc-8 g++-8 ninja-build git tar curl libcurl4-openssl-dev libssl-dev tzdata zip unzip pkg-config build-essential python3-dev gdb sudo patchutils curl -L -O https://github.com/Kitware/CMake/releases/download/v3.28.3/cmake-3.28.3-linux-x86_64.tar.gz tar -zxvf cmake-3.28.3-linux-x86_64.tar.gz -C /usr/local --strip-components=1 rm cmake-3.28.3-linux-x86_64.tar.gz diff --git a/.github/workflows/release_candidate.yaml b/.github/workflows/release_candidate.yaml index f5ec2a573..84c7b4c55 100644 --- a/.github/workflows/release_candidate.yaml +++ b/.github/workflows/release_candidate.yaml @@ -140,10 +140,10 @@ jobs: shell: bash run: ci/scripts/setup_flatc.sh - - name: Install HTTP and TLS development dependencies + - name: Install build dependencies run: | sudo apt-get update - sudo apt-get install -y libcurl4-openssl-dev libssl-dev + sudo apt-get install -y libcurl4-openssl-dev libssl-dev patchutils - name: Verify, build, test, and install shell: bash From 62505ee8047998e549cd885367428865e8ac57ed Mon Sep 17 00:00:00 2001 From: "jinli.zjw" Date: Tue, 17 Nov 2026 10:23:14 +0800 Subject: [PATCH 4/6] test(build): keep patch lifecycle checks local --- .github/workflows/build_and_test.yaml | 4 - ci/scripts/test_cmake_modules.sh | 15 +- cmake_modules/tests/thirdparty_patch_test.py | 633 ------------------- 3 files changed, 3 insertions(+), 649 deletions(-) delete mode 100644 cmake_modules/tests/thirdparty_patch_test.py diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 50e9c05a5..4a4770ef8 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -40,10 +40,6 @@ jobs: steps: - name: Checkout paimon-cpp uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - name: Install patch inspection tools - run: | - sudo apt-get update - sudo apt-get install -y patchutils - name: Run CMake module tests shell: bash run: ci/scripts/test_cmake_modules.sh diff --git a/ci/scripts/test_cmake_modules.sh b/ci/scripts/test_cmake_modules.sh index cd0af1b06..31acb6737 100755 --- a/ci/scripts/test_cmake_modules.sh +++ b/ci/scripts/test_cmake_modules.sh @@ -15,8 +15,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Runs the CMake module tests under cmake_modules/tests. These tests need no -# compiler or configured build tree, so they stay separate from ctest. +# Runs the CMake module tests under cmake_modules/tests. These are script-mode +# tests: they need neither a toolchain nor a configured build tree, so they stay +# separate from ctest and run in seconds. set -euo pipefail @@ -30,14 +31,4 @@ for test_script in "${source_dir}"/cmake_modules/tests/*_test.cmake; do fi done -for generator in "Unix Makefiles" "Ninja"; do - if [[ "${generator}" == "Ninja" ]] && ! command -v ninja >/dev/null; then - echo "Skipping Ninja patch lifecycle tests: ninja is not installed" - continue - fi - if ! python3 "${source_dir}/cmake_modules/tests/thirdparty_patch_test.py" --generator "${generator}"; then - status=1 - fi -done - exit "${status}" diff --git a/cmake_modules/tests/thirdparty_patch_test.py b/cmake_modules/tests/thirdparty_patch_test.py deleted file mode 100644 index eae60c017..000000000 --- a/cmake_modules/tests/thirdparty_patch_test.py +++ /dev/null @@ -1,633 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -"""Offline ExternalProject lifecycle tests. Requires patch and lsdiff (patchutils). - -No compiler or downloaded packages needed. - -python3 cmake_modules/tests/thirdparty_patch_test.py --generator Ninja -python3 cmake_modules/tests/thirdparty_patch_test.py --generator 'Unix Makefiles' -Use --cmake to also test the minimum supported CMake version. -""" - -import argparse -import hashlib -import os -from pathlib import Path -import shutil -import subprocess -import tarfile -import tempfile -import unittest - - -MODULES = Path(__file__).resolve().parents[1] -PARSER = argparse.ArgumentParser() -PARSER.add_argument("--generator", default="Ninja") -PARSER.add_argument("--cmake", default="cmake") -ARGS, UNITTEST_ARGS = PARSER.parse_known_args() - - -def patch(before, after, filename="value"): - return ("diff --git a/{0} b/{0}\n--- a/{0}\n+++ b/{0}\n" - "@@ -1 +1 @@\n-{1}\n+{2}\n").format(filename, before, after) - - -class ThirdpartyPatchTest(unittest.TestCase): - def setUp(self): - self.create_project() - - def create_project(self): - self.temp = tempfile.TemporaryDirectory(prefix="paimon patch ") - self.addCleanup(self.temp.cleanup) - self.root = Path(self.temp.name) - self.project = self.root / "project" - self.project.mkdir() - self.modules = self.project / "cmake_modules" - self.modules.mkdir() - for name in ["ThirdpartyPatch.cmake", "ApplyThirdpartyPatch.cmake"]: - shutil.copyfile(str(MODULES / name), str(self.modules / name)) - (self.project / "third_party").mkdir() - # Exercise git parent discovery: source archives live under this repo. - self.command(["git", "init", "-q", str(self.project)]) - self.build = self.project / "build" - self.source = self.build / "dep-prefix/src/dep" - self.install = self.build / "dep-install" - self.stamp = self.build / "dep-prefix/src/dep-stamp" - self.package = self.root / "archive-source" - self.package.mkdir() - (self.package / "value").write_text("alpha\n") - (self.package / "other").write_text("unchanged\n") - self.archive = self.project / "dep.tar.gz" - self.make_archive() - self.write_versions() - (self.project / "one.diff").write_text(patch("alpha", "beta")) - (self.project / "two.diff").write_text(patch("beta", "gamma")) - (self.project / "input.rs").write_text("input\n") - self.write_project() - - def command(self, command, success=True): - result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - universal_newlines=True) - if success: - self.assertEqual(result.returncode, 0, result.stdout) - else: - self.assertNotEqual(result.returncode, 0, result.stdout) - return result.stdout - - def make_archive(self): - with tarfile.open(str(self.archive), "w:gz") as archive: - archive.add(str(self.package), arcname="dep") - self.sha = hashlib.sha256(self.archive.read_bytes()).hexdigest() - - def write_versions(self, version="1", checksum=None, dependency="1"): - (self.project / "third_party/versions.txt").write_text( - "DEP_VERSION={}\nDEP_SHA={}\nOTHER_VERSION={}\n".format( - version, checksum or self.sha, dependency)) - - def write_project(self, patches="one.diff two.diff", extra="", layout=""): - (self.project / "record.cmake").write_text(''' -file(APPEND "${CMAKE_CURRENT_LIST_DIR}/events" "${PHASE}\\n") -if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/fail-${PHASE}") - message(FATAL_ERROR "injected ${PHASE} failure") -endif() -if(PHASE STREQUAL "build") - file(READ "${SOURCE}/value" value) - file(WRITE "${BINARY}/built-value" "${value}") -elseif(PHASE STREQUAL "install") - file(MAKE_DIRECTORY "${DEST}") - file(COPY "${BINARY}/built-value" DESTINATION "${DEST}") -endif() -''') - text = '''cmake_minimum_required(VERSION 3.22) -project(PatchTest NONE) -if(POLICY CMP0135) - cmake_policy(SET CMP0135 NEW) -endif() -include("@MODULES@/ThirdpartyPatch.cmake") -file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/third_party/versions.txt" versions) -foreach(entry IN LISTS versions) - string(REPLACE "=" ";" fields "${entry}") - list(GET fields 0 key) - list(GET fields 1 value) - set(${key} "${value}") -endforeach() -set(patches @PATCHES@) -list(TRANSFORM patches PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/") -set(dest "${CMAKE_CURRENT_BINARY_DIR}/dep-install") -set(extra_options "") -@LAYOUT@ -paimon_patch_command(dep patch_command) -externalproject_add(dep - URL "${CMAKE_CURRENT_SOURCE_DIR}/dep.tar.gz" - URL_HASH "SHA256=${DEP_SHA}" - ${extra_options} - PATCH_COMMAND ${patch_command} - CONFIGURE_COMMAND "" - BUILD_COMMAND "${CMAKE_COMMAND}" -DPHASE=build "-DSOURCE=" - "-DBINARY=" "-DDEST=${dest}" - -P "${CMAKE_CURRENT_SOURCE_DIR}/record.cmake" - INSTALL_COMMAND "${CMAKE_COMMAND}" -DPHASE=install "-DSOURCE=" - "-DBINARY=" "-DDEST=${dest}" - -P "${CMAKE_CURRENT_SOURCE_DIR}/record.cmake" - BUILD_BYPRODUCTS "${dest}/built-value") -paimon_register_patched_project(dep - VERSION "${DEP_VERSION}" - PATCHES ${patches} - INPUTS "${CMAKE_CURRENT_SOURCE_DIR}/input.rs" - INSTALL_DIR "${dest}" - VERSION_VARIABLES DEP_VERSION DEP_SHA OTHER_VERSION) -@EXTRA@ -''' - for key, value in [("MODULES", str(self.modules)), ("PATCHES", patches), - ("EXTRA", extra), ("LAYOUT", layout)]: - text = text.replace("@{}@".format(key), value) - (self.project / "CMakeLists.txt").write_text(text) - - def configure(self, success=True, options=()): - return self.command([ARGS.cmake, "-S", str(self.project), "-B", str(self.build), - "-G", ARGS.generator] + list(options), success) - - def compile(self, success=True, target="dep"): - return self.command([ARGS.cmake, "--build", str(self.build), "--target", target, - "--parallel", "4"], success) - - def built(self): - self.configure() - self.compile() - self.assertEqual((self.install / "built-value").read_text(), "gamma\n") - - def assert_cleanup(self, output): - self.assertIn("Remove these build-owned paths", " ".join(output.split())) - self.assertIn(str(self.source), output) - self.assertIn(str(self.install), output) - self.assertIn(str(self.stamp), output) - - def assert_reuse_rejected(self, targets=("dep",)): - for target in targets: - with self.subTest(target=target): - self.assert_cleanup(self.compile(False, target=target)) - self.assert_cleanup(self.command([ARGS.cmake, "--install", str(self.build)], False)) - self.assertEqual((self.project / "events").read_text(), "build\ninstall\n") - self.assertEqual((self.install / "built-value").read_text(), "gamma\n") - - def clean_reported_paths(self, output): - paths = {Path(line.strip()) for line in output.splitlines() - if line.strip().startswith(str(self.build) + os.sep)} - self.assertTrue(paths, output) - for path in paths: - self.assertIn(self.build.resolve(), path.resolve().parents) - if path.is_dir(): - shutil.rmtree(str(path)) - elif path.exists(): - path.unlink() - - def test_ordered_patches_and_repeat_build(self): - self.built() - artifact = self.install / "built-value" - timestamp = artifact.stat().st_mtime_ns - self.compile() - self.configure() - self.compile() - self.assertEqual((self.project / "events").read_text(), "build\ninstall\n") - self.assertEqual(artifact.stat().st_mtime_ns, timestamp) - - def test_changed_patch_detected_even_with_preserved_timestamp(self): - self.built() - path = self.project / "two.diff" - stat = path.stat() - path.write_text(patch("beta", "delta")) - os.utime(str(path), ns=(stat.st_atime_ns, stat.st_mtime_ns)) - self.assert_reuse_rejected() - - def test_multifile_patch_with_brackets_and_header_like_content(self): - (self.package / "brackets").write_text("unchanged\n") - (self.package / "other").write_text("-- a/phantom\n") - self.make_archive() - self.write_versions() - (self.project / "one.diff").write_text( - patch("unchanged", "[;", "brackets") + - patch("-- a/phantom", "++ b/phantom", "other") + patch("alpha", "gamma")) - self.write_project(patches="one.diff") - self.built() - for name, expected in [("brackets", "[;\n"), ("other", "++ b/phantom\n"), - ("value", "gamma\n")]: - source = self.source / name - self.assertEqual(source.read_text(), expected) - for replacement in ["corrupted\n", None]: - with self.subTest(file=name, replacement=replacement): - if replacement is None: - source.unlink() - else: - source.write_text(replacement) - try: - self.assert_reuse_rejected() - finally: - source.write_text(expected) - self.compile() - - def test_missing_lsdiff_reports_required_package(self): - output = self.configure(False, options=[ - "-DCMAKE_FIND_USE_CMAKE_SYSTEM_PATH=OFF", - "-DCMAKE_FIND_USE_SYSTEM_ENVIRONMENT_PATH=OFF", - "-DCMAKE_FIND_USE_CMAKE_ENVIRONMENT_PATH=OFF", - "-DCMAKE_MAKE_PROGRAM=" + shutil.which( - "ninja" if ARGS.generator == "Ninja" else "make"), - "-DPP_PATCH_EXECUTABLE=" + shutil.which("patch"), - ]) - self.assertIn("lsdiff is required", output) - self.assertIn("Install patchutils", output) - self.assertFalse((self.source / "value").exists()) - - def test_lsdiff_failure_or_empty_output_does_not_apply_patches(self): - for exit_code, message in [(1, "Cannot list patch targets"), - (0, "No text patch targets found")]: - with self.subTest(exit_code=exit_code): - self.create_project() - wrapper = self.project / "lsdiff" - wrapper.write_text("#!/bin/sh\nexit {}\n".format(exit_code)) - wrapper.chmod(0o755) - self.configure(options=["-DCMAKE_PROGRAM_PATH=" + str(self.project)]) - self.assertIn(message, self.compile(False)) - self.assertEqual((self.source / "value").read_text(), "alpha\n") - self.assertFalse((self.install / "built-value").exists()) - - def test_local_directory_urls_are_rejected(self): - for prefix in ["", "file://"]: - with self.subTest(prefix=prefix): - self.create_project() - cmake = self.project / "CMakeLists.txt" - cmake.write_text(cmake.read_text().replace( - '${CMAKE_CURRENT_SOURCE_DIR}/dep.tar.gz', prefix + str(self.package))) - self.write_versions(checksum="0" * 64) - output = self.configure(False) - self.assertIn("Local source directories are not supported", output) - self.assertFalse((self.source / "value").exists()) - self.assertFalse((self.stamp / "dep-download").exists()) - self.assertEqual((self.package / "value").read_text(), "alpha\n") - - def test_local_archive_replaced_by_directory_is_rejected(self): - self.configure() - self.archive.unlink() - shutil.copytree(str(self.package), str(self.archive)) - output = self.compile(False) - self.assertIn("Local source directories are not supported", output) - self.assertFalse((self.source / "value").exists()) - - def test_local_archive_urls_still_require_matching_checksum(self): - for prefix in ["", "file://"]: - for valid in [True, False]: - with self.subTest(prefix=prefix, valid_checksum=valid): - self.create_project() - cmake = self.project / "CMakeLists.txt" - cmake.write_text(cmake.read_text().replace( - '${CMAKE_CURRENT_SOURCE_DIR}/dep.tar.gz', prefix + str(self.archive))) - if not valid: - self.write_versions(checksum="0" * 64) - self.configure() - if valid: - self.compile() - self.assertEqual((self.install / "built-value").read_text(), "gamma\n") - else: - self.compile(False) - self.assertFalse((self.source / "value").exists()) - self.assertFalse((self.install / "built-value").exists()) - - def test_clean_first_stops_before_source_is_overwritten(self): - self.built() - output = self.command([ARGS.cmake, "--build", str(self.build), "--target", "dep", - "--clean-first"], False) - self.assertIn("Download completion record is missing", output) - self.assert_cleanup(output) - self.assertEqual((self.source / "value").read_text(), "gamma\n") - self.assertEqual((self.project / "events").read_text(), "build\ninstall\n") - self.clean_reported_paths(output) - self.built() - - def test_same_archive_at_new_url_requires_cleanup(self): - self.built() - mirror = self.project / "mirror.tar.gz" - shutil.copyfile(str(self.archive), str(mirror)) - cmake = self.project / "CMakeLists.txt" - cmake.write_text(cmake.read_text().replace("dep.tar.gz", "mirror.tar.gz")) - output = self.configure(False) - self.assert_cleanup(output) - self.assertEqual((self.source / "value").read_text(), "gamma\n") - self.assert_reuse_rejected() - self.clean_reported_paths(output) - self.built() - - def test_truncated_postimage_is_rejected(self): - (self.project / "two.diff").write_text( - patch("beta", "gamma") + patch("unchanged", "updated", "other")) - self.built() - record = self.build / "paimon-thirdparty/dep/postimage" - original = record.read_text() - for replacement in [original.splitlines()[0] + "\n", ""]: - with self.subTest(postimage=replacement): - record.write_text(replacement) - (self.source / "other").write_text("corrupted\n") - try: - self.assert_reuse_rejected() - finally: - record.write_text(original) - (self.source / "other").write_text("updated\n") - self.compile() - - def test_patch_list_changes_require_cleanup(self): - self.built() - (self.project / "three.diff").write_text(patch("gamma", "delta")) - for patches in ["two.diff one.diff", "", "one.diff two.diff three.diff"]: - with self.subTest(patches=patches): - self.write_project(patches=patches) - self.assert_cleanup(self.configure(False)) - self.assertEqual((self.install / "built-value").read_text(), "gamma\n") - - def test_source_and_build_input_changes_require_cleanup(self): - cases = { - "version": lambda: self.write_versions(version="2"), - "checksum": lambda: self.write_versions(checksum="0" * 64), - "dependency": lambda: self.write_versions(dependency="2"), - "additional input": lambda: (self.project / "input.rs").write_text("changed\n"), - "missing patch": lambda: (self.project / "one.diff").unlink(), - "build type": lambda: self.write_project(layout='set(CMAKE_BUILD_TYPE Release)'), - } - for name, change in cases.items(): - with self.subTest(change=name): - self.create_project() - self.built() - change() - self.assert_reuse_rejected() - - def test_failed_second_patch_blocks_retry(self): - (self.project / "two.diff").write_text(patch("wrong", "gamma")) - self.configure() - self.assert_cleanup(self.compile(False)) - self.assertEqual((self.source / "value").read_text(), "beta\n") - self.command([ARGS.cmake, "--install", str(self.build)], False) - output = self.compile(False) - self.assertIn("interrupted", output) - self.assertFalse((self.install / "built-value").exists()) - - def test_interrupted_patch_blocks_retry(self): - real_patch = shutil.which("patch") - wrapper = self.project / "patch" - wrapper.write_text('''#!/usr/bin/env python3 -import os, signal, subprocess, sys -result = subprocess.call([@PATCH@] + sys.argv[1:]) -if result == 0 and '--dry-run' not in sys.argv: - os.kill(os.getppid(), signal.SIGKILL) -sys.exit(result) -'''.replace("@PATCH@", repr(real_patch))) - wrapper.chmod(0o755) - self.configure(options=["-DCMAKE_PROGRAM_PATH=" + str(self.project)]) - self.compile(False) - self.assertEqual((self.source / "value").read_text(), "beta\n") - self.assertIn("interrupted", self.compile(False)) - self.command([ARGS.cmake, "--install", str(self.build)], False) - - def test_build_and_install_failures_can_resume(self): - for phase in ["build", "install"]: - with self.subTest(phase=phase): - self.create_project() - failure = self.project / ("fail-" + phase) - failure.touch() - self.configure() - self.compile(False) - self.command([ARGS.cmake, "--install", str(self.build)], False) - self.assertFalse((self.install / "built-value").exists()) - failure.unlink() - self.compile() - self.command([ARGS.cmake, "--install", str(self.build)]) - self.assertEqual((self.install / "built-value").read_text(), "gamma\n") - - def test_source_damage_cannot_bypass_guard(self): - self.write_project(extra="externalproject_add_steptargets(dep build)") - self.built() - source = self.source / "value" - for content in ["alpha\n", None]: - with self.subTest(content=content): - if content is None: - source.unlink() - else: - source.write_text(content) - try: - self.assert_reuse_rejected(targets=("dep", "dep-build")) - finally: - source.write_text("gamma\n") - self.compile() - - def test_install_completion_does_not_check_individual_outputs(self): - self.built() - (self.install / "built-value").unlink() - # A regular build tool can rebuild declared byproducts. The patch guard - # itself accepts a completed identity without inspecting installed files. - self.configure() - self.command([ARGS.cmake, "--install", str(self.build)]) - self.assertEqual((self.project / "events").read_text(), "build\ninstall\n") - self.assertFalse((self.install / "built-value").exists()) - - def test_damaged_completion_state_is_not_success(self): - self.built() - # Fault injection deliberately knows the state filenames. Verify the - # public build/install behavior, not the contents of a valid marker. - state = self.build / "paimon-thirdparty/dep" - for name, replacement in [("installed", None), ("installed", "outdated identity"), - ("patched", None)]: - with self.subTest(marker=name, replacement=replacement): - marker = state / name - original = marker.read_bytes() - if replacement is None: - marker.unlink() - else: - marker.write_text(replacement) - try: - self.assert_reuse_rejected() - finally: - marker.write_bytes(original) - - def test_legacy_artifacts_are_not_adopted_or_deleted(self): - for kind in ["source", "install", "stamp"]: - with self.subTest(kind=kind): - self.create_project() - artifact = {"source": self.source / "value", - "install": self.install / "obsolete.h", - "stamp": self.stamp / "dep-install"}[kind] - artifact.parent.mkdir(parents=True) - artifact.write_text("old data\n") - if kind == "source": - (self.source / ".patched").touch() - self.assert_cleanup(self.configure(False)) - self.assertEqual(artifact.read_text(), "old data\n") - - def test_file_addition_and_deletion(self): - (self.project / "two.diff").write_text('''diff --git a/other b/other ---- a/other -+++ /dev/null -@@ -1 +0,0 @@ --unchanged -diff --git a/added b/added ---- /dev/null -+++ b/added -@@ -0,0 +1 @@ -+created -''') - self.configure() - self.compile() - self.compile() - self.assertFalse((self.source / "other").exists()) - self.assertEqual((self.source / "added").read_text(), "created\n") - (self.source / "other").write_text("old\n") - self.assertIn("Deleted patch target reappeared", self.compile(False)) - - def test_layout_migration_reports_old_paths(self): - layouts = { - "orc": '''set(dest "${CMAKE_CURRENT_BINARY_DIR}/dep-prefix") -set(extra_options SOURCE_DIR "${dest}/cpp" BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/build/dep" - UPDATE_DISCONNECTED TRUE)''', - "cargo": '''set(extra_options BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/dep-cargo")''', - } - for name, layout in layouts.items(): - with self.subTest(layout=name): - self.create_project() - self.write_project(layout=layout) - if name == "orc": - self.install = self.build / "dep-prefix" - self.source = self.install / "cpp" - old_binary = self.build / ("build/dep" if name == "orc" else "dep-cargo") - self.built() - old_source, old_install = self.source, self.install - self.write_project() - self.source = self.build / "dep-prefix/src/dep" - self.install = self.build / "dep-install" - # Repeated failed configuration must retain the original advice. - for _ in range(2): - output = self.configure(False) - self.assert_cleanup(output) - for path in [old_source, old_install, old_binary]: - self.assertIn(str(path), output) - self.assertEqual((old_install / "built-value").read_text(), "gamma\n") - self.clean_reported_paths(output) - self.built() - - def test_version_file_is_relative_to_module_in_subproject(self): - parent = self.root / "parent" - parent.mkdir() - (parent / "third_party").mkdir() - (parent / "third_party/versions.txt").write_text("unrelated parent versions\n") - (parent / "CMakeLists.txt").write_text('''cmake_minimum_required(VERSION 3.22) -project(Parent NONE) -add_subdirectory("{}" dependency) -'''.format(self.project)) - self.command([ARGS.cmake, "-S", str(parent), "-B", str(self.build), - "-G", ARGS.generator]) - self.compile() - self.install = self.build / "dependency/dep-install" - self.source = self.build / "dependency/dep-prefix/src/dep" - self.stamp = self.build / "dependency/dep-prefix/src/dep-stamp" - self.assertEqual((self.install / "built-value").read_text(), "gamma\n") - self.write_versions(version="2") - self.assert_cleanup(self.compile(False)) - - def test_vortex_build_uses_standard_directory_and_source_working_directory(self): - # Exercise the production adapter with Cargo simulated, without requiring - # a Rust toolchain or building the Vortex workspace in this offline suite. - toolchain = (MODULES / "ThirdpartyToolchain.cmake").read_text() - macro = "macro(build_vortex)" + toolchain.split("macro(build_vortex)", 1)[1] - macro = macro.split("endmacro()", 1)[0] + "endmacro()\n" - (self.project / "vortex-adapter.cmake").write_text(macro) - (self.package / "vortex-ffi/cinclude").mkdir(parents=True) - (self.package / "vortex-ffi/src").mkdir() - (self.package / "vortex-ffi/cinclude/vortex.h").write_text("header\n") - (self.package / ".cargo").mkdir() - (self.package / ".cargo/config.toml").write_text("# workspace config\n") - self.make_archive() - (self.project / "third_party/versions.txt").write_text( - "PAIMON_VORTEX_BUILD_VERSION=1\nPAIMON_VORTEX_BUILD_SHA256_CHECKSUM={}\n".format(self.sha)) - shutil.copyfile(str(self.project / "one.diff"), str(self.project / "vortex.diff")) - callback_dir = self.project / "crates/vortex_callback_io" - callback_dir.mkdir(parents=True) - (callback_dir / "callback_io.rs").write_text("callback\n") - cargo = self.project / "cargo" - cargo.write_text('''#!/usr/bin/env python3 -import os -from pathlib import Path -assert Path('.cargo/config.toml').is_file() -assert Path('value').read_text() == 'beta\\n' -assert Path('vortex-ffi/src/callback_io.rs').read_text() == 'callback\\n' -target = Path(os.environ['CARGO_TARGET_DIR']) -(target / 'release').mkdir(parents=True, exist_ok=True) -(target / 'release/libvortex_ffi.a').write_text('simulated library\\n') -(target / 'working-directory').write_text(str(Path.cwd())) -''') - cargo.chmod(0o755) - (self.project / "CMakeLists.txt").write_text('''cmake_minimum_required(VERSION 3.22) -project(VortexAdapter NONE) -if(POLICY CMP0135) - cmake_policy(SET CMP0135 NEW) -endif() -include("${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules/ThirdpartyPatch.cmake") -include("${CMAKE_CURRENT_SOURCE_DIR}/vortex-adapter.cmake") -set(PAIMON_CARGO_EXECUTABLE "${CMAKE_CURRENT_SOURCE_DIR}/cargo") -set(VORTEX_FLATC "${CMAKE_COMMAND}") -set(PAIMON_VORTEX_BUILD_VERSION 1) -file(SHA256 "${CMAKE_CURRENT_SOURCE_DIR}/dep.tar.gz" PAIMON_VORTEX_BUILD_SHA256_CHECKSUM) -set(VORTEX_SOURCE_URL "${CMAKE_CURRENT_SOURCE_DIR}/dep.tar.gz") -set(CMAKE_STATIC_LIBRARY_PREFIX lib) -set(CMAKE_STATIC_LIBRARY_SUFFIX .a) -set(CMAKE_INSTALL_LIBDIR lib) -add_library(Threads::Threads INTERFACE IMPORTED) -build_vortex() -''') - self.configure() - self.compile(target="vortex_ep") - binary = self.build / "vortex_ep-prefix/src/vortex_ep-build" - source = self.build / "vortex_ep-prefix/src/vortex_ep" - self.assertEqual((binary / "working-directory").read_text(), str(source)) - installed = self.build / "vortex_ep-install/lib/libvortex_ffi.a" - self.assertEqual(installed.read_text(), "simulated library\n") - timestamp = installed.stat().st_mtime_ns - self.compile(target="vortex_ep") - self.assertEqual(installed.stat().st_mtime_ns, timestamp) - self.assertFalse((self.build / "vortex_ep-cargo").exists()) - - def test_clean_rebuild_after_version_change(self): - self.built() - self.write_versions(version="2") - output = self.configure(False) - self.assert_cleanup(output) - self.clean_reported_paths(output) - self.built() - - def test_empty_patch_list(self): - self.write_project(patches="") - self.configure() - self.compile() - self.compile() - self.assertEqual((self.install / "built-value").read_text(), "alpha\n") - - def test_escaping_patch_path(self): - for name in ["../outside", "file[", "semi;colon", "back\\slash"]: - with self.subTest(path=name): - self.create_project() - (self.project / "one.diff").write_text(patch("alpha", "beta", name)) - self.configure() - self.assertIn("Unsafe patch path", self.compile(False)) - - -if __name__ == "__main__": - unittest.main(argv=[__file__] + UNITTEST_ARGS, verbosity=2) From f4bc1322bed0adbaee1fd6eb283355eee6d4a372 Mon Sep 17 00:00:00 2001 From: "jinli.zjw" Date: Tue, 17 Nov 2026 10:27:33 +0800 Subject: [PATCH 5/6] build(deps): retain the v1 patch fingerprint identifier --- cmake_modules/ApplyThirdpartyPatch.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake_modules/ApplyThirdpartyPatch.cmake b/cmake_modules/ApplyThirdpartyPatch.cmake index b45419690..4666f4618 100644 --- a/cmake_modules/ApplyThirdpartyPatch.cmake +++ b/cmake_modules/ApplyThirdpartyPatch.cmake @@ -37,7 +37,7 @@ function(paimon_patch_fingerprint output) endforeach() # The same ordered list is used here and by the application loop below. set(identity - "paimon-patch-v2\n${PP_NAME}\n${PP_VERSION}\n${PP_ARCHIVE}\n${PP_URLS}\n${PP_SOURCE}\n${PP_BINARY}\n${PP_INSTALL}\n${PP_CONTEXT}\n" + "paimon-patch-v1\n${PP_NAME}\n${PP_VERSION}\n${PP_ARCHIVE}\n${PP_URLS}\n${PP_SOURCE}\n${PP_BINARY}\n${PP_INSTALL}\n${PP_CONTEXT}\n" ) file(SHA256 "${CMAKE_CURRENT_FUNCTION_LIST_FILE}" runner_hash) string(APPEND identity From 8e095c895d29136f53ef5cfbdcf3f79637c449ef Mon Sep 17 00:00:00 2001 From: "jinli.zjw" Date: Tue, 17 Nov 2026 11:47:03 +0800 Subject: [PATCH 6/6] build(deps): align Rust dependency build directories --- cmake_modules/ThirdpartyToolchain.cmake | 6 ++++-- crates/vortex_callback_io/README.md | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake index 4536e8d65..4b78d0e74 100644 --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake @@ -1415,7 +1415,8 @@ macro(build_mosaic) set(MOSAIC_DYNAMIC_LIB "${MOSAIC_LIB_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}paimon_mosaic_ffi${CMAKE_SHARED_LIBRARY_SUFFIX}" ) - set(MOSAIC_CARGO_TARGET_DIR "${CMAKE_CURRENT_BINARY_DIR}/mosaic_ep-cargo") + set(MOSAIC_CARGO_TARGET_DIR + "${CMAKE_CURRENT_BINARY_DIR}/mosaic_ep-prefix/src/mosaic_ep-build") set(MOSAIC_CARGO_DYNAMIC_LIB "${MOSAIC_CARGO_TARGET_DIR}/release/${CMAKE_SHARED_LIBRARY_PREFIX}paimon_mosaic_ffi${CMAKE_SHARED_LIBRARY_SUFFIX}" ) @@ -1459,7 +1460,8 @@ macro(build_full_text) set(FULL_TEXT_DYNAMIC_LIB "${FULL_TEXT_LIB_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}paimon_ftindex_ffi${CMAKE_SHARED_LIBRARY_SUFFIX}" ) - set(FULL_TEXT_CARGO_TARGET_DIR "${CMAKE_CURRENT_BINARY_DIR}/full_text_ep-cargo") + set(FULL_TEXT_CARGO_TARGET_DIR + "${CMAKE_CURRENT_BINARY_DIR}/full_text_ep-prefix/src/full_text_ep-build") set(FULL_TEXT_CARGO_DYNAMIC_LIB "${FULL_TEXT_CARGO_TARGET_DIR}/release/${CMAKE_SHARED_LIBRARY_PREFIX}paimon_ftindex_ffi${CMAKE_SHARED_LIBRARY_SUFFIX}" ) diff --git a/crates/vortex_callback_io/README.md b/crates/vortex_callback_io/README.md index 2e92064b8..9a07c2eb7 100644 --- a/crates/vortex_callback_io/README.md +++ b/crates/vortex_callback_io/README.md @@ -55,9 +55,9 @@ build directory (`build-clang` below), the source directory and cargo cache are cmake --build build-clang --target vortex_ep # Run this module's Rust tests directly. -cd "$(sed -n 's/^source_dir=//p' \ - build-clang/vortex_ep-prefix/src/vortex_ep-stamp/vortex_ep-source_dirinfo.txt)" -FLATC= CARGO_TARGET_DIR=/build-clang/vortex_ep-cargo \ +cd build-clang/vortex_ep-prefix/src/vortex_ep +FLATC= \ + CARGO_TARGET_DIR=/build-clang/vortex_ep-prefix/src/vortex_ep-build \ cargo test --locked --release -p vortex-ffi callback_io ```