From 0b4e94616494cdae883daf27937ea8c43dadcb40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=83=A1=E7=90=B3=28HuLin=29?= Date: Sat, 29 Aug 2026 17:02:35 +0800 Subject: [PATCH] [Enhancement] Migrate Paimon C++ dependency to official Apache release v0.3.0 Fixes #67264 - Update thirdparty/vars.sh to download from apache/paimon-cpp v0.3.0 instead of doris-thirdparty snapshot - Add paimon-cpp-v0.3.0-external-arrow.patch to support external Arrow integration - Update download-thirdparty.sh to apply the new patch --- thirdparty/download-thirdparty.sh | 4 +- .../paimon-cpp-v0.3.0-external-arrow.patch | 132 ++++++++++++++++++ thirdparty/vars.sh | 8 +- 3 files changed, 138 insertions(+), 6 deletions(-) create mode 100644 thirdparty/patches/paimon-cpp-v0.3.0-external-arrow.patch diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index b4d3e5b42d744c..0c813865cc6e82 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -733,8 +733,8 @@ fi if [[ " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP " ]]; then cd "${TP_SOURCE_DIR}/${PAIMON_CPP_SOURCE}" if [[ ! -f "${PATCHED_MARK}" ]]; then - if patch -p1 -N --batch --dry-run <"${TP_PATCH_DIR}/paimon-cpp-buildutils-static-deps.patch" >/dev/null 2>&1; then - patch -p1 -N --batch <"${TP_PATCH_DIR}/paimon-cpp-buildutils-static-deps.patch" + if patch -p1 -N --batch --dry-run <"${TP_PATCH_DIR}/paimon-cpp-v0.3.0-external-arrow.patch" >/dev/null 2>&1; then + patch -p1 -N --batch <"${TP_PATCH_DIR}/paimon-cpp-v0.3.0-external-arrow.patch" else echo "Skip paimon-cpp patch: already applied or not applicable for current source" fi diff --git a/thirdparty/patches/paimon-cpp-v0.3.0-external-arrow.patch b/thirdparty/patches/paimon-cpp-v0.3.0-external-arrow.patch new file mode 100644 index 00000000000000..7e0fe483bb3f26 --- /dev/null +++ b/thirdparty/patches/paimon-cpp-v0.3.0-external-arrow.patch @@ -0,0 +1,132 @@ +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -46,7 +46,9 @@ + + string(TOUPPER "${CMAKE_BUILD_TYPE}" UPPERCASE_BUILD_TYPE) + +-set(CMAKE_CXX_STANDARD 17) ++if(NOT DEFINED CMAKE_CXX_STANDARD) ++ set(CMAKE_CXX_STANDARD 17) ++endif() + set(CMAKE_CXX_STANDARD_REQUIRED ON) + set(CMAKE_CXX_EXTENSIONS OFF) + set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +--- a/cmake_modules/ThirdpartyToolchain.cmake ++++ b/cmake_modules/ThirdpartyToolchain.cmake +@@ -36,6 +36,16 @@ + + set(EP_COMMON_TOOLCHAIN "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}" + "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}") ++ ++option(PAIMON_USE_EXTERNAL_ARROW "Reuse external Arrow/Parquet instead of building arrow_ep" OFF) ++set(PAIMON_EXTERNAL_ARROW_INCLUDE_DIR "" CACHE PATH ++ "Include directory for external Arrow/Parquet headers") ++set(PAIMON_EXTERNAL_ARROW_LIB "" CACHE FILEPATH "Path to external libarrow.a") ++set(PAIMON_EXTERNAL_ARROW_DATASET_LIB "" CACHE FILEPATH "Path to external libarrow_dataset.a") ++set(PAIMON_EXTERNAL_ARROW_ACERO_LIB "" CACHE FILEPATH "Path to external libarrow_acero.a") ++set(PAIMON_EXTERNAL_PARQUET_LIB "" CACHE FILEPATH "Path to external libparquet.a") ++set(PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB "" CACHE FILEPATH ++ "Path to external libarrow_bundled_dependencies.a") + + macro(set_urls URLS) + set(${URLS} ${ARGN}) +@@ -1528,8 +1538,91 @@ + endmacro() + + macro(build_arrow) +- message(STATUS "Building Arrow from source") ++ if(PAIMON_USE_EXTERNAL_ARROW) ++ set(ARROW_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/doris_external_arrow_include") ++ file(MAKE_DIRECTORY "${ARROW_INCLUDE_DIR}") ++ if(NOT EXISTS "${ARROW_INCLUDE_DIR}/arrow") ++ execute_process(COMMAND "${CMAKE_COMMAND}" -E create_symlink ++ "${PAIMON_EXTERNAL_ARROW_INCLUDE_DIR}/arrow" ++ "${ARROW_INCLUDE_DIR}/arrow") ++ endif() ++ if(EXISTS "${PAIMON_EXTERNAL_ARROW_INCLUDE_DIR}/parquet" ++ AND NOT EXISTS "${ARROW_INCLUDE_DIR}/parquet") ++ execute_process(COMMAND "${CMAKE_COMMAND}" -E create_symlink ++ "${PAIMON_EXTERNAL_ARROW_INCLUDE_DIR}/parquet" ++ "${ARROW_INCLUDE_DIR}/parquet") ++ endif() ++ ++ if(NOT PAIMON_EXTERNAL_ARROW_INCLUDE_DIR) ++ message(FATAL_ERROR ++ "PAIMON_EXTERNAL_ARROW_INCLUDE_DIR must be set when PAIMON_USE_EXTERNAL_ARROW=ON" ++ ) ++ endif() ++ if(NOT EXISTS "${PAIMON_EXTERNAL_ARROW_INCLUDE_DIR}") ++ message(FATAL_ERROR ++ "PAIMON_EXTERNAL_ARROW_INCLUDE_DIR not found: ${PAIMON_EXTERNAL_ARROW_INCLUDE_DIR}" ++ ) ++ endif() ++ ++ foreach(_paimon_external_lib ++ IN ITEMS PAIMON_EXTERNAL_ARROW_LIB ++ PAIMON_EXTERNAL_ARROW_DATASET_LIB ++ PAIMON_EXTERNAL_ARROW_ACERO_LIB ++ PAIMON_EXTERNAL_PARQUET_LIB ++ PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB) ++ if(NOT ${_paimon_external_lib}) ++ message(FATAL_ERROR ++ "${_paimon_external_lib} must be set when PAIMON_USE_EXTERNAL_ARROW=ON") ++ endif() ++ if(NOT EXISTS "${${_paimon_external_lib}}") ++ message(FATAL_ERROR ++ "${_paimon_external_lib} not found: ${${_paimon_external_lib}}") ++ endif() ++ endforeach() ++ ++ add_library(arrow STATIC IMPORTED) ++ set_target_properties(arrow ++ PROPERTIES IMPORTED_LOCATION "${PAIMON_EXTERNAL_ARROW_LIB}" ++ INTERFACE_INCLUDE_DIRECTORIES ++ "${ARROW_INCLUDE_DIR}") ++ ++ add_library(arrow_dataset STATIC IMPORTED) ++ set_target_properties(arrow_dataset ++ PROPERTIES IMPORTED_LOCATION ++ "${PAIMON_EXTERNAL_ARROW_DATASET_LIB}" ++ INTERFACE_INCLUDE_DIRECTORIES ++ "${ARROW_INCLUDE_DIR}") ++ ++ add_library(arrow_acero STATIC IMPORTED) ++ set_target_properties(arrow_acero ++ PROPERTIES IMPORTED_LOCATION ++ "${PAIMON_EXTERNAL_ARROW_ACERO_LIB}" ++ INTERFACE_INCLUDE_DIRECTORIES ++ "${ARROW_INCLUDE_DIR}") ++ ++ add_library(parquet STATIC IMPORTED) ++ set_target_properties(parquet ++ PROPERTIES IMPORTED_LOCATION "${PAIMON_EXTERNAL_PARQUET_LIB}" ++ INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}") ++ ++ add_library(arrow_bundled_dependencies STATIC IMPORTED) ++ set_target_properties(arrow_bundled_dependencies ++ PROPERTIES IMPORTED_LOCATION ++ "${PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB}" ++ INTERFACE_INCLUDE_DIRECTORIES ++ "${ARROW_INCLUDE_DIR}") ++ ++ target_link_libraries(arrow_acero INTERFACE arrow) ++ target_link_libraries(arrow_dataset INTERFACE arrow_acero) ++ target_link_libraries(arrow ++ INTERFACE zstd snappy lz4 zlib ++ arrow_bundled_dependencies) ++ target_link_libraries(parquet ++ INTERFACE zstd snappy lz4 zlib ++ arrow_bundled_dependencies arrow_dataset) ++ else() ++ message(STATUS "Building Arrow from source") ++ + get_target_property(ARROW_SNAPPY_INCLUDE_DIR snappy INTERFACE_INCLUDE_DIRECTORIES) + get_filename_component(ARROW_SNAPPY_ROOT "${ARROW_SNAPPY_INCLUDE_DIR}" DIRECTORY) + +@@ -1697,6 +1790,7 @@ + zlib + arrow_bundled_dependencies + arrow_dataset) ++ endif() + + endmacro(build_arrow) diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index f8d18a3a2bea16..dc8001781b3398 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -568,10 +568,10 @@ PUGIXML_SOURCE=pugixml-1.15 PUGIXML_MD5SUM="3b894c29455eb33a40b165c6e2de5895" # paimon-cpp -PAIMON_CPP_DOWNLOAD="https://github.com/apache/doris-thirdparty/archive/refs/tags/paimon-cpp-0a4f4e2.tar.gz" -PAIMON_CPP_NAME="paimon-cpp-0a4f4e2.tar.gz" -PAIMON_CPP_SOURCE="doris-thirdparty-paimon-cpp-0a4f4e2" -PAIMON_CPP_MD5SUM="b8599a0421dbf1ec05e2f1a481d64e87" +PAIMON_CPP_DOWNLOAD="https://github.com/apache/paimon-cpp/archive/refs/tags/v0.3.0.tar.gz" +PAIMON_CPP_NAME="paimon-cpp-0.3.0.tar.gz" +PAIMON_CPP_SOURCE="paimon-cpp-0.3.0" +PAIMON_CPP_MD5SUM="313f4f9fee7ccc92428c83dc50cab9f6" # lance-c LANCE_C_DOWNLOAD="https://github.com/lance-format/lance-c/archive/refs/tags/v0.1.2.tar.gz"