Skip to content

[Enhancement] Migrate Paimon C++ dependency to official Apache release v0.3.0 #67304

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
hulincup wants to merge 1 commit into
base: master
Choose a base branch
from
+138 −6
Open

[Enhancement] Migrate Paimon C++ dependency to official Apache release v0.3.0 #67304

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions thirdparty/download-thirdparty.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -733,8 +733,8 @@ fi
if [[ " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP " ]]; then
cd "${TP_SOURCE_DIR}/${PAIMON_CPP_SOURCE}"
if [[ ! -f "${PATCHED_MARK}" ]]; then
if patch -p1 -N --batch --dry-run <"${TP_PATCH_DIR}/paimon-cpp-buildutils-static-deps.patch" >/dev/null 2>&1; then
patch -p1 -N --batch <"${TP_PATCH_DIR}/paimon-cpp-buildutils-static-deps.patch"
if patch -p1 -N --batch --dry-run <"${TP_PATCH_DIR}/paimon-cpp-v0.3.0-external-arrow.patch" >/dev/null 2>&1; then
patch -p1 -N --batch <"${TP_PATCH_DIR}/paimon-cpp-v0.3.0-external-arrow.patch"
else
echo "Skip paimon-cpp patch: already applied or not applicable for current source"
fi
Expand Down
132 changes: 132 additions & 0 deletions thirdparty/patches/paimon-cpp-v0.3.0-external-arrow.patch
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -46,7 +46,9 @@

string(TOUPPER "${CMAKE_BUILD_TYPE}" UPPERCASE_BUILD_TYPE)

-set(CMAKE_CXX_STANDARD 17)
+if(NOT DEFINED CMAKE_CXX_STANDARD)
+ set(CMAKE_CXX_STANDARD 17)
+endif()
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_EXTENSIONS OFF)
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
--- a/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cmake_modules/ThirdpartyToolchain.cmake
@@ -36,6 +36,16 @@

set(EP_COMMON_TOOLCHAIN "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}"
"-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}")
+
+option(PAIMON_USE_EXTERNAL_ARROW "Reuse external Arrow/Parquet instead of building arrow_ep" OFF)
+set(PAIMON_EXTERNAL_ARROW_INCLUDE_DIR "" CACHE PATH
+ "Include directory for external Arrow/Parquet headers")
+set(PAIMON_EXTERNAL_ARROW_LIB "" CACHE FILEPATH "Path to external libarrow.a")
+set(PAIMON_EXTERNAL_ARROW_DATASET_LIB "" CACHE FILEPATH "Path to external libarrow_dataset.a")
+set(PAIMON_EXTERNAL_ARROW_ACERO_LIB "" CACHE FILEPATH "Path to external libarrow_acero.a")
+set(PAIMON_EXTERNAL_PARQUET_LIB "" CACHE FILEPATH "Path to external libparquet.a")
+set(PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB "" CACHE FILEPATH
+ "Path to external libarrow_bundled_dependencies.a")

macro(set_urls URLS)
set(${URLS} ${ARGN})
@@ -1528,8 +1538,91 @@
endmacro()

macro(build_arrow)
- message(STATUS "Building Arrow from source")
+ if(PAIMON_USE_EXTERNAL_ARROW)
+ set(ARROW_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/doris_external_arrow_include")
+ file(MAKE_DIRECTORY "${ARROW_INCLUDE_DIR}")
+ if(NOT EXISTS "${ARROW_INCLUDE_DIR}/arrow")
+ execute_process(COMMAND "${CMAKE_COMMAND}" -E create_symlink
+ "${PAIMON_EXTERNAL_ARROW_INCLUDE_DIR}/arrow"
+ "${ARROW_INCLUDE_DIR}/arrow")
+ endif()
+ if(EXISTS "${PAIMON_EXTERNAL_ARROW_INCLUDE_DIR}/parquet"
+ AND NOT EXISTS "${ARROW_INCLUDE_DIR}/parquet")
+ execute_process(COMMAND "${CMAKE_COMMAND}" -E create_symlink
+ "${PAIMON_EXTERNAL_ARROW_INCLUDE_DIR}/parquet"
+ "${ARROW_INCLUDE_DIR}/parquet")
+ endif()
+
+ if(NOT PAIMON_EXTERNAL_ARROW_INCLUDE_DIR)
+ message(FATAL_ERROR
+ "PAIMON_EXTERNAL_ARROW_INCLUDE_DIR must be set when PAIMON_USE_EXTERNAL_ARROW=ON"
+ )
+ endif()
+ if(NOT EXISTS "${PAIMON_EXTERNAL_ARROW_INCLUDE_DIR}")
+ message(FATAL_ERROR
+ "PAIMON_EXTERNAL_ARROW_INCLUDE_DIR not found: ${PAIMON_EXTERNAL_ARROW_INCLUDE_DIR}"
+ )
+ endif()
+
+ foreach(_paimon_external_lib
+ IN ITEMS PAIMON_EXTERNAL_ARROW_LIB
+ PAIMON_EXTERNAL_ARROW_DATASET_LIB
+ PAIMON_EXTERNAL_ARROW_ACERO_LIB
+ PAIMON_EXTERNAL_PARQUET_LIB
+ PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB)
+ if(NOT ${_paimon_external_lib})
+ message(FATAL_ERROR
+ "${_paimon_external_lib} must be set when PAIMON_USE_EXTERNAL_ARROW=ON")
+ endif()
+ if(NOT EXISTS "${${_paimon_external_lib}}")
+ message(FATAL_ERROR
+ "${_paimon_external_lib} not found: ${${_paimon_external_lib}}")
+ endif()
+ endforeach()
+
+ add_library(arrow STATIC IMPORTED)
+ set_target_properties(arrow
+ PROPERTIES IMPORTED_LOCATION "${PAIMON_EXTERNAL_ARROW_LIB}"
+ INTERFACE_INCLUDE_DIRECTORIES
+ "${ARROW_INCLUDE_DIR}")
+
+ add_library(arrow_dataset STATIC IMPORTED)
+ set_target_properties(arrow_dataset
+ PROPERTIES IMPORTED_LOCATION
+ "${PAIMON_EXTERNAL_ARROW_DATASET_LIB}"
+ INTERFACE_INCLUDE_DIRECTORIES
+ "${ARROW_INCLUDE_DIR}")
+
+ add_library(arrow_acero STATIC IMPORTED)
+ set_target_properties(arrow_acero
+ PROPERTIES IMPORTED_LOCATION
+ "${PAIMON_EXTERNAL_ARROW_ACERO_LIB}"
+ INTERFACE_INCLUDE_DIRECTORIES
+ "${ARROW_INCLUDE_DIR}")
+
+ add_library(parquet STATIC IMPORTED)
+ set_target_properties(parquet
+ PROPERTIES IMPORTED_LOCATION "${PAIMON_EXTERNAL_PARQUET_LIB}"
+ INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}")
+
+ add_library(arrow_bundled_dependencies STATIC IMPORTED)
+ set_target_properties(arrow_bundled_dependencies
+ PROPERTIES IMPORTED_LOCATION
+ "${PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB}"
+ INTERFACE_INCLUDE_DIRECTORIES
+ "${ARROW_INCLUDE_DIR}")
+
+ target_link_libraries(arrow_acero INTERFACE arrow)
+ target_link_libraries(arrow_dataset INTERFACE arrow_acero)
+ target_link_libraries(arrow
+ INTERFACE zstd snappy lz4 zlib
+ arrow_bundled_dependencies)
+ target_link_libraries(parquet
+ INTERFACE zstd snappy lz4 zlib
+ arrow_bundled_dependencies arrow_dataset)
+ else()
+ message(STATUS "Building Arrow from source")
+
get_target_property(ARROW_SNAPPY_INCLUDE_DIR snappy INTERFACE_INCLUDE_DIRECTORIES)
get_filename_component(ARROW_SNAPPY_ROOT "${ARROW_SNAPPY_INCLUDE_DIR}" DIRECTORY)

@@ -1697,6 +1790,7 @@
zlib
arrow_bundled_dependencies
arrow_dataset)
+ endif()

endmacro(build_arrow)
8 changes: 4 additions & 4 deletions thirdparty/vars.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -568,10 +568,10 @@ PUGIXML_SOURCE=pugixml-1.15
PUGIXML_MD5SUM="3b894c29455eb33a40b165c6e2de5895"

# paimon-cpp
PAIMON_CPP_DOWNLOAD="https://github.com/apache/doris-thirdparty/archive/refs/tags/paimon-cpp-0a4f4e2.tar.gz"
PAIMON_CPP_NAME="paimon-cpp-0a4f4e2.tar.gz"
PAIMON_CPP_SOURCE="doris-thirdparty-paimon-cpp-0a4f4e2"
PAIMON_CPP_MD5SUM="b8599a0421dbf1ec05e2f1a481d64e87"
PAIMON_CPP_DOWNLOAD="https://github.com/apache/paimon-cpp/archive/refs/tags/v0.3.0.tar.gz"
PAIMON_CPP_NAME="paimon-cpp-0.3.0.tar.gz"
PAIMON_CPP_SOURCE="paimon-cpp-0.3.0"
PAIMON_CPP_MD5SUM="313f4f9fee7ccc92428c83dc50cab9f6"

# lance-c
LANCE_C_DOWNLOAD="https://github.com/lance-format/lance-c/archive/refs/tags/v0.1.2.tar.gz"
Expand Down