diff --git a/.github/workflows/build-thirdparty.yml b/.github/workflows/build-thirdparty.yml index 7ae7cb2d43ed1b..03244548080032 100644 --- a/.github/workflows/build-thirdparty.yml +++ b/.github/workflows/build-thirdparty.yml @@ -58,7 +58,10 @@ jobs: - 'thirdparty/paimon-cpp-cache.cmake' - 'thirdparty/patches/apache-arrow-*.patch' - 'thirdparty/patches/paimon-cpp-*.patch' + - 'regression-test/pipeline/external/conf/fe.conf' - 'thirdparty/test/arrow-paimon-lifecycle-test.sh' + - 'thirdparty/test/adbc-jni-config-test.sh' + - 'thirdparty/test/paimon-codec-dependency-test.sh' - '.github/workflows/build-thirdparty.yml' arrow_paimon_lifecycle_test: @@ -71,7 +74,10 @@ jobs: uses: actions/checkout@v4 - name: Test focused thirdparty lifecycle - run: thirdparty/test/arrow-paimon-lifecycle-test.sh + run: | + thirdparty/test/arrow-paimon-lifecycle-test.sh + thirdparty/test/adbc-jni-config-test.sh + thirdparty/test/paimon-codec-dependency-test.sh script_test: name: Thirdparty Script Test diff --git a/regression-test/pipeline/external/conf/fe.conf b/regression-test/pipeline/external/conf/fe.conf index d5c3e734433745..4466998ae67163 100644 --- a/regression-test/pipeline/external/conf/fe.conf +++ b/regression-test/pipeline/external/conf/fe.conf @@ -28,7 +28,7 @@ DATE = `date +%Y%m%d-%H%M%S` JAVA_OPTS="-Xmx4096m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$DORIS_HOME/log/fe.jmap -XX:+UseMembar -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintClassHistogramAfterFullGC -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xloggc:$DORIS_HOME/log/fe.gc.log.$DATE -Dcom.mysql.cj.disableAbandonedConnectionCleanup=true" # For jdk 17+, this JAVA_OPTS will be used as default JVM options -# Use Doris' compatible ADBC JNI build because the Maven-bundled library requires a newer glibc. +# External deployment replaces the packaged fe.conf, so keep Doris' glibc-compatible ADBC JNI build. JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8 -Djavax.security.auth.useSubjectCredsOnly=false -Xmx4096m -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$LOG_DIR -Xlog:gc*,classhisto*=trace:$LOG_DIR/fe.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M -Darrow.enable_null_check_for_get=false -Darrow.adbc.driver.jni.library.path=${DORIS_HOME}/lib --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.management/sun.management=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.xml/com.sun.org.apache.xerces.internal.jaxp=ALL-UNNAMED" ## diff --git a/thirdparty/arrow-paimon-vars.sh b/thirdparty/arrow-paimon-vars.sh index e6f8cfb7c6f27c..d532a740e63d68 100644 --- a/thirdparty/arrow-paimon-vars.sh +++ b/thirdparty/arrow-paimon-vars.sh @@ -61,7 +61,9 @@ PAIMON_BUILD_SCHEMA_VERSION="1" # mandatory before reuse. Delete the lists once master has published a prebuilt that # carries the semantic marker itself. ARROW_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT="ab79ab0bbfbf93f9860050fb751b20fee9e40d96" -PAIMON_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT="cb82e41ba46f534e611cdd52e66b53c227d49bf8" +# Dependency ordering does not change the installed Paimon artifacts, so existing +# complete prebuilts remain compatible with the corrected clean-build graph. +PAIMON_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT="9887cf1ec13106da4250f8d7ea931a258879089c" ARROW_LEGACY_BUILD_FINGERPRINTS=( # published 2026-08-19, master after apache/doris#66842 8356a3a464b93f3a59a6bdae6a51258947ab35d2 diff --git a/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch b/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch index fc1fe1553126b9..213cc279f52e5c 100644 --- a/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch +++ b/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch @@ -343,3 +343,20 @@ diff --git a/src/paimon/format/blob/blob_format_writer.cpp b/src/paimon/format/b + list(GET Thrift_VERSION_LIST 0 Thrift_VERSION_MAJOR) + list(GET Thrift_VERSION_LIST 1 Thrift_VERSION_MINOR) + list(GET Thrift_VERSION_LIST 2 Thrift_VERSION_PATCH) + +diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt +index 76dd03b..9317324 100644 +--- a/src/paimon/CMakeLists.txt ++++ b/src/paimon/CMakeLists.txt +@@ -281,6 +281,11 @@ add_paimon_lib(paimon + ${PAIMON_COMMON_SRCS} + ${PAIMON_CORE_SRCS} + DEPENDENCIES ++ # Codec headers come from ExternalProject install steps, so object ++ # compilation must not race those steps in clean parallel builds. ++ zstd ++ snappy ++ lz4 + arrow + tbb + glog diff --git a/thirdparty/test/adbc-jni-config-test.sh b/thirdparty/test/adbc-jni-config-test.sh new file mode 100755 index 00000000000000..6f2cf074e3cb76 --- /dev/null +++ b/thirdparty/test/adbc-jni-config-test.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." &>/dev/null && pwd)" +EXTERNAL_FE_CONF="${ROOT}/regression-test/pipeline/external/conf/fe.conf" +# shellcheck disable=SC2016 # Match the literal option before fe.conf expands DORIS_HOME. +JNI_LIBRARY_OPTION='-Darrow.adbc.driver.jni.library.path=${DORIS_HOME}/lib' + +# External Regression replaces the packaged FE configuration instead of merging it. +# Keep the JNI override in that replacement so ADBC cannot fall back to the jar binary. +java_opts_assignment="$(grep '^JAVA_OPTS_FOR_JDK_17=' "${EXTERNAL_FE_CONF}" || true)" +if [[ " ${java_opts_assignment} " != *" ${JNI_LIBRARY_OPTION} "* ]]; then + echo "FAIL: ${EXTERNAL_FE_CONF} drops the packaged ADBC JNI library path." >&2 + echo "Add ${JNI_LIBRARY_OPTION} to its FE JVM options." >&2 + exit 1 +fi + +echo "PASS" diff --git a/thirdparty/test/paimon-codec-dependency-test.sh b/thirdparty/test/paimon-codec-dependency-test.sh new file mode 100755 index 00000000000000..4d3b92458b8c26 --- /dev/null +++ b/thirdparty/test/paimon-codec-dependency-test.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." &>/dev/null && pwd)" + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +tmpdir="$(mktemp -d)" +trap 'rm -rf "${tmpdir}"' EXIT + +fixture="${tmpdir}/paimon-codec-dependency" +focused_patch="${fixture}/codec-dependency.patch" +mkdir -p "${fixture}/src/paimon" +{ + printf '%s\n' 'add_paimon_lib(paimon' + printf '%s\n' ' SOURCES' + printf '%s\n' ' ${PAIMON_COMMON_SRCS}' + printf '%s\n' ' ${PAIMON_CORE_SRCS}' + printf '%s\n' ' DEPENDENCIES' + printf '%s\n' ' arrow' + printf '%s\n' ' tbb' + printf '%s\n' ' glog' + printf '%s\n' ' fmt' + printf '%s\n' ' roaring_bitmap' + printf '%s\n' ' xxhash' + printf '%s\n' ' Threads::Threads' + printf '%s\n' ' RapidJSON' + printf '%s\n' ' STATIC_LINK_LIBS' + printf '%s\n' ' arrow)' +} >"${fixture}/src/paimon/CMakeLists.txt" + +awk ' + /^diff --git / { + if (found) { + exit + } + if ($0 == "diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt") { + found = 1 + } + } + found { print } +' "${ROOT}/patches/paimon-cpp-buildutils-static-deps.patch" >"${focused_patch}" +[[ -s "${focused_patch}" ]] || fail "the Paimon codec dependency patch is missing" +( + cd "${fixture}" + patch -s -p1 <"${focused_patch}" +) || fail "the Paimon codec dependency patch did not apply" + +dependencies="$(sed -n '/^[[:space:]]*DEPENDENCIES$/,/^[[:space:]]*STATIC_LINK_LIBS$/p' \ + "${fixture}/src/paimon/CMakeLists.txt")" +grep -Eq '^[[:space:]]+zstd$' <<<"${dependencies}" || + fail "paimon_objlib does not wait for the ZSTD headers" +grep -Eq '^[[:space:]]+snappy$' <<<"${dependencies}" || + fail "paimon_objlib does not wait for the Snappy headers" +grep -Eq '^[[:space:]]+lz4$' <<<"${dependencies}" || + fail "paimon_objlib does not wait for the LZ4 headers" + +echo "PASS"