From 5a149640c021015b609f73aa8d12c1bcc2eec02a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 23 Apr 2026 13:19:17 +0200 Subject: [PATCH 01/22] GH-49321: [C++][Python] Add pixi build for Arrow and PyArrow --- ci/docker/pixi-cpp.dockerfile | 39 ++++++++++++++++++++++ ci/pixi/.gitignore | 19 +++++++++++ ci/pixi/pixi.toml | 62 +++++++++++++++++++++++++++++++++++ compose.yaml | 29 ++++++++++++++++ dev/tasks/tasks.yml | 6 ++++ 5 files changed, 155 insertions(+) create mode 100644 ci/docker/pixi-cpp.dockerfile create mode 100644 ci/pixi/.gitignore create mode 100644 ci/pixi/pixi.toml diff --git a/ci/docker/pixi-cpp.dockerfile b/ci/docker/pixi-cpp.dockerfile new file mode 100644 index 000000000000..41efd45d10db --- /dev/null +++ b/ci/docker/pixi-cpp.dockerfile @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG arch=amd64 +FROM ${arch}/ubuntu:24.04 + +# install build essentials +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update -y -q && \ + apt-get install -y -q \ + curl \ + gdb \ + libc6-dbg \ + tzdata \ + wget && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# install pixi +RUN curl -fsSL https://pixi.sh/install.sh -o /tmp/install-pixi.sh +RUN PIXI_HOME=/opt/pixi PIXI_NO_PATH_UPDATE=1 sh /tmp/install-pixi.sh && rm /tmp/install-pixi.sh +ENV PATH=/opt/pixi/bin:$PATH + +# Verify pixi installation +RUN pixi --version \ No newline at end of file diff --git a/ci/pixi/.gitignore b/ci/pixi/.gitignore new file mode 100644 index 000000000000..a4ac42feacaa --- /dev/null +++ b/ci/pixi/.gitignore @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +**/.pixi +**/pixi.lock \ No newline at end of file diff --git a/ci/pixi/pixi.toml b/ci/pixi/pixi.toml new file mode 100644 index 000000000000..cad28b928fbc --- /dev/null +++ b/ci/pixi/pixi.toml @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[workspace] +channels = ["https://prefix.dev/conda-forge"] +platforms = ["linux-64", "linux-aarch64", "osx-arm64"] +preview = ["pixi-build"] + +[environments] +default = [] +test = ["test"] + +[feature.test.activation.env] +ARROW_TEST_DATA = "/arrow/testing/data" + +[feature.test.dependencies] +binutils = "*" +cmake = "*" +python = "*" + +[feature.test.tasks] +# This path mangling feels extremely finicky. Investigate alternatives to make this more robust. +test = "cd $(ls -d /arrow/ci/pixi/.pixi/build/work/arrow-cpp-*/work/build | head -n 1) && ctest --output-on-failure" + +[package] +name = "arrow-cpp" +# TODO: Incorporate this to the bump version script and tests +version = "25.0.0.dev" +authors = ["Apache Arrow"] + +[package.build] +source.path = "../../cpp" + +[package.build.backend] +name = "pixi-build-cmake" +version = "*" + +[package.build.config] +extra-args = ["-DCMAKE_BUILD_TYPE=Release", + "-DARROW_BUILD_TESTS=ON"] + +[package.host-dependencies] +gflags = "*" +gmock = ">=1.10.0" +gtest = ">=1.10.0" +libboost-devel = "*" +rapidjson = "*" +xsimd = ">=14.0" diff --git a/compose.yaml b/compose.yaml index c13bd831ecff..b298f93a0824 100644 --- a/compose.yaml +++ b/compose.yaml @@ -138,6 +138,8 @@ x-hierarchy: - fedora-cpp: - fedora-python - fedora-r-clang + - pixi-cpp: + - pixi-python - python-sdist - ubuntu-cpp: - ubuntu-cpp-static @@ -200,6 +202,8 @@ volumes: name: python-wheel-manylinux-2-28-ccache python-wheel-musllinux-1-2-ccache: name: python-wheel-musllinux-1-2-ccache + pixi-cpp-build: + name: ${ARCH}-pixi-cpp-build ubuntu-ccache: name: ${ARCH_SHORT}-ubuntu-${UBUNTU}-ccache @@ -1144,6 +1148,31 @@ services: /arrow/ci/scripts/cpp_build.sh /arrow /build && /arrow/ci/scripts/python_sdist_test.sh /arrow" + ############################ Pixi builds ################################## + + pixi-cpp: + # C++ build with pixi + # + # Usage: + # docker compose build pixi-cpp + # docker compose run --rm pixi-cpp + # Parameters: + # ARCH: amd64, arm32v7 + image: ${REPO}:${ARCH}-pixi + build: + context: . + dockerfile: ci/docker/pixi-cpp.dockerfile + cache_from: + - ${REPO}:${ARCH}-pixi-cpp + args: + arch: ${ARCH} + volumes: + - .:/arrow:delegated + - ${DOCKER_VOLUME_PREFIX}pixi-cpp-build:/arrow/ci/pixi/.pixi:delegated + command: > + /bin/sh -c "pixi build --path /arrow/ci/pixi && + pixi run --environment test --manifest-path /arrow/ci/pixi test" + ############################ Python wheels ################################## # See available versions at: diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index f7cb2b92addd..c5477314025d 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -364,6 +364,12 @@ tasks: params: image: conda-cpp-valgrind + test-pixi-cpp: + ci: github + template: docker-tests/github.linux.yml + params: + image: pixi-cpp + test-ubuntu-22.04-cpp: ci: github template: docker-tests/github.linux.yml From bc67422116544972931a1f8349b90f8b60d4eec9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 23 Apr 2026 13:21:31 +0200 Subject: [PATCH 02/22] Minor clean-up --- ci/docker/pixi-cpp.dockerfile | 2 +- ci/pixi/.gitignore | 2 +- compose.yaml | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/ci/docker/pixi-cpp.dockerfile b/ci/docker/pixi-cpp.dockerfile index 41efd45d10db..1976253a9676 100644 --- a/ci/docker/pixi-cpp.dockerfile +++ b/ci/docker/pixi-cpp.dockerfile @@ -36,4 +36,4 @@ RUN PIXI_HOME=/opt/pixi PIXI_NO_PATH_UPDATE=1 sh /tmp/install-pixi.sh && rm /tmp ENV PATH=/opt/pixi/bin:$PATH # Verify pixi installation -RUN pixi --version \ No newline at end of file +RUN pixi --version diff --git a/ci/pixi/.gitignore b/ci/pixi/.gitignore index a4ac42feacaa..8913651c86dc 100644 --- a/ci/pixi/.gitignore +++ b/ci/pixi/.gitignore @@ -16,4 +16,4 @@ # under the License. **/.pixi -**/pixi.lock \ No newline at end of file +**/pixi.lock diff --git a/compose.yaml b/compose.yaml index b298f93a0824..5415b4453062 100644 --- a/compose.yaml +++ b/compose.yaml @@ -138,8 +138,7 @@ x-hierarchy: - fedora-cpp: - fedora-python - fedora-r-clang - - pixi-cpp: - - pixi-python + - pixi-cpp - python-sdist - ubuntu-cpp: - ubuntu-cpp-static From 5f20e90e1bc387912ab5e3c82f2924ad9ef684dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 24 Apr 2026 14:27:49 +0200 Subject: [PATCH 03/22] Add pixi build for Python --- .../{pixi-cpp.dockerfile => pixi.dockerfile} | 0 ci/pixi/{ => cpp}/pixi.toml | 35 +++++++- ci/pixi/python/pixi.toml | 85 +++++++++++++++++++ compose.yaml | 37 ++++++-- dev/tasks/tasks.yml | 6 ++ 5 files changed, 151 insertions(+), 12 deletions(-) rename ci/docker/{pixi-cpp.dockerfile => pixi.dockerfile} (100%) rename ci/pixi/{ => cpp}/pixi.toml (72%) create mode 100644 ci/pixi/python/pixi.toml diff --git a/ci/docker/pixi-cpp.dockerfile b/ci/docker/pixi.dockerfile similarity index 100% rename from ci/docker/pixi-cpp.dockerfile rename to ci/docker/pixi.dockerfile diff --git a/ci/pixi/pixi.toml b/ci/pixi/cpp/pixi.toml similarity index 72% rename from ci/pixi/pixi.toml rename to ci/pixi/cpp/pixi.toml index cad28b928fbc..6dc4ef5809fe 100644 --- a/ci/pixi/pixi.toml +++ b/ci/pixi/cpp/pixi.toml @@ -34,7 +34,7 @@ python = "*" [feature.test.tasks] # This path mangling feels extremely finicky. Investigate alternatives to make this more robust. -test = "cd $(ls -d /arrow/ci/pixi/.pixi/build/work/arrow-cpp-*/work/build | head -n 1) && ctest --output-on-failure" +test = "cd $(ls -d /arrow/ci/pixi/cpp/.pixi/build/work/arrow-cpp-*/work/build | head -n 1) && ctest --output-on-failure" [package] name = "arrow-cpp" @@ -43,20 +43,47 @@ version = "25.0.0.dev" authors = ["Apache Arrow"] [package.build] -source.path = "../../cpp" +source.path = "../../../cpp" [package.build.backend] name = "pixi-build-cmake" version = "*" [package.build.config] -extra-args = ["-DCMAKE_BUILD_TYPE=Release", - "-DARROW_BUILD_TESTS=ON"] +extra-args = [ + "-DCMAKE_BUILD_TYPE=Release", + "-DARROW_BUILD_TESTS=ON", + "-DARROW_COMPUTE=ON", + "-DARROW_CSV=ON", + "-DARROW_DATASET=ON", + "-DARROW_FILESYSTEM=ON", + "-DARROW_JSON=ON", + "-DARROW_PARQUET=ON", + "-DARROW_WITH_BROTLI=ON", + "-DARROW_WITH_BZ2=ON", + "-DARROW_WITH_LZ4=ON", + "-DARROW_WITH_SNAPPY=ON", + "-DARROW_WITH_ZLIB=ON", + "-DARROW_WITH_ZSTD=ON", + # libutf8proc on package-host-dependencies seems to fail. + # More investigation needed. In the meantime use bundled. + "-Dutf8proc_SOURCE=BUNDLED", + "-DOPENSSL_ROOT_DIR=$PREFIX", +] [package.host-dependencies] +brotli = "*" +bzip2 = "*" gflags = "*" gmock = ">=1.10.0" gtest = ">=1.10.0" libboost-devel = "*" +lz4-c = "*" +openssl = "*" rapidjson = "*" +re2 = "*" +snappy = "*" +thrift-cpp = ">=0.11.0" xsimd = ">=14.0" +zlib = "*" +zstd = "*" diff --git a/ci/pixi/python/pixi.toml b/ci/pixi/python/pixi.toml new file mode 100644 index 000000000000..6d096882d255 --- /dev/null +++ b/ci/pixi/python/pixi.toml @@ -0,0 +1,85 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[workspace] +channels = ["https://prefix.dev/conda-forge"] +platforms = ["linux-64", "linux-aarch64", "osx-arm64"] +preview = ["pixi-build"] + +[environments] +default = [] +test = ["test"] + +[feature.test] +platforms = ["linux-64"] + +[feature.test.activation.env] +ARROW_TEST_DATA = "/arrow/testing/data" +PARQUET_TEST_DATA = "/arrow/cpp/submodules/parquet-testing/data" + +[feature.test.dependencies] +hypothesis = "*" +pyarrow = { path = "." } +pytest = "*" +pytest-timeout = "*" + +[feature.test.tasks] +test = "pytest -r s --pyargs pyarrow --timeout=60" + +[package] +name = "pyarrow" +# TODO: Incorporate this to the bump version script and tests +version = "25.0.0.dev" +authors = ["Apache Arrow"] + +[package.build] +source.path = "../../../python" + +[package.build.backend] +name = "pixi-build-python" +version = "*" + +[package.build.config] +noarch = false +compilers = ["c", "cxx"] +env = { + PYARROW_BUNDLE_ARROW_CPP = "0", + CMAKE_GENERATOR = "Ninja", + ARROW_HOME = "$PREFIX", +} + +[package.host-dependencies] +arrow-cpp = { path = "../cpp" } # source-dep on the arrow-cpp manifest +cmake = "*" +cython = ">=3.0" +gcc_linux-64 = "*" +git = "*" +gxx_linux-64 = "*" +ninja = "*" +numpy = "*" +pip = "*" +pkg-config = "*" +python = ">=3.10,<3.14" +scikit-build-core = "*" +setuptools-scm = "*" +sysroot_linux-64 = "*" +zlib = "*" + +[package.run-dependencies] +arrow-cpp = { path = "../cpp" } +python = ">=3.10,<3.14" +numpy = "*" diff --git a/compose.yaml b/compose.yaml index 5415b4453062..7b76b3c9e217 100644 --- a/compose.yaml +++ b/compose.yaml @@ -197,12 +197,14 @@ volumes: name: ${ARCH_SHORT}-fedora-${FEDORA}-ccache maven-cache: name: maven-cache + pixi-cpp-build: + name: ${ARCH}-pixi-cpp-build + pixi-python-build: + name: ${ARCH}-pixi-python-build python-wheel-manylinux-2-28-ccache: name: python-wheel-manylinux-2-28-ccache python-wheel-musllinux-1-2-ccache: name: python-wheel-musllinux-1-2-ccache - pixi-cpp-build: - name: ${ARCH}-pixi-cpp-build ubuntu-ccache: name: ${ARCH_SHORT}-ubuntu-${UBUNTU}-ccache @@ -1160,17 +1162,36 @@ services: image: ${REPO}:${ARCH}-pixi build: context: . - dockerfile: ci/docker/pixi-cpp.dockerfile - cache_from: - - ${REPO}:${ARCH}-pixi-cpp + dockerfile: ci/docker/pixi.dockerfile args: arch: ${ARCH} volumes: - .:/arrow:delegated - - ${DOCKER_VOLUME_PREFIX}pixi-cpp-build:/arrow/ci/pixi/.pixi:delegated + - ${DOCKER_VOLUME_PREFIX}pixi-cpp-build:/arrow/ci/pixi/cpp/.pixi:delegated command: > - /bin/sh -c "pixi build --path /arrow/ci/pixi && - pixi run --environment test --manifest-path /arrow/ci/pixi test" + /bin/sh -c "pixi build --path /arrow/ci/pixi/cpp && + pixi run --environment test --manifest-path /arrow/ci/pixi/cpp test" + + pixi-python: + # CPython build with pixi + # + # Usage: + # docker compose build pixi-python + # docker compose run --rm pixi-python + # Parameters: + # ARCH: amd64, arm32v7 + image: ${REPO}:${ARCH}-pixi + build: + context: . + dockerfile: ci/docker/pixi.dockerfile + args: + arch: ${ARCH} + volumes: + - .:/arrow:delegated + - ${DOCKER_VOLUME_PREFIX}pixi-python-build:/arrow/ci/pixi/python/.pixi:delegated + command: > + /bin/sh -c "pixi build --path /arrow/ci/pixi/python && + pixi run --environment test --manifest-path /arrow/ci/pixi/python test" ############################ Python wheels ################################## diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index c5477314025d..9c541ab39f4a 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -370,6 +370,12 @@ tasks: params: image: pixi-cpp + test-pixi-python: + ci: github + template: docker-tests/github.linux.yml + params: + image: pixi-python + test-ubuntu-22.04-cpp: ci: github template: docker-tests/github.linux.yml From 8fdc50168fbbd9ccbe6a3d692a71a7baef875dd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 24 Apr 2026 14:36:49 +0200 Subject: [PATCH 04/22] Minor fix to x-hierarchy on compose.yaml --- compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/compose.yaml b/compose.yaml index 7b76b3c9e217..4523bbda9e55 100644 --- a/compose.yaml +++ b/compose.yaml @@ -139,6 +139,7 @@ x-hierarchy: - fedora-python - fedora-r-clang - pixi-cpp + - pixi-python - python-sdist - ubuntu-cpp: - ubuntu-cpp-static From 612d71fecfee214a5d00399c1c7c1c0822e319ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 24 Apr 2026 16:18:31 +0200 Subject: [PATCH 05/22] Add missing PARQUET_TEST_DATA for arrow cpp build --- ci/pixi/cpp/pixi.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/pixi/cpp/pixi.toml b/ci/pixi/cpp/pixi.toml index 6dc4ef5809fe..6dfe95204279 100644 --- a/ci/pixi/cpp/pixi.toml +++ b/ci/pixi/cpp/pixi.toml @@ -26,6 +26,7 @@ test = ["test"] [feature.test.activation.env] ARROW_TEST_DATA = "/arrow/testing/data" +PARQUET_TEST_DATA = "/arrow/cpp/submodules/parquet-testing/data" [feature.test.dependencies] binutils = "*" From 4dc5077792a8aa745907a82f2b33f238c98b3513 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 28 Apr 2026 11:34:29 +0200 Subject: [PATCH 06/22] Add Pixi Arrow CPP sanitizer build --- ci/pixi/asan/cpp/pixi.toml | 98 ++++++++++++++++++++++++++ ci/pixi/{ => default}/cpp/pixi.toml | 4 +- ci/pixi/{ => default}/python/pixi.toml | 2 +- compose.yaml | 41 +++++++++-- dev/tasks/tasks.yml | 14 ++-- 5 files changed, 141 insertions(+), 18 deletions(-) create mode 100644 ci/pixi/asan/cpp/pixi.toml rename ci/pixi/{ => default}/cpp/pixi.toml (94%) rename ci/pixi/{ => default}/python/pixi.toml (98%) diff --git a/ci/pixi/asan/cpp/pixi.toml b/ci/pixi/asan/cpp/pixi.toml new file mode 100644 index 000000000000..36dad3e146e0 --- /dev/null +++ b/ci/pixi/asan/cpp/pixi.toml @@ -0,0 +1,98 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[workspace] +channels = ["https://prefix.dev/conda-forge"] +platforms = ["linux-64"] +preview = ["pixi-build"] + +[environments] +default = [] +test = ["test"] + +[feature.test.activation.env] +ARROW_TEST_DATA = "/arrow/testing/data" +PARQUET_TEST_DATA = "/arrow/cpp/submodules/parquet-testing/data" +# GH-39973: debug memory pool conflicts with ASAN +ARROW_DEBUG_MEMORY_POOL = "none" +ASAN_OPTIONS = "allocator_may_return_null=1:detect_leaks=0:symbolize=1" +UBSAN_OPTIONS = "print_stacktrace=1" + +[feature.test.dependencies] +binutils = "*" +cmake = "*" +python = "*" + +[feature.test.tasks] +# This path mangling feels extremely finicky. Investigate alternatives to make this more robust. +test = "cd $(ls -d /arrow/ci/pixi/asan/cpp/.pixi/build/work/arrow-cpp-*/work/build | head -n 1) && ctest --output-on-failure" + +[package] +name = "arrow-cpp" +# TODO: Incorporate this to the bump version script and tests +version = "25.0.0.dev" +authors = ["Apache Arrow"] + +[package.build] +source.path = "../../../../cpp" + +[package.build.backend] +name = "pixi-build-cmake" +version = "*" + +[package.build.config] +extra-args = [ + "-DCMAKE_BUILD_TYPE=Debug", + "-DARROW_USE_ASAN=ON", + "-DARROW_USE_UBSAN=ON", + # Avoid creating huge static libraries with sanitizer instrumentation + "-DARROW_BUILD_STATIC=OFF", + "-DARROW_BUILD_TESTS=ON", + "-DARROW_COMPUTE=ON", + "-DARROW_CSV=ON", + "-DARROW_DATASET=ON", + "-DARROW_FILESYSTEM=ON", + "-DARROW_JSON=ON", + "-DARROW_PARQUET=ON", + "-DARROW_WITH_BROTLI=ON", + "-DARROW_WITH_BZ2=ON", + "-DARROW_WITH_LZ4=ON", + "-DARROW_WITH_SNAPPY=ON", + "-DARROW_WITH_ZLIB=ON", + "-DARROW_WITH_ZSTD=ON", + # libutf8proc on package-host-dependencies seems to fail. + # More investigation needed. In the meantime use bundled. + "-Dutf8proc_SOURCE=BUNDLED", + "-DOPENSSL_ROOT_DIR=$PREFIX", +] + +[package.host-dependencies] +brotli = "*" +bzip2 = "*" +gflags = "*" +gmock = ">=1.10.0" +gtest = ">=1.10.0" +libboost-devel = "*" +lz4-c = "*" +openssl = "*" +rapidjson = "*" +re2 = "*" +snappy = "*" +thrift-cpp = ">=0.11.0" +xsimd = ">=14.0" +zlib = "*" +zstd = "*" diff --git a/ci/pixi/cpp/pixi.toml b/ci/pixi/default/cpp/pixi.toml similarity index 94% rename from ci/pixi/cpp/pixi.toml rename to ci/pixi/default/cpp/pixi.toml index 6dfe95204279..ade4d1568511 100644 --- a/ci/pixi/cpp/pixi.toml +++ b/ci/pixi/default/cpp/pixi.toml @@ -35,7 +35,7 @@ python = "*" [feature.test.tasks] # This path mangling feels extremely finicky. Investigate alternatives to make this more robust. -test = "cd $(ls -d /arrow/ci/pixi/cpp/.pixi/build/work/arrow-cpp-*/work/build | head -n 1) && ctest --output-on-failure" +test = "cd $(ls -d /arrow/ci/pixi/default/cpp/.pixi/build/work/arrow-cpp-*/work/build | head -n 1) && ctest --output-on-failure" [package] name = "arrow-cpp" @@ -44,7 +44,7 @@ version = "25.0.0.dev" authors = ["Apache Arrow"] [package.build] -source.path = "../../../cpp" +source.path = "../../../../cpp" [package.build.backend] name = "pixi-build-cmake" diff --git a/ci/pixi/python/pixi.toml b/ci/pixi/default/python/pixi.toml similarity index 98% rename from ci/pixi/python/pixi.toml rename to ci/pixi/default/python/pixi.toml index 6d096882d255..0e753211fdc2 100644 --- a/ci/pixi/python/pixi.toml +++ b/ci/pixi/default/python/pixi.toml @@ -47,7 +47,7 @@ version = "25.0.0.dev" authors = ["Apache Arrow"] [package.build] -source.path = "../../../python" +source.path = "../../../../python" [package.build.backend] name = "pixi-build-python" diff --git a/compose.yaml b/compose.yaml index 4523bbda9e55..b4958234b2b4 100644 --- a/compose.yaml +++ b/compose.yaml @@ -138,7 +138,8 @@ x-hierarchy: - fedora-cpp: - fedora-python - fedora-r-clang - - pixi-cpp + - pixi-cpp: + - pixi-cpp-sanitizer - pixi-python - python-sdist - ubuntu-cpp: @@ -200,6 +201,8 @@ volumes: name: maven-cache pixi-cpp-build: name: ${ARCH}-pixi-cpp-build + pixi-cpp-sanitizer-build: + name: ${ARCH}-pixi-cpp-sanitizer-build pixi-python-build: name: ${ARCH}-pixi-python-build python-wheel-manylinux-2-28-ccache: @@ -1168,10 +1171,34 @@ services: arch: ${ARCH} volumes: - .:/arrow:delegated - - ${DOCKER_VOLUME_PREFIX}pixi-cpp-build:/arrow/ci/pixi/cpp/.pixi:delegated + - ${DOCKER_VOLUME_PREFIX}pixi-cpp-build:/arrow/ci/pixi/default/cpp/.pixi:delegated command: > - /bin/sh -c "pixi build --path /arrow/ci/pixi/cpp && - pixi run --environment test --manifest-path /arrow/ci/pixi/cpp test" + /bin/sh -c "pixi build --path /arrow/ci/pixi/default/cpp && + pixi run --environment test --manifest-path /arrow/ci/pixi/default/cpp test" + + pixi-cpp-sanitizer: + # C++ build with pixi and Address/Undefined Behavior sanitizers enabled + # + # Usage: + # docker compose build pixi-cpp-sanitizer + # docker compose run --rm pixi-cpp-sanitizer + # Parameters: + # ARCH: amd64, arm32v7 + image: ${REPO}:${ARCH}-pixi + cap_add: + # For LeakSanitizer + - SYS_PTRACE + build: + context: . + dockerfile: ci/docker/pixi.dockerfile + args: + arch: ${ARCH} + volumes: + - .:/arrow:delegated + - ${DOCKER_VOLUME_PREFIX}pixi-cpp-sanitizer-build:/arrow/ci/pixi/asan/cpp/.pixi:delegated + command: > + /bin/sh -c "pixi build --path /arrow/ci/pixi/asan/cpp && + pixi run --environment test --manifest-path /arrow/ci/pixi/asan/cpp test" pixi-python: # CPython build with pixi @@ -1189,10 +1216,10 @@ services: arch: ${ARCH} volumes: - .:/arrow:delegated - - ${DOCKER_VOLUME_PREFIX}pixi-python-build:/arrow/ci/pixi/python/.pixi:delegated + - ${DOCKER_VOLUME_PREFIX}pixi-python-build:/arrow/ci/pixi/default/python/.pixi:delegated command: > - /bin/sh -c "pixi build --path /arrow/ci/pixi/python && - pixi run --environment test --manifest-path /arrow/ci/pixi/python test" + /bin/sh -c "pixi build --path /arrow/ci/pixi/default/python && + pixi run --environment test --manifest-path /arrow/ci/pixi/default/python test" ############################ Python wheels ################################## diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 9c541ab39f4a..547d57a27f46 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -364,17 +364,15 @@ tasks: params: image: conda-cpp-valgrind - test-pixi-cpp: - ci: github - template: docker-tests/github.linux.yml - params: - image: pixi-cpp - - test-pixi-python: +{% for image in ["pixi-cpp", + "pixi-cpp-sanitizer", + "pixi-python"] %} + test-{{ image }}: ci: github template: docker-tests/github.linux.yml params: - image: pixi-python + image: {{ image }} +{% endfor %} test-ubuntu-22.04-cpp: ci: github From efb712b88d90340e73e7dee6c3d0334ca7b3375d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 28 Apr 2026 14:12:57 +0200 Subject: [PATCH 07/22] Initial PyArrow build against sanitizer-instrumented CPython, NumPy and Arrow C++ --- ci/docker/pixi.dockerfile | 1 + ci/pixi/asan/cpp/pixi.toml | 1 - ci/pixi/asan/python/pixi.toml | 98 +++++++++++++++++++++++++++++++++++ ci/pixi/default/cpp/pixi.toml | 1 - compose.yaml | 30 ++++++++++- 5 files changed, 128 insertions(+), 3 deletions(-) create mode 100644 ci/pixi/asan/python/pixi.toml diff --git a/ci/docker/pixi.dockerfile b/ci/docker/pixi.dockerfile index 1976253a9676..9e91935b191e 100644 --- a/ci/docker/pixi.dockerfile +++ b/ci/docker/pixi.dockerfile @@ -24,6 +24,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get install -y -q \ curl \ gdb \ + git \ libc6-dbg \ tzdata \ wget && \ diff --git a/ci/pixi/asan/cpp/pixi.toml b/ci/pixi/asan/cpp/pixi.toml index 36dad3e146e0..68934fb06ce8 100644 --- a/ci/pixi/asan/cpp/pixi.toml +++ b/ci/pixi/asan/cpp/pixi.toml @@ -35,7 +35,6 @@ UBSAN_OPTIONS = "print_stacktrace=1" [feature.test.dependencies] binutils = "*" cmake = "*" -python = "*" [feature.test.tasks] # This path mangling feels extremely finicky. Investigate alternatives to make this more robust. diff --git a/ci/pixi/asan/python/pixi.toml b/ci/pixi/asan/python/pixi.toml new file mode 100644 index 000000000000..28c710fd069b --- /dev/null +++ b/ci/pixi/asan/python/pixi.toml @@ -0,0 +1,98 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[workspace] +channels = ["https://prefix.dev/conda-forge"] +platforms = ["linux-64"] +preview = ["pixi-build"] + +[environments] +default = [] +test = ["test"] + +[feature.test] +platforms = ["linux-64"] + +[feature.test.activation.env] +ARROW_TEST_DATA = "/arrow/testing/data" +PARQUET_TEST_DATA = "/arrow/cpp/submodules/parquet-testing/data" +# GH-39973: debug memory pool conflicts with ASAN +ARROW_DEBUG_MEMORY_POOL = "none" +ASAN_OPTIONS = "detect_leaks=0:symbolize=1:strict_init_order=true:allocator_may_return_null=1:use_sigaltstack=0" +UBSAN_OPTIONS = "print_stacktrace=1" + +[feature.test.dependencies] +hypothesis = "*" +pyarrow = { path = "." } +pytest = "*" +pytest-timeout = "*" + +[feature.test.tasks] +test = "pytest -r s --pyargs pyarrow --timeout=60" + +[package] +name = "pyarrow" +# TODO: Incorporate this to the bump version script and tests +version = "25.0.0.dev" +authors = ["Apache Arrow"] + +[package.build] +source.path = "../../../../python" + +[package.build.backend] +name = "pixi-build-python" +version = "*" + +[package.build.config] +noarch = false +compilers = ["c", "cxx"] +env = { + PYARROW_BUNDLE_ARROW_CPP = "0", + CMAKE_GENERATOR = "Ninja", + ARROW_HOME = "$PREFIX", + PYARROW_CMAKE_OPTIONS = "-DARROW_USE_ASAN=ON -DARROW_USE_UBSAN=ON -DCMAKE_BUILD_TYPE=Debug", + ASAN_OPTIONS = "detect_leaks=0:symbolize=1:strict_init_order=true:allocator_may_return_null=1:use_sigaltstack=0" +} + +[package.host-dependencies] +# Sanitizer-instrumented Arrow C++ (sibling pixi package). +arrow-cpp = { path = "../cpp" } +cmake = "*" +cython = ">=3.0" +gcc_linux-64 = "*" +git = "*" +gxx_linux-64 = "*" +ninja = "*" +pip = "*" +pkg-config = "*" +scikit-build-core = "*" +setuptools-scm = "*" +sysroot_linux-64 = "*" +zlib = "*" +# Instrumented CPython built from source (v3.15.0a6). +# commit hash must match numpy's pixi-packages/asan pinned commit, +# so pixi-build doesn't build cpython twice with different versions. +python.git = "https://github.com/python/cpython" +python.subdirectory = "Tools/pixi-packages/asan" +python.rev = "15b216f30d0445469ec31bc7509fcc55a216ef7c" +# Instrumented NumPy built from source. +numpy.git = "https://github.com/numpy/numpy" +numpy.subdirectory = "pixi-packages/asan" +numpy.rev = "0419105da9cd0a15a4e02bc22019c2b65272c68a" + +[package.run-dependencies] +arrow-cpp = { path = "../cpp" } diff --git a/ci/pixi/default/cpp/pixi.toml b/ci/pixi/default/cpp/pixi.toml index ade4d1568511..4386bf592133 100644 --- a/ci/pixi/default/cpp/pixi.toml +++ b/ci/pixi/default/cpp/pixi.toml @@ -31,7 +31,6 @@ PARQUET_TEST_DATA = "/arrow/cpp/submodules/parquet-testing/data" [feature.test.dependencies] binutils = "*" cmake = "*" -python = "*" [feature.test.tasks] # This path mangling feels extremely finicky. Investigate alternatives to make this more robust. diff --git a/compose.yaml b/compose.yaml index b4958234b2b4..02d78f75f2a5 100644 --- a/compose.yaml +++ b/compose.yaml @@ -140,7 +140,8 @@ x-hierarchy: - fedora-r-clang - pixi-cpp: - pixi-cpp-sanitizer - - pixi-python + - pixi-python: + - pixi-python-sanitizer - python-sdist - ubuntu-cpp: - ubuntu-cpp-static @@ -205,6 +206,8 @@ volumes: name: ${ARCH}-pixi-cpp-sanitizer-build pixi-python-build: name: ${ARCH}-pixi-python-build + pixi-python-sanitizer-build: + name: ${ARCH}-pixi-python-sanitizer-build python-wheel-manylinux-2-28-ccache: name: python-wheel-manylinux-2-28-ccache python-wheel-musllinux-1-2-ccache: @@ -1221,6 +1224,31 @@ services: /bin/sh -c "pixi build --path /arrow/ci/pixi/default/python && pixi run --environment test --manifest-path /arrow/ci/pixi/default/python test" + pixi-python-sanitizer: + # PyArrow build with pixi against sanitizer-instrumented CPython, NumPy + # and Arrow C++. Builds CPython and NumPy from source via pixi-build. + # + # Usage: + # docker compose build pixi-python-sanitizer + # docker compose run --rm pixi-python-sanitizer + # Parameters: + # ARCH: amd64, arm32v7 + image: ${REPO}:${ARCH}-pixi + cap_add: + # For LeakSanitizer + - SYS_PTRACE + build: + context: . + dockerfile: ci/docker/pixi.dockerfile + args: + arch: ${ARCH} + volumes: + - .:/arrow:delegated + - ${DOCKER_VOLUME_PREFIX}pixi-python-sanitizer-build:/arrow/ci/pixi/asan/python/.pixi:delegated + command: > + /bin/sh -c "pixi build --path /arrow/ci/pixi/asan/python && + pixi run --environment test --manifest-path /arrow/ci/pixi/asan/python test" + ############################ Python wheels ################################## # See available versions at: From 31b5ad283d9e0b38c7ae88c1c895900128ed698a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 28 Apr 2026 14:30:20 +0200 Subject: [PATCH 08/22] Add required instrumented numpy and python to test dependencies and add missing task for pixi-python-sanitizer --- ci/pixi/asan/python/pixi.toml | 6 ++++++ dev/tasks/tasks.yml | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/ci/pixi/asan/python/pixi.toml b/ci/pixi/asan/python/pixi.toml index 28c710fd069b..f57e9c05fad0 100644 --- a/ci/pixi/asan/python/pixi.toml +++ b/ci/pixi/asan/python/pixi.toml @@ -40,6 +40,12 @@ hypothesis = "*" pyarrow = { path = "." } pytest = "*" pytest-timeout = "*" +python.git = "https://github.com/python/cpython" +python.subdirectory = "Tools/pixi-packages/asan" +python.rev = "15b216f30d0445469ec31bc7509fcc55a216ef7c" +numpy.git = "https://github.com/numpy/numpy" +numpy.subdirectory = "pixi-packages/asan" +numpy.rev = "0419105da9cd0a15a4e02bc22019c2b65272c68a" [feature.test.tasks] test = "pytest -r s --pyargs pyarrow --timeout=60" diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 547d57a27f46..d982e00cb2e5 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -366,7 +366,8 @@ tasks: {% for image in ["pixi-cpp", "pixi-cpp-sanitizer", - "pixi-python"] %} + "pixi-python", + "pixi-python-sanitizer"] %} test-{{ image }}: ci: github template: docker-tests/github.linux.yml From 7595696895f8f812d25ceaaf3d96933de7349395 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 28 Apr 2026 16:13:35 +0200 Subject: [PATCH 09/22] Increase timeout and add ASAN and UBSAN flags to PyArrow --- ci/pixi/asan/python/pixi.toml | 6 +++++- dev/tasks/tasks.yml | 3 +++ python/CMakeLists.txt | 5 +++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/ci/pixi/asan/python/pixi.toml b/ci/pixi/asan/python/pixi.toml index f57e9c05fad0..05cc53fbaf7c 100644 --- a/ci/pixi/asan/python/pixi.toml +++ b/ci/pixi/asan/python/pixi.toml @@ -66,11 +66,15 @@ version = "*" [package.build.config] noarch = false compilers = ["c", "cxx"] +extra-args = [ + "-Ccmake.build-type=Debug", + "-Ccmake.define.ARROW_USE_ASAN=ON", + "-Ccmake.define.ARROW_USE_UBSAN=ON", +] env = { PYARROW_BUNDLE_ARROW_CPP = "0", CMAKE_GENERATOR = "Ninja", ARROW_HOME = "$PREFIX", - PYARROW_CMAKE_OPTIONS = "-DARROW_USE_ASAN=ON -DARROW_USE_UBSAN=ON -DCMAKE_BUILD_TYPE=Debug", ASAN_OPTIONS = "detect_leaks=0:symbolize=1:strict_init_order=true:allocator_may_return_null=1:use_sigaltstack=0" } diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index d982e00cb2e5..cd8690a628b8 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -373,6 +373,9 @@ tasks: template: docker-tests/github.linux.yml params: image: {{ image }} + {% if image == "pixi-python-sanitizer" %} + timeout: 120 + {% endif %} {% endfor %} test-ubuntu-22.04-cpp: diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index d0ddb9009f89..8fb9afad196f 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -158,7 +158,12 @@ if(NOT DEFINED ARROW_RUNTIME_SIMD_LEVEL) "MAX" CACHE STRING "Max runtime SIMD optimization level") endif() + +option(ARROW_USE_ASAN "Enable Address Sanitizer checks" OFF) +option(ARROW_USE_UBSAN "Enable Undefined Behavior sanitizer checks" OFF) + include(SetupCxxFlags) +include(san-config) if($ENV{PYODIDE}) # These variables are needed for building PyArrow on Emscripten. From 7290be33caa5490e79fa93e46d24a2848cf3a9d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 28 Apr 2026 18:12:34 +0200 Subject: [PATCH 10/22] Generate Cython and libarrow_python ASAN failures to validate instrumentation --- python/pyarrow/includes/libarrow_python.pxd | 2 ++ python/pyarrow/lib.pyx | 15 +++++++++ .../pyarrow/src/arrow/python/numpy_convert.cc | 8 +++++ .../pyarrow/src/arrow/python/numpy_convert.h | 4 +++ python/pyarrow/tests/test_misc.py | 31 +++++++++++++++++++ 5 files changed, 60 insertions(+) diff --git a/python/pyarrow/includes/libarrow_python.pxd b/python/pyarrow/includes/libarrow_python.pxd index 385a2924d1da..77604ddb766e 100644 --- a/python/pyarrow/includes/libarrow_python.pxd +++ b/python/pyarrow/includes/libarrow_python.pxd @@ -35,6 +35,8 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py": # Requires GIL CResult[shared_ptr[CDataType]] InferArrowType( object obj, object mask, c_bool pandas_null_sentinels) + # TODO: ASAN sanity check; heap-buffer-overflow. Revert before final commit. + void AsanSanityOobNumpy() cdef extern from "arrow/python/api.h" namespace "arrow::py::internal": diff --git a/python/pyarrow/lib.pyx b/python/pyarrow/lib.pyx index 7e97177a6ec0..2e1d74f4bc86 100644 --- a/python/pyarrow/lib.pyx +++ b/python/pyarrow/lib.pyx @@ -29,6 +29,7 @@ import os import sys from cython.operator cimport dereference as deref +from libc.stdlib cimport malloc, free from pyarrow.includes.libarrow cimport * from pyarrow.includes.libarrow_python cimport * from pyarrow.includes.common cimport PyObject_to_object @@ -85,6 +86,20 @@ def set_cpu_count(int count): check_status(SetCpuThreadPoolCapacity(count)) +def _asan_sanity_oob(): + # TODO: ASAN sanity check; heap-buffer-overflow. Revert before final commit. + cdef int* a = malloc(sizeof(int)) + cdef int v = a[100] + free(a) + return v + + +def _asan_sanity_oob_numpy(): + # TODO: ASAN sanity check; calls libarrow_python.so's + # AsanSanityOobNumpy(). Revert before final commit. + libarrow_python.AsanSanityOobNumpy() + + def is_opentelemetry_enabled() -> bool: """ Returns True if OpenTelemetry is enabled in libarrow. diff --git a/python/pyarrow/src/arrow/python/numpy_convert.cc b/python/pyarrow/src/arrow/python/numpy_convert.cc index 6e59835286d9..0fc2a1ba86b0 100644 --- a/python/pyarrow/src/arrow/python/numpy_convert.cc +++ b/python/pyarrow/src/arrow/python/numpy_convert.cc @@ -560,5 +560,13 @@ Status TensorToSparseCSFTensor(const std::shared_ptr& tensor, return SparseCSFTensor::Make(*tensor).Value(out); } +// TODO: ASAN sanity check; heap-buffer-overflow. Revert before final commit. +void AsanSanityOobNumpy() { + int* a = static_cast(std::malloc(sizeof(int))); + volatile int v = a[100]; + std::free(a); + (void)v; +} + } // namespace py } // namespace arrow diff --git a/python/pyarrow/src/arrow/python/numpy_convert.h b/python/pyarrow/src/arrow/python/numpy_convert.h index 2d1086e13552..0454281854ff 100644 --- a/python/pyarrow/src/arrow/python/numpy_convert.h +++ b/python/pyarrow/src/arrow/python/numpy_convert.h @@ -118,5 +118,9 @@ ARROW_PYTHON_EXPORT Status TensorToSparseCSFTensor(const std::shared_ptr& tensor, std::shared_ptr* csparse_tensor); +// TODO: ASAN sanity check; heap-buffer-overflow in libarrow_python's +// numpy conversion module. Revert before final commit. +ARROW_PYTHON_EXPORT void AsanSanityOobNumpy(); + } // namespace py } // namespace arrow diff --git a/python/pyarrow/tests/test_misc.py b/python/pyarrow/tests/test_misc.py index 856873f441b4..71be7640e837 100644 --- a/python/pyarrow/tests/test_misc.py +++ b/python/pyarrow/tests/test_misc.py @@ -30,6 +30,37 @@ def test_get_include(): assert os.path.exists(os.path.join(include_dir, 'arrow', 'api.h')) +def test_asan_sanity_oob(): + # TODO: ASAN sanity check (cython side). ASAN aborts the subprocess on + # the heap-buffer-overflow; the test fails with the report visible in the + # assertion message. Revert before final commit. + res = subprocess.run( + [sys.executable, "-c", + "import pyarrow.lib; pyarrow.lib._asan_sanity_oob()"], + capture_output=True, text=True, timeout=30, + ) + assert res.returncode == 0, ( + f"ASAN caught cython-emitted heap-buffer-overflow.\n" + f"returncode={res.returncode}\n" + f"stderr:\n{res.stderr}" + ) + + +def test_asan_sanity_oob_numpy(): + # TODO: ASAN sanity check (libarrow_python.so / numpy_convert.cc). + # Same pattern as test_asan_sanity_oob. Revert before final commit. + res = subprocess.run( + [sys.executable, "-c", + "import pyarrow.lib; pyarrow.lib._asan_sanity_oob_numpy()"], + capture_output=True, text=True, timeout=30, + ) + assert res.returncode == 0, ( + f"ASAN caught heap-buffer-overflow in libarrow_python.\n" + f"returncode={res.returncode}\n" + f"stderr:\n{res.stderr}" + ) + + @pytest.mark.skipif('sys.platform != "win32"') def test_get_library_dirs_win32(): assert any(os.path.exists(os.path.join(directory, 'arrow.lib')) From 3d9be28a63f98aefdf9f65088eebd25df120fb9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 29 Apr 2026 11:58:11 +0200 Subject: [PATCH 11/22] Some minor cleanups --- ci/pixi/asan/cpp/pixi.toml | 1 - ci/pixi/asan/python/pixi.toml | 18 ++++++++++-------- ci/pixi/default/cpp/pixi.toml | 1 - ci/pixi/default/python/pixi.toml | 2 -- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/ci/pixi/asan/cpp/pixi.toml b/ci/pixi/asan/cpp/pixi.toml index 68934fb06ce8..3d2fb73c7d11 100644 --- a/ci/pixi/asan/cpp/pixi.toml +++ b/ci/pixi/asan/cpp/pixi.toml @@ -21,7 +21,6 @@ platforms = ["linux-64"] preview = ["pixi-build"] [environments] -default = [] test = ["test"] [feature.test.activation.env] diff --git a/ci/pixi/asan/python/pixi.toml b/ci/pixi/asan/python/pixi.toml index 05cc53fbaf7c..cd7044277586 100644 --- a/ci/pixi/asan/python/pixi.toml +++ b/ci/pixi/asan/python/pixi.toml @@ -21,7 +21,6 @@ platforms = ["linux-64"] preview = ["pixi-build"] [environments] -default = [] test = ["test"] [feature.test] @@ -40,12 +39,6 @@ hypothesis = "*" pyarrow = { path = "." } pytest = "*" pytest-timeout = "*" -python.git = "https://github.com/python/cpython" -python.subdirectory = "Tools/pixi-packages/asan" -python.rev = "15b216f30d0445469ec31bc7509fcc55a216ef7c" -numpy.git = "https://github.com/numpy/numpy" -numpy.subdirectory = "pixi-packages/asan" -numpy.rev = "0419105da9cd0a15a4e02bc22019c2b65272c68a" [feature.test.tasks] test = "pytest -r s --pyargs pyarrow --timeout=60" @@ -64,7 +57,6 @@ name = "pixi-build-python" version = "*" [package.build.config] -noarch = false compilers = ["c", "cxx"] extra-args = [ "-Ccmake.build-type=Debug", @@ -106,3 +98,13 @@ numpy.rev = "0419105da9cd0a15a4e02bc22019c2b65272c68a" [package.run-dependencies] arrow-cpp = { path = "../cpp" } +# The following are necessary to run the Python test suite, +# which otherwise won't be run with the instrumented Python and NumPy. +# Important to use the same revisions as in host-dependencies to avoid +# mismatches between build and test environments. +python.git = "https://github.com/python/cpython" +python.subdirectory = "Tools/pixi-packages/asan" +python.rev = "15b216f30d0445469ec31bc7509fcc55a216ef7c" +numpy.git = "https://github.com/numpy/numpy" +numpy.subdirectory = "pixi-packages/asan" +numpy.rev = "0419105da9cd0a15a4e02bc22019c2b65272c68a" diff --git a/ci/pixi/default/cpp/pixi.toml b/ci/pixi/default/cpp/pixi.toml index 4386bf592133..a4b850034f1a 100644 --- a/ci/pixi/default/cpp/pixi.toml +++ b/ci/pixi/default/cpp/pixi.toml @@ -21,7 +21,6 @@ platforms = ["linux-64", "linux-aarch64", "osx-arm64"] preview = ["pixi-build"] [environments] -default = [] test = ["test"] [feature.test.activation.env] diff --git a/ci/pixi/default/python/pixi.toml b/ci/pixi/default/python/pixi.toml index 0e753211fdc2..123f3deb4f69 100644 --- a/ci/pixi/default/python/pixi.toml +++ b/ci/pixi/default/python/pixi.toml @@ -21,7 +21,6 @@ platforms = ["linux-64", "linux-aarch64", "osx-arm64"] preview = ["pixi-build"] [environments] -default = [] test = ["test"] [feature.test] @@ -54,7 +53,6 @@ name = "pixi-build-python" version = "*" [package.build.config] -noarch = false compilers = ["c", "cxx"] env = { PYARROW_BUNDLE_ARROW_CPP = "0", From 673edc099b8820e93df6ce0e9068b89c028c4d51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 29 Apr 2026 16:22:59 +0200 Subject: [PATCH 12/22] Add expected failures --- ci/docker/pixi.dockerfile | 1 + ci/pixi/asan/python/pixi.toml | 2 +- python/pyarrow/tests/test_dlpack.py | 2 ++ python/pyarrow/tests/test_memory.py | 6 ++++++ 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/ci/docker/pixi.dockerfile b/ci/docker/pixi.dockerfile index 9e91935b191e..f7a810669c31 100644 --- a/ci/docker/pixi.dockerfile +++ b/ci/docker/pixi.dockerfile @@ -27,6 +27,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ git \ libc6-dbg \ tzdata \ + tzdata-legacy \ wget && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/ci/pixi/asan/python/pixi.toml b/ci/pixi/asan/python/pixi.toml index cd7044277586..01fdc2512576 100644 --- a/ci/pixi/asan/python/pixi.toml +++ b/ci/pixi/asan/python/pixi.toml @@ -41,7 +41,7 @@ pytest = "*" pytest-timeout = "*" [feature.test.tasks] -test = "pytest -r s --pyargs pyarrow --timeout=60" +test = "pytest -r s --pyargs pyarrow --timeout=180 -m 'not fail_asan'" [package] name = "pyarrow" diff --git a/python/pyarrow/tests/test_dlpack.py b/python/pyarrow/tests/test_dlpack.py index 7452c8eb4a13..0f9482fb1b3e 100644 --- a/python/pyarrow/tests/test_dlpack.py +++ b/python/pyarrow/tests/test_dlpack.py @@ -55,6 +55,8 @@ def wrapper(*args, **kwargs): return wrapper +# Asan tests are flacky on check_bytes_allocated assertion. +@pytest.mark.fail_asan @check_bytes_allocated @pytest.mark.parametrize( ('value_type', 'np_type_str'), diff --git a/python/pyarrow/tests/test_memory.py b/python/pyarrow/tests/test_memory.py index 56196bd9c5b3..41df95036068 100644 --- a/python/pyarrow/tests/test_memory.py +++ b/python/pyarrow/tests/test_memory.py @@ -257,16 +257,22 @@ def check_debug_memory_pool_disabled(pool_factory, env_value, msg): assert msg in res.stderr +# Asan test fails as we are doing an out-of-bounds write. +@pytest.mark.fail_asan @pytest.mark.parametrize('pool_factory', supported_factories()) def test_debug_memory_pool_none(pool_factory): check_debug_memory_pool_disabled(pool_factory, "none", "") +# Asan test fails as we are doing an out-of-bounds write. +@pytest.mark.fail_asan @pytest.mark.parametrize('pool_factory', supported_factories()) def test_debug_memory_pool_empty(pool_factory): check_debug_memory_pool_disabled(pool_factory, "", "") +# Asan test fails as we are doing an out-of-bounds write. +@pytest.mark.fail_asan @pytest.mark.parametrize('pool_factory', supported_factories()) def test_debug_memory_pool_unknown(pool_factory): env_value = "some_arbitrary_value" From 634bb97ae826b7ba83279efb5a8bafba2b5f5297 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 30 Apr 2026 16:40:24 +0200 Subject: [PATCH 13/22] Improve ASAN detection and pytest markers --- ci/pixi/asan/python/pixi.toml | 2 +- python/pyarrow/conftest.py | 11 +++++++++++ python/pyarrow/tests/test_dlpack.py | 2 +- python/pyarrow/tests/test_memory.py | 6 +++--- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/ci/pixi/asan/python/pixi.toml b/ci/pixi/asan/python/pixi.toml index 01fdc2512576..d7fa56349252 100644 --- a/ci/pixi/asan/python/pixi.toml +++ b/ci/pixi/asan/python/pixi.toml @@ -41,7 +41,7 @@ pytest = "*" pytest-timeout = "*" [feature.test.tasks] -test = "pytest -r s --pyargs pyarrow --timeout=180 -m 'not fail_asan'" +test = "pytest -r s --pyargs pyarrow --timeout=180" [package] name = "pyarrow" diff --git a/python/pyarrow/conftest.py b/python/pyarrow/conftest.py index 9fed75dac3ee..004aab236e69 100644 --- a/python/pyarrow/conftest.py +++ b/python/pyarrow/conftest.py @@ -17,6 +17,7 @@ import pytest +import ctypes import os import pyarrow as pa from pyarrow import Codec @@ -25,6 +26,14 @@ import sys +def _is_asan_build(): + try: + ctypes.CDLL(None).__asan_init + return True + except (OSError, AttributeError): + return False + + groups = [ 'acero', 'azure', @@ -43,6 +52,7 @@ 'large_memory', 'lz4', 'memory_leak', + 'noasan', 'nopandas', 'nonumpy', 'numpy', @@ -80,6 +90,7 @@ 'large_memory': False, 'lz4': Codec.is_available('lz4'), 'memory_leak': False, + 'noasan': not _is_asan_build(), 'nopandas': False, 'nonumpy': False, 'numpy': False, diff --git a/python/pyarrow/tests/test_dlpack.py b/python/pyarrow/tests/test_dlpack.py index 0f9482fb1b3e..f9aca70582fa 100644 --- a/python/pyarrow/tests/test_dlpack.py +++ b/python/pyarrow/tests/test_dlpack.py @@ -56,7 +56,7 @@ def wrapper(*args, **kwargs): # Asan tests are flacky on check_bytes_allocated assertion. -@pytest.mark.fail_asan +@pytest.mark.noasan @check_bytes_allocated @pytest.mark.parametrize( ('value_type', 'np_type_str'), diff --git a/python/pyarrow/tests/test_memory.py b/python/pyarrow/tests/test_memory.py index 41df95036068..dc1d185b838e 100644 --- a/python/pyarrow/tests/test_memory.py +++ b/python/pyarrow/tests/test_memory.py @@ -258,21 +258,21 @@ def check_debug_memory_pool_disabled(pool_factory, env_value, msg): # Asan test fails as we are doing an out-of-bounds write. -@pytest.mark.fail_asan +@pytest.mark.noasan @pytest.mark.parametrize('pool_factory', supported_factories()) def test_debug_memory_pool_none(pool_factory): check_debug_memory_pool_disabled(pool_factory, "none", "") # Asan test fails as we are doing an out-of-bounds write. -@pytest.mark.fail_asan +@pytest.mark.noasan @pytest.mark.parametrize('pool_factory', supported_factories()) def test_debug_memory_pool_empty(pool_factory): check_debug_memory_pool_disabled(pool_factory, "", "") # Asan test fails as we are doing an out-of-bounds write. -@pytest.mark.fail_asan +@pytest.mark.noasan @pytest.mark.parametrize('pool_factory', supported_factories()) def test_debug_memory_pool_unknown(pool_factory): env_value = "some_arbitrary_value" From cbaeea272a6c987392eaad74e0d192ed402fbd55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 30 Apr 2026 17:46:11 +0200 Subject: [PATCH 14/22] Fix for Windows --- python/pyarrow/conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/pyarrow/conftest.py b/python/pyarrow/conftest.py index 004aab236e69..480bf9af4fe7 100644 --- a/python/pyarrow/conftest.py +++ b/python/pyarrow/conftest.py @@ -27,6 +27,8 @@ def _is_asan_build(): + if sys.platform == "win32": + return False try: ctypes.CDLL(None).__asan_init return True From be1ed5e2fad7ae040bba2cc459f4619f21ff9d3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 4 May 2026 11:36:10 +0200 Subject: [PATCH 15/22] Remove failing tests with OOB --- python/pyarrow/includes/libarrow_python.pxd | 2 -- python/pyarrow/lib.pyx | 14 --------- .../pyarrow/src/arrow/python/numpy_convert.cc | 8 ----- .../pyarrow/src/arrow/python/numpy_convert.h | 4 --- python/pyarrow/tests/test_misc.py | 31 ------------------- 5 files changed, 59 deletions(-) diff --git a/python/pyarrow/includes/libarrow_python.pxd b/python/pyarrow/includes/libarrow_python.pxd index 77604ddb766e..385a2924d1da 100644 --- a/python/pyarrow/includes/libarrow_python.pxd +++ b/python/pyarrow/includes/libarrow_python.pxd @@ -35,8 +35,6 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py": # Requires GIL CResult[shared_ptr[CDataType]] InferArrowType( object obj, object mask, c_bool pandas_null_sentinels) - # TODO: ASAN sanity check; heap-buffer-overflow. Revert before final commit. - void AsanSanityOobNumpy() cdef extern from "arrow/python/api.h" namespace "arrow::py::internal": diff --git a/python/pyarrow/lib.pyx b/python/pyarrow/lib.pyx index 2e1d74f4bc86..44183917b66d 100644 --- a/python/pyarrow/lib.pyx +++ b/python/pyarrow/lib.pyx @@ -86,20 +86,6 @@ def set_cpu_count(int count): check_status(SetCpuThreadPoolCapacity(count)) -def _asan_sanity_oob(): - # TODO: ASAN sanity check; heap-buffer-overflow. Revert before final commit. - cdef int* a = malloc(sizeof(int)) - cdef int v = a[100] - free(a) - return v - - -def _asan_sanity_oob_numpy(): - # TODO: ASAN sanity check; calls libarrow_python.so's - # AsanSanityOobNumpy(). Revert before final commit. - libarrow_python.AsanSanityOobNumpy() - - def is_opentelemetry_enabled() -> bool: """ Returns True if OpenTelemetry is enabled in libarrow. diff --git a/python/pyarrow/src/arrow/python/numpy_convert.cc b/python/pyarrow/src/arrow/python/numpy_convert.cc index 0fc2a1ba86b0..6e59835286d9 100644 --- a/python/pyarrow/src/arrow/python/numpy_convert.cc +++ b/python/pyarrow/src/arrow/python/numpy_convert.cc @@ -560,13 +560,5 @@ Status TensorToSparseCSFTensor(const std::shared_ptr& tensor, return SparseCSFTensor::Make(*tensor).Value(out); } -// TODO: ASAN sanity check; heap-buffer-overflow. Revert before final commit. -void AsanSanityOobNumpy() { - int* a = static_cast(std::malloc(sizeof(int))); - volatile int v = a[100]; - std::free(a); - (void)v; -} - } // namespace py } // namespace arrow diff --git a/python/pyarrow/src/arrow/python/numpy_convert.h b/python/pyarrow/src/arrow/python/numpy_convert.h index 0454281854ff..2d1086e13552 100644 --- a/python/pyarrow/src/arrow/python/numpy_convert.h +++ b/python/pyarrow/src/arrow/python/numpy_convert.h @@ -118,9 +118,5 @@ ARROW_PYTHON_EXPORT Status TensorToSparseCSFTensor(const std::shared_ptr& tensor, std::shared_ptr* csparse_tensor); -// TODO: ASAN sanity check; heap-buffer-overflow in libarrow_python's -// numpy conversion module. Revert before final commit. -ARROW_PYTHON_EXPORT void AsanSanityOobNumpy(); - } // namespace py } // namespace arrow diff --git a/python/pyarrow/tests/test_misc.py b/python/pyarrow/tests/test_misc.py index 71be7640e837..856873f441b4 100644 --- a/python/pyarrow/tests/test_misc.py +++ b/python/pyarrow/tests/test_misc.py @@ -30,37 +30,6 @@ def test_get_include(): assert os.path.exists(os.path.join(include_dir, 'arrow', 'api.h')) -def test_asan_sanity_oob(): - # TODO: ASAN sanity check (cython side). ASAN aborts the subprocess on - # the heap-buffer-overflow; the test fails with the report visible in the - # assertion message. Revert before final commit. - res = subprocess.run( - [sys.executable, "-c", - "import pyarrow.lib; pyarrow.lib._asan_sanity_oob()"], - capture_output=True, text=True, timeout=30, - ) - assert res.returncode == 0, ( - f"ASAN caught cython-emitted heap-buffer-overflow.\n" - f"returncode={res.returncode}\n" - f"stderr:\n{res.stderr}" - ) - - -def test_asan_sanity_oob_numpy(): - # TODO: ASAN sanity check (libarrow_python.so / numpy_convert.cc). - # Same pattern as test_asan_sanity_oob. Revert before final commit. - res = subprocess.run( - [sys.executable, "-c", - "import pyarrow.lib; pyarrow.lib._asan_sanity_oob_numpy()"], - capture_output=True, text=True, timeout=30, - ) - assert res.returncode == 0, ( - f"ASAN caught heap-buffer-overflow in libarrow_python.\n" - f"returncode={res.returncode}\n" - f"stderr:\n{res.stderr}" - ) - - @pytest.mark.skipif('sys.platform != "win32"') def test_get_library_dirs_win32(): assert any(os.path.exists(os.path.join(directory, 'arrow.lib')) From 82904cbe8ec740fbba4b8a447a540a3b97e928e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 4 May 2026 11:42:02 +0200 Subject: [PATCH 16/22] Update version to match what we use on pyproject.toml --- ci/pixi/asan/cpp/pixi.toml | 3 +-- ci/pixi/asan/python/pixi.toml | 3 +-- ci/pixi/default/cpp/pixi.toml | 3 +-- ci/pixi/default/python/pixi.toml | 3 +-- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/ci/pixi/asan/cpp/pixi.toml b/ci/pixi/asan/cpp/pixi.toml index 3d2fb73c7d11..7a27e3476878 100644 --- a/ci/pixi/asan/cpp/pixi.toml +++ b/ci/pixi/asan/cpp/pixi.toml @@ -42,7 +42,7 @@ test = "cd $(ls -d /arrow/ci/pixi/asan/cpp/.pixi/build/work/arrow-cpp-*/work/bui [package] name = "arrow-cpp" # TODO: Incorporate this to the bump version script and tests -version = "25.0.0.dev" +version = "25.0.0a0" authors = ["Apache Arrow"] [package.build] @@ -50,7 +50,6 @@ source.path = "../../../../cpp" [package.build.backend] name = "pixi-build-cmake" -version = "*" [package.build.config] extra-args = [ diff --git a/ci/pixi/asan/python/pixi.toml b/ci/pixi/asan/python/pixi.toml index d7fa56349252..1bed7211307d 100644 --- a/ci/pixi/asan/python/pixi.toml +++ b/ci/pixi/asan/python/pixi.toml @@ -46,7 +46,7 @@ test = "pytest -r s --pyargs pyarrow --timeout=180" [package] name = "pyarrow" # TODO: Incorporate this to the bump version script and tests -version = "25.0.0.dev" +version = "25.0.0a0" authors = ["Apache Arrow"] [package.build] @@ -54,7 +54,6 @@ source.path = "../../../../python" [package.build.backend] name = "pixi-build-python" -version = "*" [package.build.config] compilers = ["c", "cxx"] diff --git a/ci/pixi/default/cpp/pixi.toml b/ci/pixi/default/cpp/pixi.toml index a4b850034f1a..b87a231c2aff 100644 --- a/ci/pixi/default/cpp/pixi.toml +++ b/ci/pixi/default/cpp/pixi.toml @@ -38,7 +38,7 @@ test = "cd $(ls -d /arrow/ci/pixi/default/cpp/.pixi/build/work/arrow-cpp-*/work/ [package] name = "arrow-cpp" # TODO: Incorporate this to the bump version script and tests -version = "25.0.0.dev" +version = "25.0.0a0" authors = ["Apache Arrow"] [package.build] @@ -46,7 +46,6 @@ source.path = "../../../../cpp" [package.build.backend] name = "pixi-build-cmake" -version = "*" [package.build.config] extra-args = [ diff --git a/ci/pixi/default/python/pixi.toml b/ci/pixi/default/python/pixi.toml index 123f3deb4f69..0d9084d9e461 100644 --- a/ci/pixi/default/python/pixi.toml +++ b/ci/pixi/default/python/pixi.toml @@ -42,7 +42,7 @@ test = "pytest -r s --pyargs pyarrow --timeout=60" [package] name = "pyarrow" # TODO: Incorporate this to the bump version script and tests -version = "25.0.0.dev" +version = "25.0.0a0" authors = ["Apache Arrow"] [package.build] @@ -50,7 +50,6 @@ source.path = "../../../../python" [package.build.backend] name = "pixi-build-python" -version = "*" [package.build.config] compilers = ["c", "cxx"] From 3954735114eef3150d13854d573534c7d775174d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 4 May 2026 11:48:09 +0200 Subject: [PATCH 17/22] Add pixi.toml(s) to bump versions --- ci/pixi/asan/cpp/pixi.toml | 1 - ci/pixi/asan/python/pixi.toml | 1 - ci/pixi/default/cpp/pixi.toml | 1 - ci/pixi/default/python/pixi.toml | 1 - dev/release/utils-prepare.sh | 8 ++++++++ 5 files changed, 8 insertions(+), 4 deletions(-) diff --git a/ci/pixi/asan/cpp/pixi.toml b/ci/pixi/asan/cpp/pixi.toml index 7a27e3476878..47e7213eccb5 100644 --- a/ci/pixi/asan/cpp/pixi.toml +++ b/ci/pixi/asan/cpp/pixi.toml @@ -41,7 +41,6 @@ test = "cd $(ls -d /arrow/ci/pixi/asan/cpp/.pixi/build/work/arrow-cpp-*/work/bui [package] name = "arrow-cpp" -# TODO: Incorporate this to the bump version script and tests version = "25.0.0a0" authors = ["Apache Arrow"] diff --git a/ci/pixi/asan/python/pixi.toml b/ci/pixi/asan/python/pixi.toml index 1bed7211307d..3d16e4a24bf4 100644 --- a/ci/pixi/asan/python/pixi.toml +++ b/ci/pixi/asan/python/pixi.toml @@ -45,7 +45,6 @@ test = "pytest -r s --pyargs pyarrow --timeout=180" [package] name = "pyarrow" -# TODO: Incorporate this to the bump version script and tests version = "25.0.0a0" authors = ["Apache Arrow"] diff --git a/ci/pixi/default/cpp/pixi.toml b/ci/pixi/default/cpp/pixi.toml index b87a231c2aff..cbd653c93ab7 100644 --- a/ci/pixi/default/cpp/pixi.toml +++ b/ci/pixi/default/cpp/pixi.toml @@ -37,7 +37,6 @@ test = "cd $(ls -d /arrow/ci/pixi/default/cpp/.pixi/build/work/arrow-cpp-*/work/ [package] name = "arrow-cpp" -# TODO: Incorporate this to the bump version script and tests version = "25.0.0a0" authors = ["Apache Arrow"] diff --git a/ci/pixi/default/python/pixi.toml b/ci/pixi/default/python/pixi.toml index 0d9084d9e461..5d1061cd8b01 100644 --- a/ci/pixi/default/python/pixi.toml +++ b/ci/pixi/default/python/pixi.toml @@ -41,7 +41,6 @@ test = "pytest -r s --pyargs pyarrow --timeout=60" [package] name = "pyarrow" -# TODO: Incorporate this to the bump version script and tests version = "25.0.0a0" authors = ["Apache Arrow"] diff --git a/dev/release/utils-prepare.sh b/dev/release/utils-prepare.sh index e67439467cc2..a0ba64cff275 100644 --- a/dev/release/utils-prepare.sh +++ b/dev/release/utils-prepare.sh @@ -70,6 +70,14 @@ update_versions() { git add PKGBUILD popd + pushd "${ARROW_DIR}/ci/pixi" + sed -i.bak -E -e \ + "s/^version = \".+\"/version = \"${python_version}\"/" \ + */*/pixi.toml + rm -f */*/pixi.toml.bak + git add */*/pixi.toml + popd + pushd "${ARROW_DIR}/cpp" sed -i.bak -E -e \ "s/^set\(ARROW_VERSION \".+\"\)/set(ARROW_VERSION \"${version}\")/" \ From f34bcbf84d4d19730e8877b0453340bb6f89dec5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 4 May 2026 12:05:26 +0200 Subject: [PATCH 18/22] Try to fix version bump tests --- dev/release/post-10-bump-versions-test.rb | 28 +++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/dev/release/post-10-bump-versions-test.rb b/dev/release/post-10-bump-versions-test.rb index aad4d7c699cb..a8d4a3168b7b 100644 --- a/dev/release/post-10-bump-versions-test.rb +++ b/dev/release/post-10-bump-versions-test.rb @@ -257,6 +257,34 @@ def test_version_post_tag "+set(PYARROW_VERSION \"#{@next_snapshot_version}\")"], ], }, + { + path: "ci/pixi/asan/cpp.toml", + hunks: [ + ["-version = \"#{@release_version}a0\"", + "+version = \"#{@next_version}a0\""], + ], + }, + { + path: "ci/pixi/asan/python/pixi.toml", + hunks: [ + ["-version = \"#{@release_version}a0\"", + "+version = \"#{@next_version}a0\""], + ], + }, + { + path: "ci/pixi/default/cpp/pixi.toml", + hunks: [ + ["-version = \"#{@release_version}a0\"", + "+version = \"#{@next_version}a0\""], + ], + }, + { + path: "ci/pixi/default/python/pixi.toml", + hunks: [ + ["-version = \"#{@release_version}a0\"", + "+version = \"#{@next_version}a0\""], + ], + }, { path: "python/pyproject.toml", hunks: [ From a5120842d1a571f46ecc82d80c94bc26dcaef0be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 4 May 2026 12:12:34 +0200 Subject: [PATCH 19/22] Fix wrong path :) --- dev/release/post-10-bump-versions-test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/post-10-bump-versions-test.rb b/dev/release/post-10-bump-versions-test.rb index a8d4a3168b7b..ec03f0c2b47f 100644 --- a/dev/release/post-10-bump-versions-test.rb +++ b/dev/release/post-10-bump-versions-test.rb @@ -258,7 +258,7 @@ def test_version_post_tag ], }, { - path: "ci/pixi/asan/cpp.toml", + path: "ci/pixi/asan/cpp/pixi.toml", hunks: [ ["-version = \"#{@release_version}a0\"", "+version = \"#{@next_version}a0\""], From 7e5fdb304527c1b01090bd6d74ab08bc680ab781 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 4 May 2026 12:25:38 +0200 Subject: [PATCH 20/22] And there's a different test file --- dev/release/01-prepare-test.rb | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/dev/release/01-prepare-test.rb b/dev/release/01-prepare-test.rb index 22afe6180024..77911ca27fd4 100644 --- a/dev/release/01-prepare-test.rb +++ b/dev/release/01-prepare-test.rb @@ -228,6 +228,34 @@ def test_version_pre_tag "+set(MLARROW_VERSION \"#{@release_version}\")"], ], }, + { + path: "ci/pixi/asan/cpp/pixi.toml", + hunks: [ + ["-version = \"#{@release_version}a0\"", + "+version = \"#{@release_version}\""], + ], + }, + { + path: "ci/pixi/asan/python/pixi.toml", + hunks: [ + ["-version = \"#{@release_version}a0\"", + "+version = \"#{@release_version}\""], + ], + }, + { + path: "ci/pixi/default/cpp/pixi.toml", + hunks: [ + ["-version = \"#{@release_version}a0\"", + "+version = \"#{@release_version}\""], + ], + }, + { + path: "ci/pixi/default/python/pixi.toml", + hunks: [ + ["-version = \"#{@release_version}a0\"", + "+version = \"#{@release_version}\""], + ], + }, { path: "python/CMakeLists.txt", hunks: [ From d351eeeb7a0caa234fef8f39f8cf55e97c43728a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 4 May 2026 17:21:33 +0200 Subject: [PATCH 21/22] Add back required version for build backend and update regex so it does not match --- ci/pixi/asan/cpp/pixi.toml | 1 + ci/pixi/asan/python/pixi.toml | 1 + ci/pixi/default/cpp/pixi.toml | 1 + ci/pixi/default/python/pixi.toml | 1 + dev/release/utils-prepare.sh | 2 +- 5 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ci/pixi/asan/cpp/pixi.toml b/ci/pixi/asan/cpp/pixi.toml index 47e7213eccb5..da79afa236b2 100644 --- a/ci/pixi/asan/cpp/pixi.toml +++ b/ci/pixi/asan/cpp/pixi.toml @@ -49,6 +49,7 @@ source.path = "../../../../cpp" [package.build.backend] name = "pixi-build-cmake" +version = "*" [package.build.config] extra-args = [ diff --git a/ci/pixi/asan/python/pixi.toml b/ci/pixi/asan/python/pixi.toml index 3d16e4a24bf4..011e3581e208 100644 --- a/ci/pixi/asan/python/pixi.toml +++ b/ci/pixi/asan/python/pixi.toml @@ -53,6 +53,7 @@ source.path = "../../../../python" [package.build.backend] name = "pixi-build-python" +version = "*" [package.build.config] compilers = ["c", "cxx"] diff --git a/ci/pixi/default/cpp/pixi.toml b/ci/pixi/default/cpp/pixi.toml index cbd653c93ab7..cf5a713a4bc5 100644 --- a/ci/pixi/default/cpp/pixi.toml +++ b/ci/pixi/default/cpp/pixi.toml @@ -45,6 +45,7 @@ source.path = "../../../../cpp" [package.build.backend] name = "pixi-build-cmake" +version = "*" [package.build.config] extra-args = [ diff --git a/ci/pixi/default/python/pixi.toml b/ci/pixi/default/python/pixi.toml index 5d1061cd8b01..162377fc2b5f 100644 --- a/ci/pixi/default/python/pixi.toml +++ b/ci/pixi/default/python/pixi.toml @@ -49,6 +49,7 @@ source.path = "../../../../python" [package.build.backend] name = "pixi-build-python" +version = "*" [package.build.config] compilers = ["c", "cxx"] diff --git a/dev/release/utils-prepare.sh b/dev/release/utils-prepare.sh index a0ba64cff275..953ff4f0b831 100644 --- a/dev/release/utils-prepare.sh +++ b/dev/release/utils-prepare.sh @@ -72,7 +72,7 @@ update_versions() { pushd "${ARROW_DIR}/ci/pixi" sed -i.bak -E -e \ - "s/^version = \".+\"/version = \"${python_version}\"/" \ + "s/^version = \"[0-9].*\"/version = \"${python_version}\"/" \ */*/pixi.toml rm -f */*/pixi.toml.bak git add */*/pixi.toml From 5115f7d375ca1957dc3358dbb6591d8e565155ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 5 May 2026 10:33:49 +0200 Subject: [PATCH 22/22] Comment improvements and remove stray import --- ci/pixi/asan/cpp/pixi.toml | 6 +++--- ci/pixi/default/cpp/pixi.toml | 14 +++++++------- ci/pixi/default/python/pixi.toml | 2 +- python/pyarrow/lib.pyx | 1 - python/pyarrow/tests/test_memory.py | 6 +++--- 5 files changed, 14 insertions(+), 15 deletions(-) diff --git a/ci/pixi/asan/cpp/pixi.toml b/ci/pixi/asan/cpp/pixi.toml index da79afa236b2..31c83cec46da 100644 --- a/ci/pixi/asan/cpp/pixi.toml +++ b/ci/pixi/asan/cpp/pixi.toml @@ -36,7 +36,8 @@ binutils = "*" cmake = "*" [feature.test.tasks] -# This path mangling feels extremely finicky. Investigate alternatives to make this more robust. +# This path depends on pixi-build's internal work-directory layout. +# Required for running C++ tests on Docker. test = "cd $(ls -d /arrow/ci/pixi/asan/cpp/.pixi/build/work/arrow-cpp-*/work/build | head -n 1) && ctest --output-on-failure" [package] @@ -71,8 +72,7 @@ extra-args = [ "-DARROW_WITH_SNAPPY=ON", "-DARROW_WITH_ZLIB=ON", "-DARROW_WITH_ZSTD=ON", - # libutf8proc on package-host-dependencies seems to fail. - # More investigation needed. In the meantime use bundled. + # libutf8proc on package-host-dependencies fails use Bundled. "-Dutf8proc_SOURCE=BUNDLED", "-DOPENSSL_ROOT_DIR=$PREFIX", ] diff --git a/ci/pixi/default/cpp/pixi.toml b/ci/pixi/default/cpp/pixi.toml index cf5a713a4bc5..36851a3ec047 100644 --- a/ci/pixi/default/cpp/pixi.toml +++ b/ci/pixi/default/cpp/pixi.toml @@ -17,24 +17,25 @@ [workspace] channels = ["https://prefix.dev/conda-forge"] -platforms = ["linux-64", "linux-aarch64", "osx-arm64"] +platforms = ["linux-64"] preview = ["pixi-build"] [environments] test = ["test"] -[feature.test.activation.env] +[feature.test.activation.env] ARROW_TEST_DATA = "/arrow/testing/data" PARQUET_TEST_DATA = "/arrow/cpp/submodules/parquet-testing/data" [feature.test.dependencies] binutils = "*" cmake = "*" - + [feature.test.tasks] -# This path mangling feels extremely finicky. Investigate alternatives to make this more robust. +# This path depends on pixi-build's internal work-directory layout. +# Required for running C++ tests on Docker. test = "cd $(ls -d /arrow/ci/pixi/default/cpp/.pixi/build/work/arrow-cpp-*/work/build | head -n 1) && ctest --output-on-failure" - + [package] name = "arrow-cpp" version = "25.0.0a0" @@ -63,8 +64,7 @@ extra-args = [ "-DARROW_WITH_SNAPPY=ON", "-DARROW_WITH_ZLIB=ON", "-DARROW_WITH_ZSTD=ON", - # libutf8proc on package-host-dependencies seems to fail. - # More investigation needed. In the meantime use bundled. + # libutf8proc on package-host-dependencies fails use Bundled. "-Dutf8proc_SOURCE=BUNDLED", "-DOPENSSL_ROOT_DIR=$PREFIX", ] diff --git a/ci/pixi/default/python/pixi.toml b/ci/pixi/default/python/pixi.toml index 162377fc2b5f..268747afbd01 100644 --- a/ci/pixi/default/python/pixi.toml +++ b/ci/pixi/default/python/pixi.toml @@ -17,7 +17,7 @@ [workspace] channels = ["https://prefix.dev/conda-forge"] -platforms = ["linux-64", "linux-aarch64", "osx-arm64"] +platforms = ["linux-64"] preview = ["pixi-build"] [environments] diff --git a/python/pyarrow/lib.pyx b/python/pyarrow/lib.pyx index 44183917b66d..7e97177a6ec0 100644 --- a/python/pyarrow/lib.pyx +++ b/python/pyarrow/lib.pyx @@ -29,7 +29,6 @@ import os import sys from cython.operator cimport dereference as deref -from libc.stdlib cimport malloc, free from pyarrow.includes.libarrow cimport * from pyarrow.includes.libarrow_python cimport * from pyarrow.includes.common cimport PyObject_to_object diff --git a/python/pyarrow/tests/test_memory.py b/python/pyarrow/tests/test_memory.py index dc1d185b838e..7915b39a5be2 100644 --- a/python/pyarrow/tests/test_memory.py +++ b/python/pyarrow/tests/test_memory.py @@ -257,21 +257,21 @@ def check_debug_memory_pool_disabled(pool_factory, env_value, msg): assert msg in res.stderr -# Asan test fails as we are doing an out-of-bounds write. +# Asan test fails as we are deliberately doing an out-of-bounds write. @pytest.mark.noasan @pytest.mark.parametrize('pool_factory', supported_factories()) def test_debug_memory_pool_none(pool_factory): check_debug_memory_pool_disabled(pool_factory, "none", "") -# Asan test fails as we are doing an out-of-bounds write. +# Asan test fails as we are deliberately doing an out-of-bounds write. @pytest.mark.noasan @pytest.mark.parametrize('pool_factory', supported_factories()) def test_debug_memory_pool_empty(pool_factory): check_debug_memory_pool_disabled(pool_factory, "", "") -# Asan test fails as we are doing an out-of-bounds write. +# Asan test fails as we are deliberately doing an out-of-bounds write. @pytest.mark.noasan @pytest.mark.parametrize('pool_factory', supported_factories()) def test_debug_memory_pool_unknown(pool_factory):