diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 0cdf17838b5798f..2893ccd1085581e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -150,6 +150,8 @@ Tools/build/generate_sbom.py @sethmlarson # ABI check Misc/libabigail.abignore @encukou +# Multiarch +Tools/coinstall-check/ @stefanor # ---------------------------------------------------------------------------- # Platform Support diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0edf4602bfaf97a..3b4e315f9389c08 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -260,6 +260,7 @@ jobs: free-threading: ${{ matrix.free-threading }} os: ${{ matrix.os }} test-opts: ${{ matrix.test-opts || '' }} + upload-install-hashes: ${{ !matrix.bolt }} build-ubuntu-ssltests: name: 'Ubuntu SSL tests' @@ -608,6 +609,26 @@ jobs: run: | "$BUILD_DIR/cross-python/bin/python3" -m test test_sysconfig test_site test_embed + linux-install-compare: + name: Ubuntu Co-install comparison + runs-on: ubuntu-latest + needs: build-ubuntu + steps: + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.x' + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - name: Download install hashes + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + path: install-hashes + pattern: install-hashes-* + merge-multiple: true + - name: Compare install hashes + run: python3 Tools/coinstall-check/compare.py install-hashes + cifuzz: # ${{ '' } is a hack to nest jobs under the same sidebar category. name: CIFuzz${{ '' }} # zizmor: ignore[obfuscation] @@ -675,6 +696,7 @@ jobs: - build-san - cross-build-linux - cifuzz + - linux-install-compare if: always() steps: @@ -721,6 +743,7 @@ jobs: build-asan, build-san, cross-build-linux, + linux-install-compare, ' || '' }} diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index b8f264ff13b2cef..a0e47500574e622 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -8,20 +8,30 @@ on: required: false type: boolean default: false + debug: + description: Whether to create a Debug Build + required: false + type: boolean + default: true free-threading: description: Whether to use free-threaded mode required: false type: boolean default: false os: - description: OS to run the job - required: true - type: string + description: OS to run the job + required: true + type: string test-opts: - description: Extra options to pass to the test runner via TESTOPTS - required: false - type: string - default: '' + description: Extra options to pass to the test runner via TESTOPTS + required: false + type: string + default: '' + upload-install-hashes: + description: Install Python and upload the result artifact + required: false + type: boolean + default: false permissions: contents: read @@ -35,6 +45,7 @@ jobs: runs-on: ${{ inputs.os }} timeout-minutes: 60 env: + INSTALL_HASHES_FILE: install-hashes-${{ inputs.os }}-${{ case(inputs.free-threading, 't', '')}}${{ case(inputs.debug, 'd', '') }}.json.gz OPENSSL_VER: 3.5.7 PYTHONSTRICTEXTENSIONBUILD: 1 TERM: linux @@ -47,7 +58,7 @@ jobs: - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Install Clang and BOLT - if: ${{ fromJSON(inputs.bolt-optimizations) }} + if: fromJSON(inputs.bolt-optimizations) run: | sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh 19 sudo apt-get install --no-install-recommends bolt-19 @@ -84,14 +95,15 @@ jobs: PROFILE_TASK='-m test --pgo --ignore test_unpickle_module_race' ../cpython-ro-srcdir/configure --config-cache - --with-pydebug --enable-slower-safety --enable-safety --with-openssl="$OPENSSL_DIR" + ${{ fromJSON(inputs.debug) && '--with-pydebug' || '' }} ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} ${{ fromJSON(inputs.bolt-optimizations) && '--enable-bolt' || '' }} + ${{ fromJSON(inputs.upload-install-hashes) && '--prefix=/usr --libdir=/usr/lib/$(gcc --print-multiarch) --with-build-details-suffix' || '' }} - name: Build CPython out-of-tree - if: ${{ inputs.free-threading }} + if: inputs.free-threading working-directory: ${{ env.CPYTHON_BUILDDIR }} run: make -j - name: Build CPython out-of-tree (for compiler warning check) @@ -119,3 +131,42 @@ jobs: run: xvfb-run make ci EXTRATESTOPTS="${TEST_OPTS}" env: TEST_OPTS: ${{ inputs.test-opts }} + - name: Install Python + if: inputs.upload-install-hashes + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: make install DESTDIR=install + - name: Install test C extension + if: inputs.upload-install-hashes + working-directory: ${{ env.CPYTHON_BUILDDIR }} + env: + CPYTHON_TEST_EXT_NAME: c_mod + run: install/usr/bin/python3 -m pip install ../cpython-ro-srcdir/Lib/test/test_cext + - name: Install test stable ABI extension + if: inputs.upload-install-hashes && !inputs.free-threading + working-directory: ${{ env.CPYTHON_BUILDDIR }} + env: + CPYTHON_TEST_EXT_NAME: abi3_mod + CPYTHON_TEST_LIMITED: 1 + run: install/usr/bin/python3 -m pip install ../cpython-ro-srcdir/Lib/test/test_cext + - name: Install test free-threaded stable ABI extension + if: inputs.upload-install-hashes + working-directory: ${{ env.CPYTHON_BUILDDIR }} + env: + CPYTHON_TEST_EXT_NAME: abi3t_mod + CPYTHON_TEST_ABI3T: 1 + run: install/usr/bin/python3 -m pip install ../cpython-ro-srcdir/Lib/test/test_cext + - name: Hash the installed Python + if: inputs.upload-install-hashes + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: >- + install/usr/bin/python3 + ../cpython-ro-srcdir/Tools/coinstall-check/hash-r.py + install -o "$INSTALL_HASHES_FILE" + - name: Upload the installed Python hashes + if: inputs.upload-install-hashes + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: ${{ env.INSTALL_HASHES_FILE }} + path: ${{ env.CPYTHON_BUILDDIR }}/${{ env.INSTALL_HASHES_FILE }} + archive: false + retention-days: 1 diff --git a/Lib/test/test_build_details.py b/Lib/test/test_build_details.py index 30d9c213077ab75..8d836be1e1ad645 100644 --- a/Lib/test/test_build_details.py +++ b/Lib/test/test_build_details.py @@ -130,7 +130,8 @@ def location(self): dirname = os.path.join(projectdir, f.read()) else: dirname = sysconfig.get_path('stdlib') - return os.path.join(dirname, 'build-details.json') + filename = sysconfig.get_config_var('BUILD_DETAILS') + return os.path.join(dirname, filename) @property def contents(self): diff --git a/Lib/test/test_cext/setup.py b/Lib/test/test_cext/setup.py index 1eca44bdf823dc2..9a5f0a3f57815ea 100644 --- a/Lib/test/test_cext/setup.py +++ b/Lib/test/test_cext/setup.py @@ -107,6 +107,8 @@ def main(): if internal: cflags.append('-DTEST_INTERNAL_C_API=1') + py_limited_api = limited or abi3t + # Add additional include and library directories, typically for in-tree # testing where not all directories are inferred include_dirs = [] @@ -131,7 +133,9 @@ def main(): sources=sources, extra_compile_args=cflags, include_dirs=include_dirs, - library_dirs=library_dirs) + library_dirs=library_dirs, + py_limited_api=py_limited_api, + ) setup(name=f'internal_{module_name}', version='0.0', ext_modules=[ext]) diff --git a/Misc/NEWS.d/next/Tests/2026-07-01-18-14-25.gh-issue-122931.OeRDeo.rst b/Misc/NEWS.d/next/Tests/2026-07-01-18-14-25.gh-issue-122931.OeRDeo.rst new file mode 100644 index 000000000000000..bcfdaadace7cb35 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-07-01-18-14-25.gh-issue-122931.OeRDeo.rst @@ -0,0 +1,3 @@ +CI Tests to ensure Debian `multi-arch co-installability +`_ +of Python. diff --git a/Tools/README b/Tools/README index 90acb2614820ee4..60416d3b6d60f71 100644 --- a/Tools/README +++ b/Tools/README @@ -14,6 +14,9 @@ clinic A preprocessor for CPython C files in order to automate the boilerplate involved with writing argument parsing code for "builtins". +coinstall-check A tool to ensure that multiple CPython builds can be + co-installed on Linux. + freeze Create a stand-alone executable from a Python program. ftscalingbench Benchmarks for free-threading and finding bottlenecks. diff --git a/Tools/coinstall-check/compare.py b/Tools/coinstall-check/compare.py new file mode 100755 index 000000000000000..af124c18b0cce7d --- /dev/null +++ b/Tools/coinstall-check/compare.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# Compare that multiple installs of Python don't have conflicting files +# +# This is a requirement for Debian's Multi-Arch installs of Python +# https://www.debian.org/doc/debian-policy/ch-controlfields.html#multi-arch + +from argparse import ArgumentParser +from typing import Any +from pathlib import Path +import gzip +import json + + +def compare_trees(base: Path) -> bool: + seen: dict[str, str] = {} + success: bool = True + for tree in base.iterdir(): + if not tree.is_file(): + continue + + hashes: dict[str, str] = {} + print(f"Examining {tree}") + with gzip.open(tree) as f: + data = json.load(f) + build_details = data["build_details"] + hashes = data["hashes"] + + for path, digest in hashes.items(): + if is_ignored(path, build_details): + continue + if path not in seen: + seen[path] = digest + continue + if digest != seen[path]: + print(f"Mismatch found in {tree}: {path}") + print(f"{digest} != {seen[path]}") + success = False + return success + + +def is_ignored(pathname: str, build_details: dict[str, Any]) -> bool: + """Is this a path that we should ignore?""" + + path = Path(pathname) + + if path.parent.name == "__pycache__": + # Includes a timestamp, we expect a mismatch + return True + + if path.is_relative_to("usr/bin"): + # Only libraries are multi-arch co-installed, only one arch can + # have binaries in /usr/bin at a time. + return True + + in_usr_include = path.is_relative_to("usr/include") + if in_usr_include and path.name == "pyconfig.h": + # Varies according to config, installed into a tag-specific + # include directory + return True + + in_usr_lib = path.is_relative_to("usr/lib") + in_pkgconfig = in_usr_lib and path.parent.name == "pkgconfig" + if in_pkgconfig and path.name in ("python3.pc", "python3-embed.pc"): + # Only the tag-suffixed .pc files are co-installable + return True + + version = build_details["language"]["version"] + if ( + in_pkgconfig + and build_details["abi"]["flags"] # non-default install + and path.name in (f"python-{version}.pc", f"python-{version}-embed.pc") + ): + # Only the tag-suffixed .pc files are co-installable + return True + + in_dist_info = path.parent.name.endswith(".dist-info") + if in_dist_info and path.name in ("RECORD", "WHEEL"): + # RECORD: Contains hashes, not co-installable. + # WHEEL: Contains arch and version tags. Tags can be merged but + # not architectures. + return True + + in_site_packages = path.parent.name == "site-packages" + if in_site_packages and path.name.endswith((".abi3.so", ".abi3t.so")): + # abi3 and abi3t are not current co-installable (#122931) + return True + + return False + + +def main() -> None: + p = ArgumentParser("Compare multiple hash-r files") + p.add_argument( + "base_directory", + type=Path, + help="Directory containing hashes of Python installs.", + ) + args = p.parse_args() + if not compare_trees(args.base_directory): + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/Tools/coinstall-check/hash-r.py b/Tools/coinstall-check/hash-r.py new file mode 100755 index 000000000000000..a9ad77694f4d842 --- /dev/null +++ b/Tools/coinstall-check/hash-r.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +# Export a SHA512 manifest of installed files, so that we can ensure that +# multiple installs of Python don't have conflicting files +# +# This is a requirement for Debian's Multi-Arch installs of Python +# https://www.debian.org/doc/debian-policy/ch-controlfields.html#multi-arch + +from argparse import ArgumentParser +from hashlib import file_digest +from pathlib import Path +from typing import Any, cast +import gzip +import json + + +def load_build_details(base: Path) -> dict[str, Any]: + for path in base.glob("usr/lib/python*/build-details*.json"): + details = json.loads(path.read_bytes()) + return cast(dict[str, Any], details) + raise AssertionError(f"build-details.json not found in {base}") + + +def hash_tree(base: Path, algorithm: str = "sha512") -> dict[str, str]: + hashes: dict[str, str] = {} + for dirpath, dirnames, filenames in base.walk(): + for file in filenames: + filepath = dirpath / file + with filepath.open("rb") as f: + digest = file_digest(f, algorithm) + hashes[str(filepath.relative_to(base))] = digest.hexdigest() + return hashes + + +def write_json(destdir: Path, output: Path) -> None: + """Hash the Python install at destdir, write gzipped JSON to output.""" + data = { + "build_details": load_build_details(destdir), + "hashes": hash_tree(destdir), + } + with gzip.open(output, "wt") as f: + f.write(json.dumps(data)) + + +def main() -> None: + p = ArgumentParser("Hash a python install for comparison later") + p.add_argument( + "-o", + "--output", + type=Path, + help="Output file (gzipped)", + ) + p.add_argument( + "destdir", + type=Path, + help="Directory below which Python is installed", + ) + args = p.parse_args() + write_json(args.destdir, args.output) + + +if __name__ == "__main__": + main() diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py index 1e3687dbb807a69..e8d90796ea38ce6 100644 --- a/Tools/freeze/test/freeze.py +++ b/Tools/freeze/test/freeze.py @@ -128,6 +128,7 @@ def prepare(script=None, outdir=None): # Run configure. print(f'configuring python in {builddir}...') config_args = shlex.split(sysconfig.get_config_var('CONFIG_ARGS') or '') + config_args = [arg for arg in config_args if not arg.startswith("--libdir=/")] cmd = [os.path.join(srcdir, 'configure'), *config_args] ensure_opt(cmd, 'cache-file', os.path.join(outdir, 'python-config.cache')) prefix = os.path.join(outdir, 'python-installation')