Skip to content

Commit 38ef21a

Browse files
authored
Build stdlib dependencies with relocatable.py deps (#40)
On macOS without third-party dependencies installed, CPython omits `_ssl`, `_hashlib`, `_lzma`, `_zstd` and `_decimal`, leaving gaps in the relocatability checks. This adds `relocatable.py deps` to download, verify and build static OpenSSL, XZ, zstd and mpdecimal libraries, and `configure --deps-dir` to build CPython against them: ```sh ../prebuilt-cpython/relocatable.py deps ../prebuilt-cpython/relocatable.py configure --deps-dir deps ``` Dependencies are installed under `deps/install`; already-built versions are reused. Sources come from tags of python/cpython-source-deps, as in the Windows build. Tested on macOS 14 arm64 against CPython `main`: all five modules build and link only against `libSystem`. All `stdlib.*` checks run, extension coverage increases from 57 to 62 modules, and the existing failures are unchanged. OpenSSL still embeds paths under `deps/install` for configuration, certificates and provider modules. Detecting those dependencies after relocation remains follow-up work.
1 parent 24b32f8 commit 38ef21a

2 files changed

Lines changed: 221 additions & 12 deletions

File tree

‎README.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,15 @@ It's broken up into subcommands to configure, make, gather, and test a relocatab
3636

3737
The defaults will result in builddir/ containing the build and dist/ containing the files that comprise a relocatable build.
3838

39+
Some stdlib extension modules need third-party libraries that macOS doesn't provide, so without a package manager, `_ssl`, `_hashlib`, `_lzma`, `_zstd` and `_decimal` aren't built. To build those libraries (OpenSSL, XZ, zstd and mpdecimal) from source, run `deps` first and pass its directory to `configure`:
40+
41+
```shell
42+
../prebuilt-cpython/relocatable.py deps
43+
../prebuilt-cpython/relocatable.py configure --deps-dir deps
44+
```
45+
46+
`deps` downloads each source archive from [cpython-source-deps](https://github.com/python/cpython-source-deps), checks it, and installs only static libraries into deps/install, so the extension modules don't link against anything outside the installation. OpenSSL still has compiled-in paths for its configuration and modules under deps/install. Use `--deps-dir DIR` to put them somewhere else.
47+
3948
The key thing to note is which patching approach to specify with `relocatable.py gather`. There are:
4049

4150
- `--patch` which rewrites all found paths of the placeholder `--prefix` argument to the target install directory (which defaults to dist/); this is meant to follow the approach used by conda.

‎relocatable.py‎

Lines changed: 212 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,17 @@
1111
# ///
1212

1313
import argparse
14+
import dataclasses
15+
import hashlib
1416
import os
1517
import pathlib
1618
import re
1719
import shutil
1820
import subprocess
1921
import sys
22+
import tarfile
2023
import tempfile
24+
import urllib.request
2125
from typing import TextIO
2226

2327
from relocatability import (
@@ -38,6 +42,108 @@
3842
}
3943

4044

45+
@dataclasses.dataclass(frozen=True)
46+
class Dependency:
47+
"""A third-party library that a stdlib extension module needs."""
48+
49+
name: str
50+
version: str
51+
sha256: str
52+
# Commands run in the unpacked source directory. "{prefix}" and "{jobs}"
53+
# are replaced with the install directory and the number of make jobs.
54+
commands: tuple[tuple[str, ...], ...]
55+
56+
@property
57+
def basename(self) -> str:
58+
return f"{self.name}-{self.version}"
59+
60+
@property
61+
def url(self) -> str:
62+
return f"{SOURCE_DEPS_URL}/archive/refs/tags/{self.basename}.tar.gz"
63+
64+
65+
# Libraries for the extension modules that macOS doesn't provide (_ssl,
66+
# _hashlib, _lzma, _zstd and _decimal). Only static libraries are installed,
67+
# so the extension modules don't link against anything outside the
68+
# installation.
69+
# The sources are tags of SOURCE_DEPS_URL, which CPython's Windows build also
70+
# uses. The checksums are of GitHub's archives of those tags.
71+
SOURCE_DEPS_URL = "https://github.com/python/cpython-source-deps"
72+
DEPENDENCIES = (
73+
Dependency(
74+
"openssl",
75+
"3.5.9",
76+
"d6cf0d9a651f25fe506f6b3dc133a80b9b79a0bc565f570dcf8fdbd597551485",
77+
(
78+
(
79+
"./Configure",
80+
"no-shared",
81+
"no-tests",
82+
"-fPIC",
83+
"--prefix={prefix}",
84+
"--libdir=lib",
85+
"--openssldir={prefix}/ssl",
86+
),
87+
("make", "-j{jobs}"),
88+
("make", "install_sw"),
89+
),
90+
),
91+
Dependency(
92+
"xz",
93+
# The newer xz tags only have what the Windows build needs.
94+
"5.2.5",
95+
"a15c168e39e87d750c3dc766edc7f19bdda57dacf01e509678467eace91ad282",
96+
(
97+
# This tag's scripts aren't marked executable.
98+
("sh", "-c", "chmod +x configure build-aux/*"),
99+
(
100+
"./configure",
101+
"--prefix={prefix}",
102+
"--disable-shared",
103+
"--with-pic",
104+
"--disable-dependency-tracking",
105+
"--disable-xz",
106+
"--disable-xzdec",
107+
"--disable-lzmadec",
108+
"--disable-lzmainfo",
109+
"--disable-lzma-links",
110+
"--disable-scripts",
111+
"--disable-doc",
112+
),
113+
("make", "-j{jobs}"),
114+
("make", "install"),
115+
),
116+
),
117+
Dependency(
118+
"zstd",
119+
"1.5.7",
120+
"f24b52470d12f466e9fa4fcc94e6c530625ada51d7b36de7fdc6ed7e6f499c8e",
121+
(
122+
(
123+
"make",
124+
"-C",
125+
"lib",
126+
"-j{jobs}",
127+
"install-static",
128+
"install-includes",
129+
"install-pc",
130+
"PREFIX={prefix}",
131+
),
132+
),
133+
),
134+
Dependency(
135+
"mpdecimal",
136+
"4.0.0",
137+
"338fac3fb8cdd60f406b6326431338756f58a8af94229ffd9bf1e7c2b1ad71ca",
138+
(
139+
("./configure", "--prefix={prefix}", "--disable-cxx"),
140+
("make", "-j{jobs}"),
141+
("make", "install"),
142+
),
143+
),
144+
)
145+
146+
41147
def placeholder_prefix() -> str:
42148
"""Return the placeholder prefix used when building CPython."""
43149
return "/the/knights/who/say/ni".ljust(255, "i")
@@ -112,19 +218,91 @@ def patch_origin(output_dir: pathlib.Path) -> None:
112218
)
113219

114220

115-
def run_configure(source_dir: pathlib.Path, build_dir: pathlib.Path) -> None:
221+
def download(url: str, sha256: str, downloads_dir: pathlib.Path) -> pathlib.Path:
222+
"""Download *url* into *downloads_dir* unless it's already there."""
223+
path = downloads_dir / url.rpartition("/")[2]
224+
if not path.exists():
225+
downloads_dir.mkdir(parents=True, exist_ok=True)
226+
partial = path.with_name(path.name + ".partial")
227+
with (
228+
urllib.request.urlopen(url, timeout=60) as response,
229+
partial.open("wb") as file,
230+
):
231+
shutil.copyfileobj(response, file)
232+
partial.rename(path)
233+
digest = hashlib.sha256(path.read_bytes()).hexdigest()
234+
if digest != sha256:
235+
raise ValueError(f"{path} has SHA-256 {digest}, expected {sha256}")
236+
return path
237+
238+
239+
def prepend_flags(flags: str, variable: str) -> str:
240+
"""Put *flags* before the value of the environment *variable*."""
241+
return " ".join(filter(None, [flags, os.environ.get(variable, "").strip()]))
242+
243+
244+
def run_deps(deps_dir: pathlib.Path) -> None:
245+
"""Build the third-party libraries in DEPENDENCIES into *deps_dir*."""
246+
deps_dir = deps_dir.resolve()
247+
install_dir = deps_dir / "install"
248+
jobs = os.process_cpu_count() or 1
249+
env = os.environ | {"CFLAGS": prepend_flags("-O2 -fPIC", "CFLAGS")}
250+
for dependency in DEPENDENCIES:
251+
stamp = install_dir / f".{dependency.basename}.built"
252+
if stamp.exists():
253+
continue
254+
archive = download(dependency.url, dependency.sha256, deps_dir / "downloads")
255+
source_dir = deps_dir / "src" / dependency.basename
256+
if source_dir.exists():
257+
shutil.rmtree(source_dir)
258+
with tarfile.open(archive) as tar:
259+
tar.extractall(source_dir.parent, filter="data")
260+
(source_dir.parent / f"cpython-source-deps-{dependency.basename}").rename(
261+
source_dir
262+
)
263+
for command in dependency.commands:
264+
subprocess.run(
265+
[arg.format(prefix=install_dir, jobs=jobs) for arg in command],
266+
cwd=source_dir,
267+
env=env,
268+
check=True,
269+
)
270+
stamp.touch()
271+
# Some of the builds can't skip their shared libraries. Remove them so
272+
# CPython links the static ones.
273+
for path in (install_dir / "lib").iterdir():
274+
if path.name.endswith(".dylib") or ".so" in path.suffixes:
275+
path.unlink()
276+
277+
278+
def run_configure(
279+
source_dir: pathlib.Path,
280+
build_dir: pathlib.Path,
281+
deps_dir: pathlib.Path | None = None,
282+
) -> None:
116283
"""Run `configure` in the build directory."""
117284
configure = source_dir.resolve() / "configure"
118285
build_dir.mkdir(parents=True, exist_ok=True)
119-
subprocess.run(
120-
[
121-
configure,
122-
f"--prefix={placeholder_prefix()}",
123-
"--without-static-libpython",
124-
],
125-
cwd=build_dir,
126-
check=True,
127-
)
286+
args = [
287+
configure,
288+
f"--prefix={placeholder_prefix()}",
289+
"--without-static-libpython",
290+
]
291+
env = None
292+
if deps_dir is not None:
293+
install_dir = deps_dir.resolve() / "install"
294+
# --with-openssl is needed without pkg-config; the other libraries are
295+
# found through the compiler and linker search paths.
296+
args += [
297+
f"--with-openssl={install_dir}",
298+
"CPPFLAGS=" + prepend_flags(f"-I{install_dir}/include", "CPPFLAGS"),
299+
"LDFLAGS=" + prepend_flags(f"-L{install_dir}/lib", "LDFLAGS"),
300+
]
301+
pkg_config_dirs = [os.fspath(install_dir / "lib" / "pkgconfig")]
302+
if pkg_config_path := os.environ.get("PKG_CONFIG_PATH"):
303+
pkg_config_dirs.append(pkg_config_path)
304+
env = os.environ | {"PKG_CONFIG_PATH": os.pathsep.join(pkg_config_dirs)}
305+
subprocess.run(args, cwd=build_dir, env=env, check=True)
128306

129307

130308
def run_make(source_dir: pathlib.Path, build_dir: pathlib.Path) -> None:
@@ -303,7 +481,24 @@ def parse_args(args: list[str] | None = None) -> argparse.Namespace:
303481
help="build directory (default: SOURCE_DIR/builddir)",
304482
)
305483

306-
subparsers.add_parser("configure", parents=[common_parser], help="Run `configure`")
484+
deps_parser = subparsers.add_parser(
485+
"deps",
486+
parents=[common_parser],
487+
help="Build third-party libraries for stdlib extension modules",
488+
)
489+
deps_parser.add_argument(
490+
"--deps-dir",
491+
type=pathlib.Path,
492+
help="where to build and install them (default: SOURCE_DIR/deps)",
493+
)
494+
configure_parser = subparsers.add_parser(
495+
"configure", parents=[common_parser], help="Run `configure`"
496+
)
497+
configure_parser.add_argument(
498+
"--deps-dir",
499+
type=pathlib.Path,
500+
help="use the libraries built by the `deps` subcommand in this directory",
501+
)
307502
subparsers.add_parser("make", parents=[common_parser], help="Run `make`")
308503
gather_parser = subparsers.add_parser(
309504
"gather",
@@ -372,6 +567,8 @@ def parse_args(args: list[str] | None = None) -> argparse.Namespace:
372567
namespace = parser.parse_args(args)
373568
if namespace.build_dir is None:
374569
namespace.build_dir = namespace.source_dir / "builddir"
570+
if namespace.command == "deps" and namespace.deps_dir is None:
571+
namespace.deps_dir = namespace.source_dir / "deps"
375572
if namespace.command == "gather" and namespace.output_dir is None:
376573
namespace.output_dir = namespace.source_dir / "dist"
377574
if namespace.command == "gather":
@@ -390,8 +587,11 @@ def parse_args(args: list[str] | None = None) -> argparse.Namespace:
390587
def main(args: list[str] | None = None) -> int:
391588
namespace = parse_args(args)
392589
match namespace.command:
590+
case "deps":
591+
run_deps(namespace.deps_dir)
592+
return 0
393593
case "configure":
394-
run_configure(namespace.source_dir, namespace.build_dir)
594+
run_configure(namespace.source_dir, namespace.build_dir, namespace.deps_dir)
395595
return 0
396596
case "make":
397597
run_make(namespace.source_dir, namespace.build_dir)

0 commit comments

Comments
 (0)