Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions news/repack-whl-zipper.fixed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(pypi) Hosts without a native `zip` binary can now patch wheels.
56 changes: 53 additions & 3 deletions python/private/pypi/patch_whl.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@

This replaces the previous approach that depended on running Python to
repack patched wheels. Instead, after extracting and patching, we fix
the RECORD file natively in Starlark and repack using system zip.
the RECORD file natively in Starlark and repack using Bazel's zipper
(falling back to system zip when paths cannot be passed to zipper).
"""

load("@rules_python_internal//:rules_python_config.bzl", rp_config = "config")
Expand Down Expand Up @@ -80,7 +81,8 @@ def patch_whl(rctx, *, whl_path, patches):

The wheel is extracted, patched, missing RECORD entries are added
with dummy sha256 and size values, and finally the wheel is
repacked using system zip.
repacked using Bazel's zipper (falling back to system zip when paths
cannot be passed to zipper).

Args:
rctx: repository_ctx
Expand Down Expand Up @@ -222,14 +224,45 @@ def _collect_files(rctx):
return result

def _repack_whl(rctx, *, output):
"""Repack the current directory into a wheel file using system zip."""
"""Repack the current directory into a wheel file."""
os_name = repo_utils.get_platforms_os_name(rctx)
if os_name == "windows":
_repack_whl_windows(rctx, output = output)
else:
_repack_whl_unix(rctx, output = output)

def zipper_params_content(files):
"""Returns the params file content that makes Bazel's zipper pack `files`.

The zipper reads one `<name in the zip>=<path on disk>` entry per line and
has no way to quote a name, so a path with `=` or a newline can't be passed.

Args:
files: {type}`list[str]` file paths relative to the repo root.

Returns:
{type}`str | None` the content, with the entries sorted so the
result is stable, or `None` if a path can't be expressed.
"""
for path in files:
if "=" in path or "\n" in path:
return None
return "".join(["{0}={0}\n".format(path) for path in sorted(files)])

def _repack_whl_unix(rctx, *, output):
# Bazel's zipper doesn't need a `zip` binary on the host, and it writes a
# fixed timestamp so the repacked wheel is reproducible.
zipper = rctx.path(Label("@bazel_tools//tools/zip:zipper/zipper"))
params = zipper_params_content(_collect_files(rctx).keys())
if params != None:
_repack_whl_zipper(
rctx,
zipper = zipper,
params = params,
output = output,
)
return

repo_utils.execute_checked(
rctx,
op = "PatchWhl",
Expand All @@ -243,6 +276,23 @@ def _repack_whl_unix(rctx, *, output):
],
)

def _repack_whl_zipper(rctx, *, zipper, params, output):
# The paths go through a params file because a wheel can have more files
# than fit on one command line.
params_path = "zipper_params.txt"
rctx.file(params_path, params)
repo_utils.execute_checked(
rctx,
op = "PatchWhl",
arguments = [
str(zipper),
"c", # Create uncompressed archive (omits C).
str(output),
"@" + params_path,
],
)
Comment thread
rickeylev marked this conversation as resolved.
Resolved
rctx.delete(params_path)

def _repack_whl_windows(rctx, *, output):
powershell_exe = rctx.which("powershell.exe") or rctx.which("powershell")
if not powershell_exe:
Expand Down
29 changes: 28 additions & 1 deletion tests/pypi/patch_whl/patch_whl_tests.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,12 @@
""

load("@rules_testing//lib:test_suite.bzl", "test_suite")
load("//python/private/pypi:patch_whl.bzl", "fix_record_content", "patched_whl_name") # buildifier: disable=bzl-visibility
load(
"//python/private/pypi:patch_whl.bzl",
"fix_record_content",
"patched_whl_name",
"zipper_params_content",
) # buildifier: disable=bzl-visibility

_tests = []

Expand Down Expand Up @@ -129,6 +134,28 @@ foo-1.0.dist-info/RECORD,,

_tests.append(_test_fix_record_skips_whl_files)

def _test_zipper_params_sorted(env):
got = zipper_params_content([
"foo/b.py",
"foo-1.0.dist-info/RECORD",
"foo/__init__.py",
])
env.expect.that_str(got).equals("""\
foo-1.0.dist-info/RECORD=foo-1.0.dist-info/RECORD
foo/__init__.py=foo/__init__.py
foo/b.py=foo/b.py
""")

_tests.append(_test_zipper_params_sorted)

def _test_zipper_params_unsupported_names(env):
got_eq = zipper_params_content(["foo/a=b.py"])
env.expect.that_bool(got_eq == None).equals(True)
got_nl = zipper_params_content(["foo/a\nb.py"])
env.expect.that_bool(got_nl == None).equals(True)

_tests.append(_test_zipper_params_unsupported_names)

def patch_whl_test_suite(name):
"""Create the test suite.

Expand Down
Loading