diff --git a/news/repack-whl-zipper.fixed.md b/news/repack-whl-zipper.fixed.md new file mode 100644 index 0000000000..21bafa835a --- /dev/null +++ b/news/repack-whl-zipper.fixed.md @@ -0,0 +1 @@ +(pypi) Hosts without a native `zip` binary can now patch wheels. diff --git a/python/private/pypi/patch_whl.bzl b/python/private/pypi/patch_whl.bzl index b257f2b4fe..d70c674b94 100644 --- a/python/private/pypi/patch_whl.bzl +++ b/python/private/pypi/patch_whl.bzl @@ -16,7 +16,8 @@ This replaces the previous approach that depended on running Python to repack patched wheels. Instead, after extracting and patching, we fix -the RECORD file natively in Starlark and repack using system zip. +the RECORD file natively in Starlark and repack using Bazel's zipper +(falling back to system zip when paths cannot be passed to zipper). """ load("@rules_python_internal//:rules_python_config.bzl", rp_config = "config") @@ -80,7 +81,8 @@ def patch_whl(rctx, *, whl_path, patches): The wheel is extracted, patched, missing RECORD entries are added with dummy sha256 and size values, and finally the wheel is - repacked using system zip. + repacked using Bazel's zipper (falling back to system zip when paths + cannot be passed to zipper). Args: rctx: repository_ctx @@ -222,14 +224,45 @@ def _collect_files(rctx): return result def _repack_whl(rctx, *, output): - """Repack the current directory into a wheel file using system zip.""" + """Repack the current directory into a wheel file.""" os_name = repo_utils.get_platforms_os_name(rctx) if os_name == "windows": _repack_whl_windows(rctx, output = output) else: _repack_whl_unix(rctx, output = output) +def zipper_params_content(files): + """Returns the params file content that makes Bazel's zipper pack `files`. + + The zipper reads one `=` entry per line and + has no way to quote a name, so a path with `=` or a newline can't be passed. + + Args: + files: {type}`list[str]` file paths relative to the repo root. + + Returns: + {type}`str | None` the content, with the entries sorted so the + result is stable, or `None` if a path can't be expressed. + """ + for path in files: + if "=" in path or "\n" in path: + return None + return "".join(["{0}={0}\n".format(path) for path in sorted(files)]) + def _repack_whl_unix(rctx, *, output): + # Bazel's zipper doesn't need a `zip` binary on the host, and it writes a + # fixed timestamp so the repacked wheel is reproducible. + zipper = rctx.path(Label("@bazel_tools//tools/zip:zipper/zipper")) + params = zipper_params_content(_collect_files(rctx).keys()) + if params != None: + _repack_whl_zipper( + rctx, + zipper = zipper, + params = params, + output = output, + ) + return + repo_utils.execute_checked( rctx, op = "PatchWhl", @@ -243,6 +276,23 @@ def _repack_whl_unix(rctx, *, output): ], ) +def _repack_whl_zipper(rctx, *, zipper, params, output): + # The paths go through a params file because a wheel can have more files + # than fit on one command line. + params_path = "zipper_params.txt" + rctx.file(params_path, params) + repo_utils.execute_checked( + rctx, + op = "PatchWhl", + arguments = [ + str(zipper), + "c", # Create uncompressed archive (omits C). + str(output), + "@" + params_path, + ], + ) + rctx.delete(params_path) + def _repack_whl_windows(rctx, *, output): powershell_exe = rctx.which("powershell.exe") or rctx.which("powershell") if not powershell_exe: diff --git a/tests/pypi/patch_whl/patch_whl_tests.bzl b/tests/pypi/patch_whl/patch_whl_tests.bzl index 5010f03dc0..9577605c89 100644 --- a/tests/pypi/patch_whl/patch_whl_tests.bzl +++ b/tests/pypi/patch_whl/patch_whl_tests.bzl @@ -15,7 +15,12 @@ "" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private/pypi:patch_whl.bzl", "fix_record_content", "patched_whl_name") # buildifier: disable=bzl-visibility +load( + "//python/private/pypi:patch_whl.bzl", + "fix_record_content", + "patched_whl_name", + "zipper_params_content", +) # buildifier: disable=bzl-visibility _tests = [] @@ -129,6 +134,28 @@ foo-1.0.dist-info/RECORD,, _tests.append(_test_fix_record_skips_whl_files) +def _test_zipper_params_sorted(env): + got = zipper_params_content([ + "foo/b.py", + "foo-1.0.dist-info/RECORD", + "foo/__init__.py", + ]) + env.expect.that_str(got).equals("""\ +foo-1.0.dist-info/RECORD=foo-1.0.dist-info/RECORD +foo/__init__.py=foo/__init__.py +foo/b.py=foo/b.py +""") + +_tests.append(_test_zipper_params_sorted) + +def _test_zipper_params_unsupported_names(env): + got_eq = zipper_params_content(["foo/a=b.py"]) + env.expect.that_bool(got_eq == None).equals(True) + got_nl = zipper_params_content(["foo/a\nb.py"]) + env.expect.that_bool(got_nl == None).equals(True) + +_tests.append(_test_zipper_params_unsupported_names) + def patch_whl_test_suite(name): """Create the test suite.