From 00572680d73345f56413faff62cdb371d42be897 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 15 Nov 2026 20:23:00 -0800 Subject: [PATCH] refactor(whl_filegroup): convert extract_wheel_files build tool to Rust Extracting wheel files for `whl_filegroup` currently runs a Python process per target to parse `RECORD` and unpack matching entries. Add `//crates/extract_wheel_files:extract_wheel_files_lib` using `zip`, `csv`, and `regex`, register `extract_wheel_files` on `shrubbery`, and route `PyExtractWheelFiles` through `get_shrubbery_tool`. --- crates/extract_wheel_files/BUILD.bazel | 23 + crates/extract_wheel_files/src/lib.rs | 252 +++++++ crates/shrubbery/BUILD.bazel | 1 + crates/shrubbery/src/lib.rs | 3 + python/private/whl_filegroup/BUILD.bazel | 4 + .../private/whl_filegroup/whl_filegroup.bzl | 16 +- tests/extract_wheel_files/BUILD.bazel | 20 + .../extract_wheel_files_test.rs | 643 ++++++++++++++++++ tests/shrubbery/shrubbery_test.rs | 2 +- 9 files changed, 961 insertions(+), 3 deletions(-) create mode 100644 crates/extract_wheel_files/BUILD.bazel create mode 100644 crates/extract_wheel_files/src/lib.rs create mode 100644 tests/extract_wheel_files/BUILD.bazel create mode 100644 tests/extract_wheel_files/extract_wheel_files_test.rs diff --git a/crates/extract_wheel_files/BUILD.bazel b/crates/extract_wheel_files/BUILD.bazel new file mode 100644 index 0000000000..400b10beea --- /dev/null +++ b/crates/extract_wheel_files/BUILD.bazel @@ -0,0 +1,23 @@ +load("@rules_rust//rust:defs.bzl", "rust_library") +load("//python/private:distribution_filegroup.bzl", "distribution_filegroup") # buildifier: disable=bzl-visibility + +package(default_visibility = ["//:__subpackages__"]) + +licenses(["notice"]) + +rust_library( + name = "extract_wheel_files_lib", + srcs = ["src/lib.rs"], + edition = "2024", + deps = [ + "@crates//:anyhow", + "@crates//:csv", + "@crates//:lexopt", + "@crates//:regex", + "@crates//:zip", + ], +) + +distribution_filegroup( + name = "distribution", +) diff --git a/crates/extract_wheel_files/src/lib.rs b/crates/extract_wheel_files/src/lib.rs new file mode 100644 index 0000000000..e98b9da42f --- /dev/null +++ b/crates/extract_wheel_files/src/lib.rs @@ -0,0 +1,252 @@ +//! Library for extracting files from a Python wheel (`.whl`) archive matching a regex pattern. +//! +//! Reads the `.dist-info/RECORD` file from ``, filters column 0 file paths using +//! start-anchored regex matching (`re.match` semantics), and extracts matching files into +//! ``. + +// Errors must be returned, not panicked; see the `rust-lint` config in .bazelrc. +#![deny(clippy::expect_used, clippy::unwrap_used)] + +use std::ffi::OsString; +use std::fs::{self, File}; +use std::io::{self, BufReader, Read}; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; + +use anyhow::{Context, Result, bail}; +use lexopt::prelude::*; +use regex::Regex; +use zip::ZipArchive; + +mod flags { + // keep-sorted start + pub const HELP: &str = "help"; + pub const HELP_SHORT: char = 'h'; + // keep-sorted end +} + +/// Usage text for `--help`. `lexopt` does not generate help, so this is maintained by hand. +const USAGE: &str = "\ +Usage: extract_wheel_files [regex_pattern] + +Extract the files listed in a wheel's RECORD that match a regex pattern. + +Arguments: + Path to the wheel to extract from + Directory to extract the matching files into + [regex_pattern] Start-anchored regex the RECORD paths must match (default: match all) + +Options: + -h, --help Print this help"; + +/// Compiles a start-anchored regular expression (`Python re.match` semantics). +pub fn compile_pattern(regex_pattern: &str) -> Result { + let anchored_pattern = format!(r"\A(?:{regex_pattern})"); + Regex::new(&anchored_pattern).with_context(|| format!("invalid regex '{regex_pattern}'")) +} + +/// Parses CSV `RECORD` bytes and returns the list of file paths (column 0). +pub fn parse_record(record_bytes: &[u8]) -> Result> { + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .flexible(true) + .from_reader(record_bytes); + let mut paths = Vec::new(); + for result in reader.records() { + let record = result.context("parsing RECORD CSV")?; + if let Some(first) = record.get(0) { + paths.push(first.to_owned()); + } + } + Ok(paths) +} + +/// Reads the `.dist-info/RECORD` file from `whl_path` and returns the list of file paths (column 0). +pub fn get_record(whl_path: &Path) -> Result> { + let mut archive = open_wheel_archive(whl_path)?; + // zip 9 returns `Result>` per entry name; a bad name means the + // archive is malformed, so surface it rather than skipping the entry. + let mut record_names: Vec = Vec::new(); + for name in archive.file_names() { + let name = name.with_context(|| format!("{}: invalid entry name", whl_path.display()))?; + if name.ends_with(".dist-info/RECORD") { + record_names.push(name.into_owned()); + } + } + + let [record_name] = record_names.as_slice() else { + bail!( + "{} doesn't contain exactly one .dist-info/RECORD", + whl_path.display() + ); + }; + + let mut record_bytes = Vec::new(); + archive + .by_name(record_name) + .and_then(|mut entry| Ok(entry.read_to_end(&mut record_bytes)?)) + .with_context(|| format!("reading {record_name} from {}", whl_path.display()))?; + + parse_record(&record_bytes) +} + +/// Filters `whl_record` entries by a pre-compiled `regex`. +pub fn get_files(whl_record: I, regex: &Regex) -> Vec +where + I: IntoIterator, + S: AsRef, +{ + whl_record + .into_iter() + .filter(|entry| regex.is_match(entry.as_ref())) + .map(|entry| entry.as_ref().to_owned()) + .collect() +} + +/// Extracts `files` from `whl_path` into `outdir`. +pub fn extract_files(whl_path: &Path, files: I, outdir: &Path) -> Result<()> +where + I: IntoIterator, + S: AsRef, +{ + fs::create_dir_all(outdir).with_context(|| format!("creating {}", outdir.display()))?; + + let mut archive = open_wheel_archive(whl_path)?; + for file in files { + let entry_name = file.as_ref(); + let mut entry = archive + .by_name(entry_name) + .with_context(|| format!("opening '{entry_name}' in {}", whl_path.display()))?; + + let dest_path = sanitize_extract_path(outdir, entry_name); + if entry.is_dir() { + fs::create_dir_all(&dest_path) + .with_context(|| format!("creating {}", dest_path.display()))?; + continue; + } + + if let Some(parent) = dest_path.parent() { + fs::create_dir_all(parent).with_context(|| format!("creating {}", parent.display()))?; + } + + let mut out_file = File::create(&dest_path) + .with_context(|| format!("creating {}", dest_path.display()))?; + io::copy(&mut entry, &mut out_file) + .with_context(|| format!("extracting '{entry_name}' to {}", dest_path.display(),))?; + } + + Ok(()) +} + +fn open_wheel_archive(whl_path: &Path) -> Result>> { + let file = File::open(whl_path).with_context(|| format!("opening {}", whl_path.display()))?; + ZipArchive::new(BufReader::new(file)) + .with_context(|| format!("{} is not a valid zip file", whl_path.display())) +} + +/// Mirrors `zipfile.ZipFile._extract_member`: drops empty, `.`, and `..` path components so an +/// entry cannot escape `outdir`. +fn sanitize_extract_path(outdir: &Path, entry_name: &str) -> PathBuf { + let mut dest = outdir.to_path_buf(); + dest.extend( + entry_name + .split(['/', '\\']) + .filter(|part| !part.is_empty() && *part != "." && *part != ".."), + ); + dest +} + +/// Parsed CLI options for `extract_wheel_files`. +#[derive(Debug, Clone)] +pub struct Options { + /// Path to the input `.whl` file. + pub whl_path: PathBuf, + /// Directory to extract matching files into. + pub outdir: PathBuf, + /// Start-anchored pattern RECORD paths must match. + pub regex: Regex, +} + +/// Result of parsing the command line. +#[derive(Debug, Clone)] +pub enum Cli { + /// `--help` was requested. + Help, + /// Extract files with the given options. + Run(Options), +} + +/// Parses command-line arguments (excluding `argv[0]` and subcommand name) into [`Cli`]. +pub fn parse_args(args: I) -> Result +where + I: IntoIterator, + T: Into, +{ + let mut positional: Vec = Vec::new(); + + let mut parser = lexopt::Parser::from_args(args.into_iter().map(Into::into)); + while let Some(arg) = parser.next()? { + match arg { + Short(flags::HELP_SHORT) | Long(flags::HELP) => return Ok(Cli::Help), + Value(val) => positional.push(val), + _ => bail!("{}", arg.unexpected()), + } + } + + if positional.len() != 2 && positional.len() != 3 { + bail!( + "expected 2 or 3 positional arguments ( [regex_pattern]), got {}", + positional.len() + ); + } + + // The length check above guarantees the first two positionals exist. + let whl_path = PathBuf::from(positional.remove(0)); + let outdir = PathBuf::from(positional.remove(0)); + let regex_pattern = match positional.pop() { + Some(raw) => raw + .into_string() + .ok() + .context("regex pattern is not valid UTF-8")?, + None => String::new(), + }; + let regex = compile_pattern(®ex_pattern)?; + + Ok(Cli::Run(Options { + whl_path, + outdir, + regex, + })) +} + +/// Executes the full `extract_wheel_files` workflow for the given options. +pub fn run(options: &Options) -> Result<()> { + let record = get_record(&options.whl_path)?; + let files = get_files(&record, &options.regex); + extract_files(&options.whl_path, &files, &options.outdir) +} + +/// Parses CLI arguments (excluding `argv[0]` and subcommand name), extracts matching wheel files, +/// and returns the process exit code. +pub fn run_cli(args: I) -> ExitCode +where + I: IntoIterator, + T: Into, +{ + let options = match parse_args(args) { + Ok(Cli::Help) => { + println!("{USAGE}"); + return ExitCode::SUCCESS; + } + Ok(Cli::Run(opts)) => opts, + Err(err) => { + eprintln!("extract_wheel_files: error: {err:#}"); + return ExitCode::FAILURE; + } + }; + if let Err(err) = run(&options) { + eprintln!("extract_wheel_files: error: {err:#}"); + return ExitCode::FAILURE; + } + ExitCode::SUCCESS +} diff --git a/crates/shrubbery/BUILD.bazel b/crates/shrubbery/BUILD.bazel index 682ec241ce..78198491e0 100644 --- a/crates/shrubbery/BUILD.bazel +++ b/crates/shrubbery/BUILD.bazel @@ -11,6 +11,7 @@ rust_library( edition = "2024", deps = [ "//crates/exe_zip_maker:exe_zip_maker_lib", + "//crates/extract_wheel_files:extract_wheel_files_lib", "//crates/py_console_script_gen:py_console_script_gen_lib", "//crates/py_wheel_dist:py_wheel_dist_lib", "//crates/wheelmaker:wheelmaker_lib", diff --git a/crates/shrubbery/src/lib.rs b/crates/shrubbery/src/lib.rs index 5116fe98ea..1ef313a025 100644 --- a/crates/shrubbery/src/lib.rs +++ b/crates/shrubbery/src/lib.rs @@ -11,6 +11,7 @@ use std::process::ExitCode; pub mod subcommands { // keep-sorted start case=no pub const EXE_ZIP_MAKER: &str = "exe_zip_maker"; + pub const EXTRACT_WHEEL_FILES: &str = "extract_wheel_files"; pub const PY_CONSOLE_SCRIPT_GEN: &str = "py_console_script_gen"; pub const PY_WHEEL_DIST: &str = "py_wheel_dist"; pub const WHEELMAKER: &str = "wheelmaker"; @@ -23,6 +24,7 @@ pub mod subcommands { pub const SUBCOMMANDS: &[&str] = &[ // keep-sorted start case=no subcommands::EXE_ZIP_MAKER, + subcommands::EXTRACT_WHEEL_FILES, subcommands::PY_CONSOLE_SCRIPT_GEN, subcommands::PY_WHEEL_DIST, subcommands::WHEELMAKER, @@ -70,6 +72,7 @@ where match sub_raw.as_ref() { // keep-sorted start case=no subcommands::EXE_ZIP_MAKER => exe_zip_maker_lib::run_cli(iter), + subcommands::EXTRACT_WHEEL_FILES => extract_wheel_files_lib::run_cli(iter), subcommands::PY_CONSOLE_SCRIPT_GEN => py_console_script_gen_lib::run_cli(iter), subcommands::PY_WHEEL_DIST => py_wheel_dist_lib::run_cli(iter), subcommands::WHEELMAKER => wheelmaker_lib::run_cli(iter), diff --git a/python/private/whl_filegroup/BUILD.bazel b/python/private/whl_filegroup/BUILD.bazel index 13ad2adac5..31f22a429e 100644 --- a/python/private/whl_filegroup/BUILD.bazel +++ b/python/private/whl_filegroup/BUILD.bazel @@ -17,4 +17,8 @@ py_binary( bzl_library( name = "whl_filegroup", srcs = ["whl_filegroup.bzl"], + deps = [ + "//python/private:toolchain_types", + "//python/private/toolchains:shrubbery_toolchain", + ], ) diff --git a/python/private/whl_filegroup/whl_filegroup.bzl b/python/private/whl_filegroup/whl_filegroup.bzl index c52211bfbc..7e12f9aa6c 100644 --- a/python/private/whl_filegroup/whl_filegroup.bzl +++ b/python/private/whl_filegroup/whl_filegroup.bzl @@ -1,16 +1,25 @@ """Implementation of whl_filegroup rule.""" +load("//python/private:toolchain_types.bzl", "SHRUBBERY_TOOLCHAIN_TYPE") +load("//python/private/toolchains:shrubbery_toolchain.bzl", "get_shrubbery_tool") + def _whl_filegroup_impl(ctx): out_dir = ctx.actions.declare_directory(ctx.attr.name) + tool = get_shrubbery_tool( + ctx, + "extract_wheel_files", + ctx.attr._extract_wheel_files_tool, + ) ctx.actions.run( outputs = [out_dir], inputs = [ctx.file.whl], - arguments = [ + arguments = tool.prefix_args + [ ctx.file.whl.path, out_dir.path, ctx.attr.pattern, ], - executable = ctx.executable._extract_wheel_files_tool, + executable = tool.files_to_run, + toolchain = tool.toolchain, mnemonic = "PyExtractWheelFiles", progress_message = "Extracting %s files from %s" % (ctx.attr.pattern, ctx.file.whl.short_path), ) @@ -61,4 +70,7 @@ from the already extracted whl file. executable = True, ), }, + toolchains = [ + config_common.toolchain_type(SHRUBBERY_TOOLCHAIN_TYPE, mandatory = False), + ], ) diff --git a/tests/extract_wheel_files/BUILD.bazel b/tests/extract_wheel_files/BUILD.bazel new file mode 100644 index 0000000000..fcbdfa719b --- /dev/null +++ b/tests/extract_wheel_files/BUILD.bazel @@ -0,0 +1,20 @@ +load("@rules_rust//rust:defs.bzl", "rust_test") + +package(default_visibility = ["//:__subpackages__"]) + +licenses(["notice"]) + +rust_test( + name = "extract_wheel_files_test", + size = "small", + srcs = ["extract_wheel_files_test.rs"], + data = [ + "//crates/shrubbery", + "//examples/wheel:minimal_with_py_package", + ], + edition = "2024", + deps = [ + "//crates/extract_wheel_files:extract_wheel_files_lib", + "@crates//:zip", + ], +) diff --git a/tests/extract_wheel_files/extract_wheel_files_test.rs b/tests/extract_wheel_files/extract_wheel_files_test.rs new file mode 100644 index 0000000000..0f33d65c50 --- /dev/null +++ b/tests/extract_wheel_files/extract_wheel_files_test.rs @@ -0,0 +1,643 @@ +use std::collections::HashSet; +use std::env; +use std::fs::{self, File}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use extract_wheel_files_lib::{ + Cli, Options, compile_pattern, extract_files, get_files, get_record, parse_args, parse_record, + run, run_cli, +}; +use zip::write::SimpleFileOptions; +use zip::{CompressionMethod, ZipWriter}; + +static TEST_COUNTER: AtomicUsize = AtomicUsize::new(0); + +const EXAMPLE_WHEEL_REL_PATH: &str = + "examples/wheel/example_minimal_package-0.0.1-py3-none-any.whl"; + +/// Parses `args` and unwraps the [`Cli::Run`] options. +fn parse_run(args: [&str; N]) -> Options { + match parse_args(args).expect("arguments should parse") { + Cli::Run(opts) => opts, + Cli::Help => panic!("expected Cli::Run, got Cli::Help"), + } +} + +struct TempDir { + path: PathBuf, +} + +impl TempDir { + fn new(prefix: &str) -> Self { + let id = TEST_COUNTER.fetch_add(1, Ordering::SeqCst); + let path = env::temp_dir().join(format!( + "extract_wheel_files_test_{}_{}_{}", + prefix, + std::process::id(), + id + )); + let _ = fs::remove_dir_all(&path); + fs::create_dir_all(&path).unwrap(); + Self { path } + } + + fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } +} + +/// Helper to build a valid ZIP archive on disk using the `zip` crate. +fn write_test_zip( + zip_path: &Path, + entries: &[(&str, &[u8], CompressionMethod)], + comment: Option<&str>, +) { + let file = File::create(zip_path).unwrap(); + let mut writer = ZipWriter::new(file); + if let Some(c) = comment { + writer.set_comment(c).unwrap(); + } + for &(name, data, method) in entries { + let options = SimpleFileOptions::default() + .compression_method(method) + .unix_permissions(0o644); + writer.start_file(name, options).unwrap(); + writer.write_all(data).unwrap(); + } + writer.finish().unwrap(); +} + +/// Recursively collects all regular files under `dir` as relative POSIX paths. +fn collect_extracted_files(dir: &Path) -> HashSet { + let mut out = HashSet::new(); + let mut stack = vec![dir.to_path_buf()]; + while let Some(curr) = stack.pop() { + if let Ok(entries) = fs::read_dir(&curr) { + for entry in entries.flatten() { + let p = entry.path(); + if p.is_dir() { + stack.push(p); + } else if p.is_file() { + let rel = p + .strip_prefix(dir) + .unwrap() + .components() + .map(|c| c.as_os_str().to_string_lossy().into_owned()) + .collect::>() + .join("/"); + out.insert(rel); + } + } + } + } + out +} + +fn create_synthetic_minimal_wheel( + whl_path: &Path, + method: CompressionMethod, + comment: Option<&str>, +) { + let record_content = concat!( + "\"examples/wheel/lib/data,with,commas.txt\",sha256=abc,12\n", + "examples/wheel/lib/data.txt,sha256=def,8\n", + "examples/wheel/lib/module_with_data.py,sha256=ghi,20\n", + "examples/wheel/lib/module_with_type_annotations.py,sha256=jkl,25\n", + "examples/wheel/lib/module_with_type_annotations.pyi,sha256=mno,26\n", + "examples/wheel/lib/simple_module.py,sha256=pqr,15\n", + "examples/wheel/main.py,sha256=stu,18\n", + "example_minimal_package-0.0.1.dist-info/WHEEL,sha256=vwx,30\n", + "example_minimal_package-0.0.1.dist-info/METADATA,sha256=yz0,40\n", + "example_minimal_package-0.0.1.dist-info/RECORD,,\n", + ); + let entries: &[(&str, &[u8], CompressionMethod)] = &[ + ( + "examples/wheel/lib/data,with,commas.txt", + b"comma,data\n", + method, + ), + ("examples/wheel/lib/data.txt", b"data\n", method), + ( + "examples/wheel/lib/module_with_data.py", + b"def foo(): pass\n", + method, + ), + ( + "examples/wheel/lib/module_with_type_annotations.py", + b"x: int = 1\n", + method, + ), + ( + "examples/wheel/lib/module_with_type_annotations.pyi", + b"x: int\n", + method, + ), + ("examples/wheel/lib/simple_module.py", b"y = 2\n", method), + ( + "examples/wheel/main.py", + b"if __name__ == '__main__': pass\n", + method, + ), + ( + "example_minimal_package-0.0.1.dist-info/WHEEL", + b"Wheel-Version: 1.0\n", + method, + ), + ( + "example_minimal_package-0.0.1.dist-info/METADATA", + b"Name: example_minimal_package\nVersion: 0.0.1\n", + method, + ), + ( + "example_minimal_package-0.0.1.dist-info/RECORD", + record_content.as_bytes(), + method, + ), + ]; + write_test_zip(whl_path, entries, comment); +} + +#[test] +fn test_parse_record_csv_edge_cases() { + let csv_bytes = concat!( + "\"pkg/file,with,commas.txt\",sha256=111,5\r\n", + "\r\n", + "\"pkg/file\"\"with\"\"quotes.txt\",sha256=222,6\n", + "pkg/single_column_flexible\n", + "pkg-1.0.dist-info/RECORD,,\r\n", + ); + let paths = parse_record(csv_bytes.as_bytes()).unwrap(); + assert_eq!( + paths, + vec![ + "pkg/file,with,commas.txt", + "pkg/file\"with\"quotes.txt", + "pkg/single_column_flexible", + "pkg-1.0.dist-info/RECORD", + ] + ); +} + +#[test] +fn test_get_wheel_record() { + let expected = vec![ + "examples/wheel/lib/data,with,commas.txt", + "examples/wheel/lib/data.txt", + "examples/wheel/lib/module_with_data.py", + "examples/wheel/lib/module_with_type_annotations.py", + "examples/wheel/lib/module_with_type_annotations.pyi", + "examples/wheel/lib/simple_module.py", + "examples/wheel/main.py", + "example_minimal_package-0.0.1.dist-info/WHEEL", + "example_minimal_package-0.0.1.dist-info/METADATA", + "example_minimal_package-0.0.1.dist-info/RECORD", + ]; + + let example_wheel = Path::new(EXAMPLE_WHEEL_REL_PATH); + if example_wheel.exists() { + let record = get_record(example_wheel).unwrap(); + assert_eq!(record, expected); + } + + let tmp = TempDir::new("get_record"); + let synth_whl = tmp.path().join("synth.whl"); + create_synthetic_minimal_wheel(&synth_whl, CompressionMethod::Stored, None); + let record = get_record(&synth_whl).unwrap(); + assert_eq!(record, expected); +} + +#[test] +fn test_get_files_with_pattern_and_empty() { + let tmp = TempDir::new("get_files"); + let whl_path = tmp.path().join("minimal.whl"); + create_synthetic_minimal_wheel(&whl_path, CompressionMethod::Deflated, None); + + let record = get_record(&whl_path).unwrap(); + + // Pattern from tests/whl_filegroup/extract_wheel_files_test.py + let regex = compile_pattern(r"(examples/wheel/lib/.*\.txt$|.*main)").unwrap(); + let files = get_files(&record, ®ex); + let expected = vec![ + "examples/wheel/lib/data,with,commas.txt", + "examples/wheel/lib/data.txt", + "examples/wheel/main.py", + ]; + assert_eq!(files, expected); + + // Empty pattern matches all files in RECORD + let empty_regex = compile_pattern("").unwrap(); + let all_files = get_files(&record, &empty_regex); + assert_eq!(all_files, record); +} + +#[test] +fn test_extract_files() { + let files = [ + "examples/wheel/lib/data,with,commas.txt", + "examples/wheel/lib/data.txt", + "examples/wheel/main.py", + ]; + let expected_set: HashSet = files.iter().map(|s| (*s).to_string()).collect(); + + let example_wheel = Path::new(EXAMPLE_WHEEL_REL_PATH); + if example_wheel.exists() { + let tmp = TempDir::new("extract_real"); + let outdir = tmp.path().join("out"); + extract_files(example_wheel, files, &outdir).unwrap(); + let extracted = collect_extracted_files(&outdir); + assert_eq!(extracted, expected_set); + } + + let tmp = TempDir::new("extract_synth"); + let synth_whl = tmp.path().join("synth.whl"); + create_synthetic_minimal_wheel(&synth_whl, CompressionMethod::Deflated, None); + let outdir = tmp.path().join("out"); + extract_files(&synth_whl, files, &outdir).unwrap(); + let extracted = collect_extracted_files(&outdir); + assert_eq!(extracted, expected_set); + assert_eq!( + fs::read_to_string(outdir.join("examples/wheel/lib/data,with,commas.txt")).unwrap(), + "comma,data\n" + ); +} + +#[test] +fn test_invalid_zip_error() { + let tmp = TempDir::new("invalid_zip"); + let bad_zip = tmp.path().join("not_a_zip.whl"); + fs::write(&bad_zip, b"not a zip archive at all").unwrap(); + + let err = get_record(&bad_zip).unwrap_err(); + assert_eq!( + err.to_string(), + format!("{} is not a valid zip file", bad_zip.display()) + ); + + let short_zip = tmp.path().join("short.whl"); + fs::write(&short_zip, b"short").unwrap(); + let err2 = get_record(&short_zip).unwrap_err(); + assert_eq!( + err2.to_string(), + format!("{} is not a valid zip file", short_zip.display()) + ); + + // A missing file is an open error, not a "not a valid zip file" error. + let missing = tmp.path().join("missing.whl"); + let err3 = get_record(&missing).unwrap_err(); + assert_eq!(err3.to_string(), format!("opening {}", missing.display())); +} + +#[test] +fn test_missing_or_multiple_record_files_error() { + let tmp = TempDir::new("record_count"); + + // 0 .dist-info/RECORD files + let zero_rec_whl = tmp.path().join("zero_record.whl"); + write_test_zip( + &zero_rec_whl, + &[("pkg/mod.py", b"x = 1\n", CompressionMethod::Stored)], + None, + ); + let err_zero = get_record(&zero_rec_whl).unwrap_err(); + assert_eq!( + err_zero.to_string(), + format!( + "{} doesn't contain exactly one .dist-info/RECORD", + zero_rec_whl.display() + ) + ); + + // 2 .dist-info/RECORD files + let two_rec_whl = tmp.path().join("two_record.whl"); + write_test_zip( + &two_rec_whl, + &[ + ( + "a-1.0.dist-info/RECORD", + b"a-1.0.dist-info/RECORD,,\n", + CompressionMethod::Stored, + ), + ( + "b-1.0.dist-info/RECORD", + b"b-1.0.dist-info/RECORD,,\n", + CompressionMethod::Stored, + ), + ], + None, + ); + let err_two = get_record(&two_rec_whl).unwrap_err(); + assert_eq!( + err_two.to_string(), + format!( + "{} doesn't contain exactly one .dist-info/RECORD", + two_rec_whl.display() + ) + ); +} + +#[test] +fn test_csv_quoted_paths_and_deflated_entries() { + let tmp = TempDir::new("csv_and_deflate"); + let whl_path = tmp.path().join("special.whl"); + let record_csv = concat!( + "\"pkg/file,with,commas.txt\",sha256=111,5\r\n", + "\"pkg/file\"\"with\"\"quotes.txt\",sha256=222,6\r\n", + "pkg/deflated.txt,sha256=333,1300\r\n", + "special-1.0.dist-info/RECORD,,\r\n", + ); + let repeated = b"hello wheel! ".repeat(100); + write_test_zip( + &whl_path, + &[ + ( + "pkg/file,with,commas.txt", + b"hello", + CompressionMethod::Stored, + ), + ( + "pkg/file\"with\"quotes.txt", + b"quotes", + CompressionMethod::Deflated, + ), + ("pkg/deflated.txt", &repeated, CompressionMethod::Deflated), + ( + "special-1.0.dist-info/RECORD", + record_csv.as_bytes(), + CompressionMethod::Deflated, + ), + ], + None, + ); + + let record = get_record(&whl_path).unwrap(); + assert_eq!( + record, + vec![ + "pkg/file,with,commas.txt", + "pkg/file\"with\"quotes.txt", + "pkg/deflated.txt", + "special-1.0.dist-info/RECORD", + ] + ); + + let outdir = tmp.path().join("extracted"); + let opts = Options { + whl_path: whl_path.clone(), + outdir: outdir.clone(), + regex: compile_pattern("pkg/.*").unwrap(), + }; + run(&opts).unwrap(); + + assert_eq!( + fs::read(outdir.join("pkg/file,with,commas.txt")).unwrap(), + b"hello" + ); + assert_eq!( + fs::read(outdir.join("pkg/file\"with\"quotes.txt")).unwrap(), + b"quotes" + ); + assert_eq!(fs::read(outdir.join("pkg/deflated.txt")).unwrap(), repeated); + assert!(!outdir.join("special-1.0.dist-info/RECORD").exists()); +} + +#[test] +fn test_empty_match_still_creates_outdir() { + let tmp = TempDir::new("empty_match"); + let whl_path = tmp.path().join("test.whl"); + create_synthetic_minimal_wheel(&whl_path, CompressionMethod::Stored, None); + + let outdir = tmp.path().join("nested/empty_out_dir"); + let opts = Options { + whl_path, + outdir: outdir.clone(), + regex: compile_pattern("nonexistent_prefix/.*").unwrap(), + }; + run(&opts).unwrap(); + assert!(outdir.is_dir()); + assert!(collect_extracted_files(&outdir).is_empty()); +} + +#[test] +fn test_parse_args_and_usage() { + let opts2 = parse_run(["foo.whl", "out_dir"]); + assert_eq!(opts2.whl_path, PathBuf::from("foo.whl")); + assert_eq!(opts2.outdir, PathBuf::from("out_dir")); + assert!(opts2.regex.is_match("anything/at/all.py")); + + let opts3 = parse_run(["foo.whl", "out_dir", r"numpy/.*\.h$"]); + assert_eq!(opts3.whl_path, PathBuf::from("foo.whl")); + assert_eq!(opts3.outdir, PathBuf::from("out_dir")); + assert!(opts3.regex.is_match("numpy/core/api.h")); + assert!(!opts3.regex.is_match("other/numpy/core/api.h")); + + assert!(matches!(parse_args(["--help"]), Ok(Cli::Help))); + assert!(matches!(parse_args(["-h"]), Ok(Cli::Help))); + assert!(matches!( + parse_args(["foo.whl", "--help", "out_dir"]), + Ok(Cli::Help) + )); + + for bad in [vec![], vec!["only_wheel.whl"], vec!["a", "b", "c", "d"]] { + let err = parse_args(bad.clone()).unwrap_err().to_string(); + assert!( + err.contains("expected 2 or 3 positional arguments"), + "unexpected error for {bad:?}: {err}" + ); + } + + let err_flag = parse_args(["--bogus", "foo.whl", "out_dir"]) + .unwrap_err() + .to_string(); + assert!(err_flag.contains("--bogus"), "unexpected error: {err_flag}"); + + let err_bad_regex = parse_args(["foo.whl", "out_dir", "[unclosed"]) + .unwrap_err() + .to_string(); + assert!( + err_bad_regex.contains("invalid regex"), + "unexpected error: {err_bad_regex}" + ); +} + +#[test] +fn test_regex_re_match_semantics_and_complex_patterns() { + let filter = |entries: &[&str], pat: &str| { + let re = compile_pattern(pat).unwrap(); + get_files(entries, &re) + }; + + // 1. Start-anchoring without end-anchoring (Python re.match semantics) + let matched = filter( + &[ + "numpy/core/include/numpy/arrayobject.h", + "other/numpy/core/include/numpy/arrayobject.h", + ], + "numpy/core/include/numpy", + ); + assert_eq!(matched, vec!["numpy/core/include/numpy/arrayobject.h"]); + + // 1b. Unanchored alternation must still anchor both branches at start + let matched_alt = filter(&["foo/a.py", "bar/b.py", "prefix/bar/b.py"], "foo|bar"); + assert_eq!(matched_alt, vec!["foo/a.py", "bar/b.py"]); + + // 2. Header pattern from tests/whl_filegroup/BUILD.bazel + let matched_hdr = filter( + &[ + "tests/whl_filegroup/include/whl_headers/header.h", + "tests/whl_filegroup/lib.py", + ], + r"tests/whl_filegroup/include/.*\.h", + ); + assert_eq!( + matched_hdr, + vec!["tests/whl_filegroup/include/whl_headers/header.h"] + ); + + // 3. Character classes, ranges, negation, shorthands, and quantifiers + let matched_complex = filter( + &[ + "pkg/mod_1/file_42.h", + "lib/abcde/file_0.cpp", + "pkg/a/file_42.h", + "pkg/abcdefg/file_42.h", + "pkg/mod_1/file_xx.h", + ], + r"^(?:pkg|lib)/[a-z_][a-z0-9_]{2,5}/file_\d+\.[ch](?:pp)?$", + ); + assert_eq!( + matched_complex, + vec!["pkg/mod_1/file_42.h", "lib/abcde/file_0.cpp"] + ); + + // 4. Negated character class and word boundaries + let matched_neg = filter( + &["pkg/main.py", "pkg/sub/main.py", "pkg/main2.py"], + r"[^/]+/\bmain\b\.py$", + ); + assert_eq!(matched_neg, vec!["pkg/main.py"]); + + // 5. Case-insensitive inline flag (?i) and non-greedy quantifiers + let matched_ci = filter(&["My_Pkg-1.0.WHL", "My_Pkg-1.0.tar"], r"(?i).*?\.whl$"); + assert_eq!(matched_ci, vec!["My_Pkg-1.0.WHL"]); + + // 6. Character class edge cases: literal ']' first, literal '-' escaped, hex escapes + let matched_cls = filter(&["]-abc/", "d"], r"^[]\-a-c\x2f]+$"); + assert_eq!(matched_cls, vec!["]-abc/"]); + + // 7. Pathological nested star pattern completes instantaneously without backtracking + let matched_nfa = filter(&["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaac", "aaaaab"], r"^(a*)*b$"); + assert_eq!(matched_nfa, vec!["aaaaab"]); + + // 8. Invalid regex returns error at compile_pattern / parse_args time + assert!(compile_pattern("[unclosed").is_err()); + assert!(compile_pattern("(unclosed").is_err()); + assert!(compile_pattern("a{5,2}").is_err()); + assert!(compile_pattern("[z-a]").is_err()); +} + +#[test] +fn test_zip_with_archive_comment_and_crc_mismatch() { + let tmp = TempDir::new("comment_and_crc"); + let whl_path = tmp.path().join("with_comment.whl"); + create_synthetic_minimal_wheel( + &whl_path, + CompressionMethod::Stored, + Some("wheel comment!!"), + ); + + let record = get_record(&whl_path).unwrap(); + assert_eq!(record.len(), 10); + + // Corrupt a byte inside the first stored entry's payload and verify CRC error detection. + let mut bytes = fs::read(&whl_path).unwrap(); + let needle = b"comma,data\n"; + let pos = bytes + .windows(needle.len()) + .position(|w| w == needle) + .unwrap(); + bytes[pos] ^= 0xFF; + let corrupted_whl = tmp.path().join("corrupted.whl"); + fs::write(&corrupted_whl, &bytes).unwrap(); + + let outdir = tmp.path().join("out_corrupt"); + let err = extract_files( + &corrupted_whl, + ["examples/wheel/lib/data,with,commas.txt"], + &outdir, + ) + .unwrap_err(); + let err = format!("{err:#}").to_lowercase(); + assert!( + err.contains("checksum") || err.contains("crc"), + "unexpected error: {err}" + ); +} + +#[test] +fn test_run_cli_and_subprocess() { + let tmp = TempDir::new("cli_direct"); + let whl_path = tmp.path().join("cli.whl"); + create_synthetic_minimal_wheel(&whl_path, CompressionMethod::Deflated, None); + let outdir = tmp.path().join("extracted_direct"); + + assert_eq!(run_cli(["only_one_arg"]), ExitCode::FAILURE); + assert_eq!( + run_cli([ + whl_path.as_os_str(), + outdir.as_os_str(), + std::ffi::OsStr::new(r"(examples/wheel/lib/.*\.txt$|.*main)"), + ]), + ExitCode::SUCCESS + ); + let expected: HashSet = [ + "examples/wheel/lib/data,with,commas.txt", + "examples/wheel/lib/data.txt", + "examples/wheel/main.py", + ] + .into_iter() + .map(String::from) + .collect(); + assert_eq!(collect_extracted_files(&outdir), expected); + + let bin_candidates = [ + "crates/shrubbery/shrubbery", + "crates/shrubbery/shrubbery.exe", + ]; + let Some(bin_path) = bin_candidates.iter().map(Path::new).find(|p| p.exists()) else { + return; + }; + + let bad_out = std::process::Command::new(bin_path) + .arg("extract_wheel_files") + .arg("only_one_arg") + .output() + .unwrap(); + assert_eq!(bad_out.status.code(), Some(1)); + let stderr_str = String::from_utf8_lossy(&bad_out.stderr); + assert!( + stderr_str.contains("extract_wheel_files: error:") + && stderr_str.contains("expected 2 or 3 positional arguments"), + "unexpected stderr: {stderr_str}" + ); + + let help_out = std::process::Command::new(bin_path) + .arg("extract_wheel_files") + .arg("--help") + .output() + .unwrap(); + assert_eq!(help_out.status.code(), Some(0)); + let stdout_str = String::from_utf8_lossy(&help_out.stdout); + assert!( + stdout_str.contains("Usage: extract_wheel_files [regex_pattern]"), + "unexpected stdout: {stdout_str}" + ); +} diff --git a/tests/shrubbery/shrubbery_test.rs b/tests/shrubbery/shrubbery_test.rs index 595f02abf3..6078a5a9ed 100644 --- a/tests/shrubbery/shrubbery_test.rs +++ b/tests/shrubbery/shrubbery_test.rs @@ -17,7 +17,7 @@ fn temp_test_dir(prefix: &str) -> PathBuf { #[test] fn test_subcommands_registry_sorted_and_complete() { - assert_eq!(SUBCOMMANDS.len(), 6); + assert_eq!(SUBCOMMANDS.len(), 7); let mut sorted = SUBCOMMANDS.to_vec(); sorted.sort_unstable(); assert_eq!(SUBCOMMANDS, sorted.as_slice());