diff --git a/.agents/rules/rules_python.md b/.agents/rules/rules_python.md index 5b12fad861..0659b0ba67 100644 --- a/.agents/rules/rules_python.md +++ b/.agents/rules/rules_python.md @@ -37,6 +37,11 @@ description: rules_python-specific Bzlmod, PyPI, and rule usage invariants `repr()` for Starlark calls; use `json.encode_indent()` for data constants. * **`@bazel_tools` `bzl_library` Deps**: Use `"//python/private:bazel_tools"` in `bzl_library` `deps` for files loading `@bazel_tools//...`. +* **`# buildifier: disable=bzl-visibility` on Internal `/private/` Loads**: + Add trailing `# buildifier: disable=bzl-visibility` comments when `tests/` or + internal non-`python/` packages (such as `crates/**` and `tools/**`) load + repo-internal `/private/` helpers like + `//python/private:distribution_filegroup.bzl`. ## WORKSPACE & Bzlmod Parity * **Extend Existing Rules Over Adding Parallel Rules**: When bringing a Bzlmod diff --git a/.agents/skills/review-code/review-justification-auditor-prompt.md b/.agents/skills/review-code/review-justification-auditor-prompt.md index 04eae5afed..d95aaa1512 100644 --- a/.agents/skills/review-code/review-justification-auditor-prompt.md +++ b/.agents/skills/review-code/review-justification-auditor-prompt.md @@ -10,8 +10,9 @@ thresholds across `git diff`, comments, PR descriptions, and review replies: `sys.path` manipulation to run a script via raw `python3` outside Bazel when a binary target exists). - Lint, type, or formatter suppressions (`# buildifier: disable=...` outside - test `/private/` loads, `# pyrefly: ignore[...]`, `# ruff: noqa`, - `# type: ignore`). + authorized `/private/` loads in `tests/` or internal non-`python/` packages + such as `crates/` and `tools/` loading `distribution_filegroup.bzl`, + `# pyrefly: ignore[...]`, `# ruff: noqa`, `# type: ignore`). - Custom timeouts or retry counts. - Review thread replies claiming a rule or convention does not apply. 3. Classify every exception into one of three outcomes: diff --git a/.agents/skills/review-code/review-starlark-prompt.md b/.agents/skills/review-code/review-starlark-prompt.md index c728e55b29..a8fd4144e6 100644 --- a/.agents/skills/review-code/review-starlark-prompt.md +++ b/.agents/skills/review-code/review-starlark-prompt.md @@ -8,8 +8,9 @@ in `git diff` against the project's Starlark coding rules and conventions: 2. Verify iterative algorithms are used (no recursion, no `while` loops). 3. Ensure every `.bzl` file outside `tests/` has a corresponding `bzl_library` target in its `BUILD` file with proper dependencies. -4. Ensure loads from `/private/` in test files have - `# buildifier: disable=bzl-visibility`. +4. Ensure loads from `/private/` in test files and internal non-`python/` + packages (e.g., `//python/private:distribution_filegroup.bzl` in `crates/` + or `tools/`) have `# buildifier: disable=bzl-visibility`. 5. Check multi-line rule/macro doc arguments: use triple-quoted strings (`"""`), and do NOT use trailing backslashes (`\`) on opening triple-quotes. 6. Verify analysis tests use `rules_testing` (not `bazel_skylib`), use diff --git a/AGENTS.md b/AGENTS.md index c762122d47..118bb366d0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -260,9 +260,11 @@ Dependabot alerts): When building `//docs:docs`, ignore an error about exit code 2; this is a flake, so try building again. -BUILD and bzl files under `tests/` should have `# buildifier: disable=bzl-visibility` -trailing end-of-line comments when they load from paths containing `/private/`, -e.g. +BUILD and bzl files under `tests/` (and internal non-`python/` packages such as +`crates/` and `tools/` loading repo-internal helpers like +`//python/private:distribution_filegroup.bzl`) should have +`# buildifier: disable=bzl-visibility` trailing end-of-line comments when they +load from paths containing `/private/`, e.g. ``` load("//python/private:foo.bzl", "foo") # buildifier: disable=bzl-visibility diff --git a/crates/Cargo.Bazel.lock b/crates/Cargo.Bazel.lock index dc3c029c33..4a1b392b40 100644 --- a/crates/Cargo.Bazel.lock +++ b/crates/Cargo.Bazel.lock @@ -1,5 +1,5 @@ { - "checksum": "bfd96f7f72752644dc2fe47b6acdae808884857b765d1a28115a0f4004754152", + "checksum": "269e4f7e54f04390547ffeddbf8945aa1cc32a2730cccd1792ad1ad16239b24b", "crates": { "adler2 2.0.1": { "name": "adler2", @@ -116,6 +116,103 @@ ], "license_file": "LICENSE-APACHE" }, + "argfile 1.0.0": { + "name": "argfile", + "version": "1.0.0", + "package_url": "https://github.com/rust-cli/argfile.git", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/argfile/1.0.0/download", + "sha256": "99489a733dea0d2930bfa59c243146a8513ce7b0991b9d006647687cc61f53e7" + } + }, + "targets": [ + { + "Library": { + "crate_name": "argfile", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "argfile", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "fs-err 3.3.2", + "target": "fs_err" + }, + { + "id": "os_str_bytes 7.2.1", + "target": "os_str_bytes" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "1.0.0" + }, + "license": "MIT OR Apache-2.0", + "license_ids": [ + "Apache-2.0", + "MIT" + ], + "license_file": "LICENSE-APACHE" + }, + "autocfg 1.5.1": { + "name": "autocfg", + "version": "1.5.1", + "package_url": "https://github.com/cuviper/autocfg", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/autocfg/1.5.1/download", + "sha256": "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + } + }, + "targets": [ + { + "Library": { + "crate_name": "autocfg", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "autocfg", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2015", + "version": "1.5.1" + }, + "license": "Apache-2.0 OR MIT", + "license_ids": [ + "Apache-2.0", + "MIT" + ], + "license_file": "LICENSE-APACHE" + }, "bitflags 2.13.2": { "name": "bitflags", "version": "2.13.2", @@ -686,6 +783,83 @@ ], "license_file": "LICENSE-APACHE" }, + "fs-err 3.3.2": { + "name": "fs-err", + "version": "3.3.2", + "package_url": "https://github.com/andrewhickman/fs-err", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/fs-err/3.3.2/download", + "sha256": "b5c95b673b8f6f7235229ae11c5642d81b04c2e64c1e2fb417bc0cf73ca45f29" + } + }, + "targets": [ + { + "Library": { + "crate_name": "fs_err", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "fs_err", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "extra_deps": { + "common": [ + ":build_script_build" + ], + "selects": {} + }, + "edition": "2018", + "version": "3.3.2" + }, + "build_script_attrs": { + "compile_data_glob": [ + "**" + ], + "compile_data_glob_excludes": [ + "**/*.rs" + ], + "data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "autocfg 1.5.1", + "target": "autocfg" + } + ], + "selects": {} + } + }, + "license": "MIT OR Apache-2.0", + "license_ids": [ + "Apache-2.0", + "MIT" + ], + "license_file": "LICENSE-APACHE" + }, "hashbrown 0.17.1": { "name": "hashbrown", "version": "0.17.1", @@ -1161,6 +1335,62 @@ ], "license_file": "LICENSE" }, + "os_str_bytes 7.2.1": { + "name": "os_str_bytes", + "version": "7.2.1", + "package_url": "https://github.com/dylni/os_str_bytes", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/os_str_bytes/7.2.1/download", + "sha256": "73b489f051c9d2f4299ddf8d55e33c65c86a7b76f49359af15f79ebe541d1595" + } + }, + "targets": [ + { + "Library": { + "crate_name": "os_str_bytes", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "os_str_bytes", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "memchr", + "raw_os_str" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "memchr 2.8.3", + "target": "memchr" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "7.2.1" + }, + "license": "MIT OR Apache-2.0", + "license_ids": [ + "Apache-2.0", + "MIT" + ], + "license_file": "LICENSE-APACHE" + }, "rules_python_crates 0.0.1": { "name": "rules_python_crates", "version": "0.0.1", @@ -1191,6 +1421,10 @@ "id": "anyhow 1.0.104", "target": "anyhow" }, + { + "id": "argfile 1.0.0", + "target": "argfile" + }, { "id": "flate2 1.1.11", "target": "flate2" @@ -1589,6 +1823,7 @@ }, "direct_deps": [ "anyhow 1.0.104", + "argfile 1.0.0", "flate2 1.1.11", "lexopt 0.3.2", "sha2 0.11.1", diff --git a/crates/Cargo.lock b/crates/Cargo.lock index a79004f93f..22e69799b1 100644 --- a/crates/Cargo.lock +++ b/crates/Cargo.lock @@ -14,6 +14,22 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "argfile" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99489a733dea0d2930bfa59c243146a8513ce7b0991b9d006647687cc61f53e7" +dependencies = [ + "fs-err", + "os_str_bytes", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + [[package]] name = "bitflags" version = "2.13.2" @@ -102,6 +118,15 @@ dependencies = [ "zlib-rs", ] +[[package]] +name = "fs-err" +version = "3.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5c95b673b8f6f7235229ae11c5642d81b04c2e64c1e2fb417bc0cf73ca45f29" +dependencies = [ + "autocfg", +] + [[package]] name = "hashbrown" version = "0.17.1" @@ -167,11 +192,21 @@ dependencies = [ "libc", ] +[[package]] +name = "os_str_bytes" +version = "7.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b489f051c9d2f4299ddf8d55e33c65c86a7b76f49359af15f79ebe541d1595" +dependencies = [ + "memchr", +] + [[package]] name = "rules_python_crates" version = "0.0.1" dependencies = [ "anyhow", + "argfile", "flate2", "lexopt", "sha2", diff --git a/crates/Cargo.toml b/crates/Cargo.toml index 991f833760..b739064512 100644 --- a/crates/Cargo.toml +++ b/crates/Cargo.toml @@ -15,6 +15,7 @@ path = "shrubbery/src/lib.rs" [dependencies] # keep-sorted start anyhow = "1.0.104" +argfile = "1.0.0" # Transitive dep of zip: zip's `deflate-flate2` leaves the backend unset, so a # top-level pin keeps flate2 on its default pure-Rust backend. flate2 = "1.1.11" diff --git a/crates/exe_zip_maker/src/lib.rs b/crates/exe_zip_maker/src/lib.rs index b7fc19ec22..b5c15fc8c8 100644 --- a/crates/exe_zip_maker/src/lib.rs +++ b/crates/exe_zip_maker/src/lib.rs @@ -7,6 +7,7 @@ use std::ffi::OsString; use std::fs::{self, File}; use std::io::{self, BufReader, BufWriter, Read, Write}; use std::path::{Path, PathBuf}; +use std::process::ExitCode; use sha2::{Digest, Sha256}; @@ -116,7 +117,7 @@ where /// Parses subcommand arguments (` `) and creates a self-executable zip /// archive, returning the process exit code. -pub fn run_cli(args: I) -> i32 +pub fn run_cli(args: I) -> ExitCode where I: IntoIterator, T: Into, @@ -125,7 +126,7 @@ where Ok(opts) => opts, Err(msg) => { eprintln!("exe_zip_maker: error: {msg}"); - return 1; + return ExitCode::FAILURE; } }; if let Err(e) = create_exe_zip( @@ -134,9 +135,9 @@ where &options.output_path, ) { eprintln!("exe_zip_maker: error: {e}"); - return 1; + return ExitCode::FAILURE; } - 0 + ExitCode::SUCCESS } /// Parsed command-line options for `exe_zip_maker`. diff --git a/crates/shrubbery/BUILD.bazel b/crates/shrubbery/BUILD.bazel index eba84f860c..4c7930f16f 100644 --- a/crates/shrubbery/BUILD.bazel +++ b/crates/shrubbery/BUILD.bazel @@ -11,6 +11,7 @@ rust_library( edition = "2024", deps = [ "//crates/exe_zip_maker:exe_zip_maker_lib", + "//crates/zip_main_maker:zip_main_maker_lib", "//crates/zipper:zipper_lib", ], ) diff --git a/crates/shrubbery/src/lib.rs b/crates/shrubbery/src/lib.rs index ef97e4593c..b8ef34a96c 100644 --- a/crates/shrubbery/src/lib.rs +++ b/crates/shrubbery/src/lib.rs @@ -5,12 +5,23 @@ #![deny(clippy::expect_used, clippy::unwrap_used)] use std::ffi::OsString; +use std::process::ExitCode; + +/// Subcommand name constants supported by `shrubbery`. +pub mod subcommands { + // keep-sorted start case=no + pub const EXE_ZIP_MAKER: &str = "exe_zip_maker"; + pub const ZIP_MAIN_MAKER: &str = "zip_main_maker"; + pub const ZIPPER: &str = "zipper"; + // keep-sorted end +} /// Canonical subcommand names supported by `shrubbery`. pub const SUBCOMMANDS: &[&str] = &[ - // keep-sorted start - "exe_zip_maker", - "zipper", + // keep-sorted start case=no + subcommands::EXE_ZIP_MAKER, + subcommands::ZIP_MAIN_MAKER, + subcommands::ZIPPER, // keep-sorted end ]; @@ -29,7 +40,7 @@ fn print_usage(prog_name: &str, to_stderr: bool) { /// Dispatches CLI arguments (including `argv[0]`) to the requested subcommand and returns the /// process exit code. -pub fn run_cli(args: I) -> i32 +pub fn run_cli(args: I) -> ExitCode where I: IntoIterator, T: Into, @@ -42,23 +53,24 @@ where let Some(sub_os) = iter.next() else { print_usage(&prog_name, true); - return 1; + return ExitCode::FAILURE; }; let sub_raw = sub_os.to_string_lossy(); if sub_raw == "-h" || sub_raw == "--help" || sub_raw == "help" { print_usage(&prog_name, false); - return 0; + return ExitCode::SUCCESS; } match sub_raw.as_ref() { - // keep-sorted start - "exe_zip_maker" => exe_zip_maker_lib::run_cli(iter), - "zipper" => zipper_lib::run_cli(iter), + // keep-sorted start case=no + subcommands::EXE_ZIP_MAKER => exe_zip_maker_lib::run_cli(iter), + subcommands::ZIP_MAIN_MAKER => zip_main_maker_lib::run_cli(iter), + subcommands::ZIPPER => zipper_lib::run_cli(iter), // keep-sorted end _ => { eprintln!("shrubbery: error: unknown subcommand '{}'", sub_raw); print_usage(&prog_name, true); - 1 + ExitCode::FAILURE } } } diff --git a/crates/shrubbery/src/main.rs b/crates/shrubbery/src/main.rs index 5399c67ddc..522d7eb088 100644 --- a/crates/shrubbery/src/main.rs +++ b/crates/shrubbery/src/main.rs @@ -2,8 +2,8 @@ #![deny(clippy::expect_used, clippy::unwrap_used)] use std::env; -use std::process; +use std::process::ExitCode; -fn main() { - process::exit(shrubbery_lib::run_cli(env::args_os())); +fn main() -> ExitCode { + shrubbery_lib::run_cli(env::args_os()) } diff --git a/crates/zip_main_maker/BUILD.bazel b/crates/zip_main_maker/BUILD.bazel new file mode 100644 index 0000000000..26860bc290 --- /dev/null +++ b/crates/zip_main_maker/BUILD.bazel @@ -0,0 +1,23 @@ +load("@rules_rust//rust:defs.bzl", "rust_library") +load("//python/private:distribution_filegroup.bzl", "distribution_filegroup") # buildifier: disable=bzl-visibility + +package(default_visibility = ["//:__subpackages__"]) + +licenses(["notice"]) + +rust_library( + name = "zip_main_maker_lib", + srcs = ["src/lib.rs"], + edition = "2024", + deps = [ + "//crates/zip_manifest", + "@crates//:anyhow", + "@crates//:argfile", + "@crates//:lexopt", + "@crates//:sha2", + ], +) + +distribution_filegroup( + name = "distribution", +) diff --git a/crates/zip_main_maker/src/lib.rs b/crates/zip_main_maker/src/lib.rs new file mode 100644 index 0000000000..b10cebfaaf --- /dev/null +++ b/crates/zip_main_maker/src/lib.rs @@ -0,0 +1,260 @@ +//! Library for creating the `__main__.py` for a zipapp by populating a template. +//! +//! This library also calculates a SHA-256 hash of the application files from a manifest to include +//! in the template, which allows making the extraction directory unique to the content of the +//! zipapp. + +// Errors must be returned, not panicked; see the `rust-lint` config in .bazelrc. +#![deny(clippy::expect_used, clippy::unwrap_used)] + +use std::collections::HashMap; +use std::ffi::OsString; +use std::fs::{self, File}; +use std::io::Read; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; + +use anyhow::{Context, Result, bail}; +use lexopt::prelude::*; +use sha2::{Digest, Sha256}; +use zip_manifest::{SymlinkMode, parse_manifest_line}; + +/// Read buffer size when streaming file contents into SHA-256 (matches `zip_main_maker.py`). +const BLOCK_SIZE: usize = 256 * 1024; + +/// Template placeholder replaced with the computed SHA-256 hex digest of the application inputs. +pub const APP_HASH_PLACEHOLDER: &str = "%APP_HASH%"; + +mod flags { + // keep-sorted start + pub const HASH_FILES_MANIFEST: &str = "hash_files_manifest"; + pub const HELP: &str = "help"; + pub const HELP_SHORT: char = 'h'; + pub const OUTPUT: &str = "output"; + pub const SUBSTITUTION: &str = "substitution"; + pub const TEMPLATE: &str = "template"; + // keep-sorted end +} + +/// Usage text for `--help`. `lexopt` does not generate help, so this is maintained by hand. +const USAGE: &str = "\ +Usage: zip_main_maker [OPTIONS] --template --output --hash_files_manifest + +Create the __main__.py for a zipapp from a template and input manifest. + +Options: + --template Path to the __main__.py template file + --output Path to the generated __main__.py output file + --substitution Template substitution (may be passed multiple times) + --hash_files_manifest File containing lines in rf-XXX formats (rf-empty, rf-file, rf-symlink, etc.) + @ Read newline-delimited arguments from + -h, --help Print this help"; + +/// Computes the SHA-256 lowercase hex digest of the application inputs listed in `manifest_path`. +/// +/// Manifest lines are parsed via [`parse_manifest_line`] and sorted lexicographically for +/// determinism; for each line, the text after the leading `|` prefix is hashed to capture +/// the runfiles structure, followed by either the symlink target UTF-8 bytes or the file's raw +/// content bytes. +pub fn compute_inputs_hash(manifest_path: &Path) -> Result { + let content = fs::read_to_string(manifest_path) + .with_context(|| format!("reading manifest {}", manifest_path.display()))?; + let mut manifest_lines: Vec<&str> = content.lines().collect(); + + // Sort lines for determinism. Hash the paths (to capture structure) and the content. + manifest_lines.sort_unstable(); + + let mut hasher = Sha256::new(); + let mut buffer = vec![0u8; BLOCK_SIZE]; + + for line in manifest_lines { + let parsed = parse_manifest_line(line) + .with_context(|| format!("in manifest {}", manifest_path.display()))?; + hasher.update(parsed.rest.as_bytes()); + + // For `rf-empty` and `symlink`, the paths (and symlink target) are captured by hashing + // `parsed.rest` above. + let Some((symlink_mode, path)) = parsed.entry.content_source() else { + continue; + }; + + let is_symlink = match symlink_mode { + SymlinkMode::Regular => false, + SymlinkMode::Symlink => true, + SymlinkMode::Infer => !path.exists(), + }; + + if is_symlink { + let link_target = fs::read_link(path) + .with_context(|| format!("reading symlink {}", path.display()))?; + hasher.update(link_target.to_string_lossy().as_bytes()); + } else { + let mut file = + File::open(path).with_context(|| format!("opening {}", path.display()))?; + loop { + let n = file + .read(&mut buffer) + .with_context(|| format!("hashing {}", path.display()))?; + if n == 0 { + break; + } + hasher.update(&buffer[..n]); + } + } + } + + let digest = hasher.finalize(); + Ok(digest.iter().map(|b| format!("{b:02x}")).collect()) +} + +/// Reads `template_path`, applies `substitutions`, and writes the result to `output_path`. +pub fn expand_template( + template_path: &Path, + output_path: &Path, + substitutions: &HashMap<&str, &str>, +) -> Result<()> { + let mut content = fs::read_to_string(template_path) + .with_context(|| format!("reading template {}", template_path.display()))?; + + for (&key, &val) in substitutions { + content = content.replace(key, val); + } + + if let Some(parent) = output_path.parent() + && !parent.as_os_str().is_empty() + { + fs::create_dir_all(parent) + .with_context(|| format!("creating directory {}", parent.display()))?; + } + + fs::write(output_path, content) + .with_context(|| format!("writing output {}", output_path.display())) +} + +/// Generates `__main__.py` according to `options`. +pub fn generate_zip_main(options: &Options) -> Result<()> { + let app_hash = compute_inputs_hash(&options.hash_files_manifest)?; + let mut substitutions = HashMap::from([(APP_HASH_PLACEHOLDER, app_hash.as_str())]); + for (key, val) in &options.substitutions { + substitutions.insert(key, val); + } + expand_template(&options.template, &options.output, &substitutions) +} + +fn parse_substitution(raw: &str) -> std::result::Result<(String, String), String> { + let (key, val) = raw.split_once('=').ok_or_else(|| { + format!( + "invalid --{} value (expected KEY=VALUE): {raw}", + flags::SUBSTITUTION + ) + })?; + Ok((key.to_string(), val.to_string())) +} + +/// Parses command-line arguments (excluding `argv[0]` and subcommand name) into [`Cli`]. +pub fn parse_args(args: I) -> Result +where + I: IntoIterator, + T: Into, +{ + let expanded = argfile::expand_args_from( + args.into_iter().map(Into::into), + argfile::parse_fromfile, + argfile::PREFIX, + ) + .context("reading @flagfile")?; + + let mut template: Option = None; + let mut output: Option = None; + let mut substitutions: HashMap = HashMap::new(); + let mut hash_files_manifest: Option = None; + + let mut parser = lexopt::Parser::from_args(expanded); + while let Some(arg) = parser.next()? { + match arg { + Short(flags::HELP_SHORT) | Long(flags::HELP) => return Ok(Cli::Help), + Long(flags::TEMPLATE) => { + template = Some(PathBuf::from(parser.value()?)); + } + Long(flags::OUTPUT) => { + output = Some(PathBuf::from(parser.value()?)); + } + Long(flags::SUBSTITUTION) => { + let (key, val) = parser.value()?.parse_with(parse_substitution)?; + substitutions.insert(key, val); + } + Long(flags::HASH_FILES_MANIFEST) => { + hash_files_manifest = Some(PathBuf::from(parser.value()?)); + } + _ => bail!("{}", arg.unexpected()), + } + } + + let template = + template.with_context(|| format!("missing required argument: --{}", flags::TEMPLATE))?; + let output = + output.with_context(|| format!("missing required argument: --{}", flags::OUTPUT))?; + let hash_files_manifest = hash_files_manifest.with_context(|| { + format!( + "missing required argument: --{}", + flags::HASH_FILES_MANIFEST + ) + })?; + + Ok(Cli::Run(Options { + template, + output, + substitutions, + hash_files_manifest, + })) +} + +/// Parses CLI arguments (excluding `argv[0]` and subcommand name), generates `__main__.py`, and +/// returns the process exit code. +pub fn run_cli(args: I) -> ExitCode +where + I: IntoIterator, + T: Into, +{ + let options = match parse_args(args) { + Ok(Cli::Help) => { + println!("{USAGE}"); + return ExitCode::SUCCESS; + } + Ok(Cli::Run(opts)) => opts, + Err(err) => { + eprintln!("zip_main_maker: error: {err:#}"); + return ExitCode::FAILURE; + } + }; + if let Err(err) = generate_zip_main(&options) { + eprintln!("zip_main_maker: error: {err:#}"); + return ExitCode::FAILURE; + } + ExitCode::SUCCESS +} + +/// Result of parsing the command line. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Cli { + /// `--help` was requested. + Help, + /// Generate `__main__.py` with the given options. + Run(Options), +} + +/// Parsed command-line options for `zip_main_maker`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Options { + /// Path to the `__main__.py` template file, relative to the current working directory or + /// absolute. + pub template: PathBuf, + /// Path to the generated `__main__.py` output file, relative to the current working directory + /// or absolute. + pub output: PathBuf, + /// `(key, value)` template substitutions passed via `--substitution`. + pub substitutions: HashMap, + /// Path to the input manifest file listing application files to hash, relative to the current + /// working directory or absolute. + pub hash_files_manifest: PathBuf, +} diff --git a/crates/zip_manifest/BUILD.bazel b/crates/zip_manifest/BUILD.bazel new file mode 100644 index 0000000000..b21b0af2c7 --- /dev/null +++ b/crates/zip_manifest/BUILD.bazel @@ -0,0 +1,16 @@ +load("@rules_rust//rust:defs.bzl", "rust_library") +load("//python/private:distribution_filegroup.bzl", "distribution_filegroup") # buildifier: disable=bzl-visibility + +package(default_visibility = ["//:__subpackages__"]) + +licenses(["notice"]) + +rust_library( + name = "zip_manifest", + srcs = ["src/lib.rs"], + edition = "2024", +) + +distribution_filegroup( + name = "distribution", +) diff --git a/crates/zip_manifest/src/lib.rs b/crates/zip_manifest/src/lib.rs new file mode 100644 index 0000000000..4a54736188 --- /dev/null +++ b/crates/zip_manifest/src/lib.rs @@ -0,0 +1,274 @@ +//! Parser for the `|`-delimited manifest format shared by `zipper` and `zip_main_maker`. + +// Errors must be returned, not panicked; see the `rust-lint` config in .bazelrc. +#![deny(clippy::expect_used, clippy::unwrap_used)] + +use std::fmt; +use std::path::Path; + +mod symlink_modes { + // keep-sorted start + pub const INFER: &str = "-1"; + pub const REGULAR: &str = "0"; + pub const SYMLINK: &str = "1"; + // keep-sorted end +} + +mod entry_kinds { + // keep-sorted start + pub const REGULAR: &str = "regular"; + pub const RF_EMPTY: &str = "rf-empty"; + pub const RF_FILE: &str = "rf-file"; + pub const RF_ROOT_SYMLINK: &str = "rf-root-symlink"; + pub const RF_SYMLINK: &str = "rf-symlink"; + pub const SYMLINK: &str = "symlink"; + // keep-sorted end +} + +/// Parses the `is_symlink` field (`"0"`, `"1"`, or `"-1"`) of a manifest line. +pub fn parse_symlink_mode(value: &str) -> Result { + match value { + symlink_modes::REGULAR => Ok(SymlinkMode::Regular), + symlink_modes::SYMLINK => Ok(SymlinkMode::Symlink), + symlink_modes::INFER => Ok(SymlinkMode::Infer), + _ => Err(ManifestParseError::InvalidSymlinkMode { + value: value.to_string(), + }), + } +} + +fn split_two(s: &str) -> Option<(&str, &str)> { + s.split_once('|').filter(|(_, b)| !b.contains('|')) +} + +fn split_three(s: &str) -> Option<(&str, &str, &str)> { + let (a, rest) = s.split_once('|')?; + let (b, c) = split_two(rest)?; + Some((a, b, c)) +} + +/// Parses a single non-empty manifest line into [`ManifestLine`]. +/// +/// Lines have one of the following `|`-separated formats: +/// +/// * `regular|is_symlink|zip_path|content_path`: Stores `zip_path` at the exact +/// zip-root-relative path given, reading its content (or symlink target) from `content_path`. +/// * `rf-empty|runfile_path`: A `runfiles.empty_filenames` entry (such as an implicit +/// `__init__.py`) stored as an empty file under `/`. +/// * `rf-file|is_symlink|runfile_path|content_path`: A `runfiles.files` entry whose destination +/// zip path is computed under `` from `runfile_path` (accounting for +/// `workspace_name` and `legacy_external_runfiles`), reading content from `content_path`. +/// * `rf-symlink|is_symlink|runfile_symlink_path|content_path`: A `runfiles.symlinks` entry +/// stored under `//`, reading content +/// from `content_path`. +/// * `rf-root-symlink|is_symlink|runfile_root_path|content_path`: A `runfiles.root_symlinks` +/// entry stored under `/`, reading content from +/// `content_path`. +/// * `symlink|runfile_root_path|link_to_rf_path`: An explicit symlink entry stored at +/// `/` whose symlink target is computed as the relative path +/// from its parent directory to `/`. Takes precedence over +/// non-symlink entries at the same `zip_path`. +/// +/// See [`SymlinkMode`] for the valid `is_symlink` values. +pub fn parse_manifest_line(line: &str) -> Result, ManifestParseError> { + let invalid = || ManifestParseError::InvalidFormat { + line: line.to_string(), + }; + let Some((kind, rest)) = line.split_once('|') else { + return Err(invalid()); + }; + let entry = match kind { + entry_kinds::REGULAR => { + let (is_symlink, zip_path, content_path) = split_three(rest).ok_or_else(invalid)?; + RawManifestEntry::Regular { + symlink_mode: parse_symlink_mode(is_symlink)?, + zip_path, + content_path: Path::new(content_path), + } + } + entry_kinds::RF_EMPTY => { + if rest.contains('|') { + return Err(invalid()); + } + RawManifestEntry::RfEmpty { runfile_path: rest } + } + entry_kinds::RF_FILE => { + let (is_symlink, runfile_path, content_path) = split_three(rest).ok_or_else(invalid)?; + RawManifestEntry::RfFile { + symlink_mode: parse_symlink_mode(is_symlink)?, + runfile_path, + content_path: Path::new(content_path), + } + } + entry_kinds::RF_SYMLINK => { + let (is_symlink, runfile_symlink_path, content_path) = + split_three(rest).ok_or_else(invalid)?; + RawManifestEntry::RfSymlink { + symlink_mode: parse_symlink_mode(is_symlink)?, + runfile_symlink_path, + content_path: Path::new(content_path), + } + } + entry_kinds::RF_ROOT_SYMLINK => { + let (is_symlink, runfile_root_path, content_path) = + split_three(rest).ok_or_else(invalid)?; + RawManifestEntry::RfRootSymlink { + symlink_mode: parse_symlink_mode(is_symlink)?, + runfile_root_path, + content_path: Path::new(content_path), + } + } + entry_kinds::SYMLINK => { + let (runfile_root_path, link_to_runfile_root_path) = + split_two(rest).ok_or_else(invalid)?; + RawManifestEntry::Symlink { + runfile_root_path, + link_to_runfile_root_path, + } + } + _ => return Err(invalid()), + }; + Ok(ManifestLine { rest, entry }) +} + +/// How an entry's `content_path` should be handled with respect to symlinks, parsed from the +/// `is_symlink` manifest field. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SymlinkMode { + /// `"0"`: Source is marked as a regular file at analysis time (note: `zipper` still preserves + /// relative source symlinks when inspecting `content_path` on disk). + Regular, + /// `"1"`: Source is a symlink whose target is read via `std::fs::read_link`. + Symlink, + /// `"-1"`: Unknown at analysis time (Bazel 7 lacks `File.is_symlink`); treated as `Symlink` + /// when `content_path` is a dangling symlink (`std::fs::metadata` fails) and `Regular` + /// otherwise. + Infer, +} + +/// A parsed line from a zipapp manifest. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ManifestLine<'a> { + /// Raw substring after the leading `|` delimiter (hashed by `zip_main_maker` to capture + /// the runfiles structure). + pub rest: &'a str, + /// Structured representation of the manifest entry. + pub entry: RawManifestEntry<'a>, +} + +/// Structured representation of a single `|`-delimited manifest entry before zip-path resolution. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RawManifestEntry<'a> { + /// `regular|is_symlink|zip_path|content_path` + Regular { + symlink_mode: SymlinkMode, + /// Destination path inside the zip archive, relative to the zip root. + zip_path: &'a str, + /// Source file or symlink path on disk, relative to the current working directory or + /// absolute. + content_path: &'a Path, + }, + /// `rf-empty|runfile_path` + RfEmpty { + /// Runfile `short_path` (relative to the main workspace root, using `..//` or + /// `external//` prefixes for external repositories). + runfile_path: &'a str, + }, + /// `rf-file|is_symlink|runfile_path|content_path` + RfFile { + symlink_mode: SymlinkMode, + /// Runfile `short_path` (relative to the main workspace root, using `..//` or + /// `external//` prefixes for external repositories). + runfile_path: &'a str, + /// Source file or symlink path on disk, relative to the current working directory or + /// absolute. + content_path: &'a Path, + }, + /// `rf-symlink|is_symlink|runfile_symlink_path|content_path` + RfSymlink { + symlink_mode: SymlinkMode, + /// Destination symlink path relative to `/`. + runfile_symlink_path: &'a str, + /// Source file or symlink path on disk, relative to the current working directory or + /// absolute. + content_path: &'a Path, + }, + /// `rf-root-symlink|is_symlink|runfile_root_path|content_path` + RfRootSymlink { + symlink_mode: SymlinkMode, + /// Destination symlink path relative to ``. + runfile_root_path: &'a str, + /// Source file or symlink path on disk, relative to the current working directory or + /// absolute. + content_path: &'a Path, + }, + /// `symlink|runfile_root_path|link_to_runfile_root_path` + Symlink { + /// Destination symlink path relative to ``. + runfile_root_path: &'a str, + /// Symlink target path relative to ``. + link_to_runfile_root_path: &'a str, + }, +} + +impl<'a> RawManifestEntry<'a> { + /// Returns `(symlink_mode, content_path)` (where `content_path` is relative to the current + /// working directory or absolute) for entry kinds backed by a file or symlink on disk + /// (`regular`, `rf-file`, `rf-symlink`, `rf-root-symlink`), or `None` for metadata-only + /// entries (`rf-empty`, `symlink`). + pub fn content_source(&self) -> Option<(SymlinkMode, &'a Path)> { + match *self { + Self::Regular { + symlink_mode, + content_path, + .. + } + | Self::RfFile { + symlink_mode, + content_path, + .. + } + | Self::RfSymlink { + symlink_mode, + content_path, + .. + } + | Self::RfRootSymlink { + symlink_mode, + content_path, + .. + } => Some((symlink_mode, content_path)), + Self::RfEmpty { .. } | Self::Symlink { .. } => None, + } + } +} + +/// Error returned when a manifest line fails syntactic validation. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ManifestParseError { + /// Line has an unknown entry type or wrong number of `|`-delimited fields. + InvalidFormat { line: String }, + /// The `is_symlink` field is not `"0"`, `"1"`, or `"-1"`. + InvalidSymlinkMode { value: String }, +} + +impl fmt::Display for ManifestParseError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InvalidFormat { line } => { + write!(f, "unknown entry type or invalid format: {line}") + } + Self::InvalidSymlinkMode { value } => { + write!( + f, + "invalid is_symlink value (expected {}, {}, or {}): {value}", + symlink_modes::REGULAR, + symlink_modes::SYMLINK, + symlink_modes::INFER, + ) + } + } + } +} + +impl std::error::Error for ManifestParseError {} diff --git a/crates/zipper/BUILD.bazel b/crates/zipper/BUILD.bazel index 344ee2f527..1d30f8d322 100644 --- a/crates/zipper/BUILD.bazel +++ b/crates/zipper/BUILD.bazel @@ -10,6 +10,7 @@ rust_library( srcs = ["src/lib.rs"], edition = "2024", deps = [ + "//crates/zip_manifest", "@crates//:anyhow", "@crates//:lexopt", "@crates//:zip", diff --git a/crates/zipper/src/lib.rs b/crates/zipper/src/lib.rs index b3a847f7ec..c89cfd6206 100644 --- a/crates/zipper/src/lib.rs +++ b/crates/zipper/src/lib.rs @@ -8,10 +8,12 @@ use std::ffi::OsString; use std::fs::{self, File, Metadata}; use std::io::{self, BufWriter, Seek, Write}; use std::path::{Path, PathBuf}; +use std::process::ExitCode; use anyhow::{Context, Result, bail}; use zip::write::SimpleFileOptions; use zip::{CompressionMethod, DateTime, System, ZIP64_BYTES_THR, ZipWriter}; +use zip_manifest::{RawManifestEntry, SymlinkMode, parse_manifest_line}; /// `BufWriter` capacity for the output zip. /// @@ -28,7 +30,7 @@ Create a zip file from a manifest. Arguments: Path to the manifest file. Line formats are documented on - `parse_entry` in crates/zipper/src/lib.rs. + `parse_manifest_line` in crates/zip_manifest/src/lib.rs. Path to the output zip file Options: @@ -131,39 +133,10 @@ fn normalize_zip_path(path: &str) -> String { path.replace('\\', "/") } -/// Parses the `is_symlink` field of a manifest line. -fn parse_symlink_mode(value: &str) -> Result { - match value { - "0" => Ok(SymlinkMode::Regular), - "1" => Ok(SymlinkMode::Symlink), - "-1" => Ok(SymlinkMode::Infer), - _ => bail!("invalid is_symlink value (expected 0, 1, or -1): {value}"), - } -} - -/// Parses a single line of a manifest file. Returns `None` for blank lines. -/// -/// Lines have one of the following `|`-separated formats: +/// Parses a single line of a manifest file and resolves its destination zip path. Returns `None` +/// for blank lines. /// -/// * `regular|is_symlink|zip_path|content_path`: Stores `zip_path` at the exact -/// zip-root-relative path given, reading its content (or symlink target) from `content_path`. -/// * `rf-empty|runfile_path`: A `runfiles.empty_filenames` entry (such as an implicit -/// `__init__.py`) stored as an empty file under `/`. -/// * `rf-file|is_symlink|runfile_path|content_path`: A `runfiles.files` entry whose destination -/// zip path is computed under `` from `runfile_path` (accounting for -/// `workspace_name` and `legacy_external_runfiles`), reading content from `content_path`. -/// * `rf-symlink|is_symlink|runfile_symlink_path|content_path`: A `runfiles.symlinks` entry -/// stored under `//`, reading content -/// from `content_path`. -/// * `rf-root-symlink|is_symlink|runfile_root_path|content_path`: A `runfiles.root_symlinks` -/// entry stored under `/`, reading content from -/// `content_path`. -/// * `symlink|runfile_root_path|link_to_rf_path`: An explicit symlink entry stored at -/// `/` whose symlink target is computed as the relative path -/// from its parent directory to `/`. Takes precedence over -/// non-symlink entries at the same `zip_path`. -/// -/// See [`SymlinkMode`] for the valid `is_symlink` values. +/// Line formats are documented on [`parse_manifest_line`]. fn parse_entry( line: &str, workspace_name: &str, @@ -175,13 +148,9 @@ fn parse_entry( return Ok(None); } - // Closures (not values): `file(...)` builds a `File` entry and `rf_path(...)` computes a - // runfiles zip path. They capture the function arguments so the match arms stay short. - let file = |content_path: &str, is_symlink: &str| -> Result { - Ok(EntryKind::File { - content_path: PathBuf::from(content_path), - symlink_mode: parse_symlink_mode(is_symlink)?, - }) + let file = |content_path: &Path, symlink_mode: SymlinkMode| EntryKind::File { + content_path: content_path.to_path_buf(), + symlink_mode, }; let rf_path = |runfile_path: &str| { get_zip_runfiles_path( @@ -192,31 +161,49 @@ fn parse_entry( ) }; - let parts: Vec<&str> = line.split('|').collect(); - let entry = match parts.as_slice() { - ["regular", is_symlink, zip_path, content_path] => ManifestEntry { + let entry = match parse_manifest_line(line)?.entry { + RawManifestEntry::Regular { + symlink_mode, + zip_path, + content_path, + } => ManifestEntry { zip_path: zip_path.to_string(), - kind: file(content_path, is_symlink)?, + kind: file(content_path, symlink_mode), }, - ["rf-empty", runfile_path] => ManifestEntry { + RawManifestEntry::RfEmpty { runfile_path } => ManifestEntry { zip_path: rf_path(runfile_path), kind: EntryKind::Empty, }, - ["rf-file", is_symlink, runfile_path, content_path] => ManifestEntry { + RawManifestEntry::RfFile { + symlink_mode, + runfile_path, + content_path, + } => ManifestEntry { zip_path: rf_path(runfile_path), - kind: file(content_path, is_symlink)?, + kind: file(content_path, symlink_mode), }, - ["rf-symlink", is_symlink, runfile_path, content_path] => ManifestEntry { - zip_path: unix_join(&[runfiles_dir, workspace_name, runfile_path]), - kind: file(content_path, is_symlink)?, + RawManifestEntry::RfSymlink { + symlink_mode, + runfile_symlink_path, + content_path, + } => ManifestEntry { + zip_path: unix_join(&[runfiles_dir, workspace_name, runfile_symlink_path]), + kind: file(content_path, symlink_mode), }, - ["rf-root-symlink", is_symlink, runfile_path, content_path] => ManifestEntry { - zip_path: unix_join(&[runfiles_dir, runfile_path]), - kind: file(content_path, is_symlink)?, + RawManifestEntry::RfRootSymlink { + symlink_mode, + runfile_root_path, + content_path, + } => ManifestEntry { + zip_path: unix_join(&[runfiles_dir, runfile_root_path]), + kind: file(content_path, symlink_mode), }, - ["symlink", runfile_path, link_to_rf_path] => { - let zip_path = unix_join(&[runfiles_dir, runfile_path]); - let link_to = unix_join(&[runfiles_dir, link_to_rf_path]); + RawManifestEntry::Symlink { + runfile_root_path, + link_to_runfile_root_path, + } => { + let zip_path = unix_join(&[runfiles_dir, runfile_root_path]); + let link_to = unix_join(&[runfiles_dir, link_to_runfile_root_path]); let parent = zip_path.rsplit_once('/').map_or("", |(dir, _)| dir); let target = relative_path(&link_to, parent); ManifestEntry { @@ -224,7 +211,6 @@ fn parse_entry( kind: EntryKind::Symlink { target }, } } - _ => bail!("unknown entry type or invalid format: {line}"), }; Ok(Some(entry)) @@ -534,7 +520,7 @@ where /// Parses CLI arguments (excluding `argv[0]` and subcommand name), creates the zip archive, and /// returns the process exit code. -pub fn run_cli(args: I) -> i32 +pub fn run_cli(args: I) -> ExitCode where I: IntoIterator, T: Into, @@ -542,19 +528,19 @@ where let options = match parse_args(args) { Ok(Cli::Help) => { println!("{USAGE}"); - return 0; + return ExitCode::SUCCESS; } Ok(Cli::Run(opts)) => opts, Err(msg) => { eprintln!("zipper: error: {msg}"); - return 1; + return ExitCode::FAILURE; } }; if let Err(err) = create_zip(&options) { eprintln!("zipper: error: {err:#}"); - return 1; + return ExitCode::FAILURE; } - 0 + ExitCode::SUCCESS } /// Result of parsing the command line. @@ -566,21 +552,7 @@ pub enum Cli { Run(ZipOptions), } -/// How a `File` entry's `content_path` should be stored with respect to symlinks. Parsed from -/// the `is_symlink` manifest field. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum SymlinkMode { - /// `"0"`: Store as a regular file read from `content_path` (unless `content_path` itself or - /// its sandbox target is a relative source symlink). - Regular, - /// `"1"`: Store as a symlink whose target is read via `read_link(content_path)`. - Symlink, - /// `"-1"`: Unknown at analysis time (Bazel 7 lacks `File.is_symlink`); treated as `Symlink` - /// when `content_path` is a dangling symlink (`fs::metadata` fails) and `Regular` otherwise. - Infer, -} - -/// What a manifest entry stores. See [`parse_entry`] for the manifest line formats. +/// What a manifest entry stores. See [`parse_manifest_line`] for the manifest line formats. #[derive(Debug, Clone, PartialEq, Eq)] enum EntryKind { /// An empty regular file (`rf-empty`). diff --git a/python/private/zipapp/py_zipapp_rule.bzl b/python/private/zipapp/py_zipapp_rule.bzl index be00b04264..425bbef059 100644 --- a/python/private/zipapp/py_zipapp_rule.bzl +++ b/python/private/zipapp/py_zipapp_rule.bzl @@ -67,10 +67,16 @@ def _create_zipapp_main_py(ctx, py_runtime, py_executable, stage2_bootstrap, run inputs.add(py_runtime.zip_main_template) _build_manifest(ctx, hash_files_manifest, runfiles, explicit_symlinks, inputs) + zip_main_maker = get_shrubbery_tool( + ctx, + "zip_main_maker", + ctx.attr._zip_main_maker, + ) actions_run( ctx, - executable = ctx.attr._zip_main_maker, - arguments = [args, hash_files_manifest], + executable = zip_main_maker.executable, + toolchain = zip_main_maker.toolchain, + arguments = zip_main_maker.prefix_args + [args, hash_files_manifest], inputs = inputs.build(), outputs = [zip_main_py], mnemonic = "PyZipAppCreateMainPy", diff --git a/tests/exe_zip_maker/exe_zip_maker_test.rs b/tests/exe_zip_maker/exe_zip_maker_test.rs index 141b4441fe..4aef612dfa 100644 --- a/tests/exe_zip_maker/exe_zip_maker_test.rs +++ b/tests/exe_zip_maker/exe_zip_maker_test.rs @@ -1,6 +1,7 @@ use std::env; use std::fs; use std::path::PathBuf; +use std::process::ExitCode; use exe_zip_maker_lib::{ Options, PLACEHOLDER, compute_file_sha256_hex, create_exe_zip, parse_args, replace_bytes, @@ -180,14 +181,14 @@ fn test_parse_args_and_run_cli() { .expect("preamble_path should be writable"); fs::write(&zip_path, b"PK\x03\x04dummy").expect("zip_path should be writable"); - assert_eq!(run_cli(["missing_args"]), 1); + assert_eq!(run_cli(["missing_args"]), ExitCode::FAILURE); assert_eq!( run_cli([ preamble_path.as_os_str(), zip_path.as_os_str(), output_path.as_os_str(), ]), - 0 + ExitCode::SUCCESS ); assert!(output_path.exists()); diff --git a/tests/integration/prebuilt_toolchain_test.py b/tests/integration/prebuilt_toolchain_test.py index 1c61188f2b..b847bdb602 100644 --- a/tests/integration/prebuilt_toolchain_test.py +++ b/tests/integration/prebuilt_toolchain_test.py @@ -80,7 +80,13 @@ def test_prebuilt_shrubbery_toolchain(self): "//:zipapp_test", ) - for mnemonic in ("PyZipAppCreateExecutableZip", "PyZipAppCreateZip"): + for mnemonic in ( + # keep-sorted start + "PyZipAppCreateExecutableZip", + "PyZipAppCreateMainPy", + "PyZipAppCreateZip", + # keep-sorted end + ): with self.subTest(mnemonic=mnemonic): aquery_res = self.run_bazel( "aquery", diff --git a/tests/shrubbery/shrubbery_test.rs b/tests/shrubbery/shrubbery_test.rs index 8d94aabde4..97496e2ac9 100644 --- a/tests/shrubbery/shrubbery_test.rs +++ b/tests/shrubbery/shrubbery_test.rs @@ -1,7 +1,7 @@ use std::env; use std::fs; use std::path::PathBuf; -use std::process::Command; +use std::process::{Command, ExitCode}; use shrubbery_lib::{SUBCOMMANDS, run_cli}; @@ -17,7 +17,7 @@ fn temp_test_dir(prefix: &str) -> PathBuf { #[test] fn test_subcommands_registry_sorted_and_complete() { - assert_eq!(SUBCOMMANDS.len(), 2); + assert_eq!(SUBCOMMANDS.len(), 3); let mut sorted = SUBCOMMANDS.to_vec(); sorted.sort_unstable(); assert_eq!(SUBCOMMANDS, sorted.as_slice()); @@ -25,15 +25,15 @@ fn test_subcommands_registry_sorted_and_complete() { #[test] fn test_help_flags_exit_zero() { - assert_eq!(run_cli(["shrubbery", "--help"]), 0); - assert_eq!(run_cli(["shrubbery", "-h"]), 0); - assert_eq!(run_cli(["shrubbery", "help"]), 0); + assert_eq!(run_cli(["shrubbery", "--help"]), ExitCode::SUCCESS); + assert_eq!(run_cli(["shrubbery", "-h"]), ExitCode::SUCCESS); + assert_eq!(run_cli(["shrubbery", "help"]), ExitCode::SUCCESS); } #[test] fn test_missing_or_unknown_subcommand_exits_one() { - assert_eq!(run_cli(["shrubbery"]), 1); - assert_eq!(run_cli(["shrubbery", "does_not_exist"]), 1); + assert_eq!(run_cli(["shrubbery"]), ExitCode::FAILURE); + assert_eq!(run_cli(["shrubbery", "does_not_exist"]), ExitCode::FAILURE); } #[test] @@ -58,9 +58,30 @@ fn test_subcommand_forwarding() { zip_in.to_str().expect("zip_in path should be UTF-8"), out_pyz.to_str().expect("out_pyz path should be UTF-8"), ]), - 0 + ExitCode::SUCCESS ); assert!(out_pyz.exists()); + + let tpl = dir.join("tpl.py"); + let manifest = dir.join("manifest.txt"); + let out_main = dir.join("__main__.py"); + fs::write(&tpl, "HASH=%APP_HASH%\n").expect("tpl should be writable"); + fs::write(&manifest, "rf-empty|__init__.py\n").expect("manifest should be writable"); + + assert_eq!( + run_cli([ + "shrubbery", + "zip_main_maker", + "--template", + tpl.to_str().expect("tpl path should be UTF-8"), + "--output", + out_main.to_str().expect("out_main path should be UTF-8"), + "--hash_files_manifest", + manifest.to_str().expect("manifest path should be UTF-8"), + ]), + ExitCode::SUCCESS + ); + assert!(out_main.exists()); } #[test] @@ -77,6 +98,15 @@ fn test_cli_binary_subprocess() { let stdout = String::from_utf8_lossy(&help_out.stdout); for sub in SUBCOMMANDS { assert!(stdout.contains(sub), "missing {} in help: {}", sub, stdout); + let sub_help = Command::new(&resolved) + .args([*sub, "--help"]) + .output() + .expect("registered subcommand should run"); + let sub_stderr = String::from_utf8_lossy(&sub_help.stderr); + assert!( + !sub_stderr.contains("unknown subcommand"), + "registered subcommand {sub} was rejected by run_cli: {sub_stderr}" + ); } let bad_out = Command::new(&resolved) diff --git a/tests/shrubbery_toolchain/shrubbery_toolchain_tests.bzl b/tests/shrubbery_toolchain/shrubbery_toolchain_tests.bzl index 7a6f639335..da79d2df7a 100644 --- a/tests/shrubbery_toolchain/shrubbery_toolchain_tests.bzl +++ b/tests/shrubbery_toolchain/shrubbery_toolchain_tests.bzl @@ -28,6 +28,7 @@ _tests = [] _COMPATIBLE_WITH = SUPPORTS_BZLMOD + NOT_WINDOWS _SHRUBBERY_SUFFIX = "crates/shrubbery/shrubbery" _PYTHON_EXE_ZIP_MAKER_SUFFIX = "tools/zipapp/exe_zip_maker_.py" +_PYTHON_ZIP_MAIN_MAKER_SUFFIX = "tools/zipapp/zip_main_maker_.py" _PYTHON_ZIPPER_SUFFIX = "tools/zipapp/zipper_.py" _NEED_SHRUBBERY_FLAG = str(Label("//python/private/toolchains:need_shrubbery")) @@ -69,8 +70,11 @@ def _test_py_zipapp_uses_toolchain_shrubbery(name): def _test_py_zipapp_uses_toolchain_shrubbery_impl(env, target): for mnemonic, subcommand in [ + # keep-sorted start ("PyZipAppCreateExecutableZip", "exe_zip_maker"), + ("PyZipAppCreateMainPy", "zip_main_maker"), ("PyZipAppCreateZip", "zipper"), + # keep-sorted end ]: action = env.expect.that_target(target).action_named(mnemonic) action.argv().contains_predicate( @@ -94,8 +98,11 @@ def _test_py_zipapp_flag_disabled_uses_python_fallbacks(name): def _test_py_zipapp_flag_disabled_uses_python_fallbacks_impl(env, target): for mnemonic, py_suffix in [ + # keep-sorted start ("PyZipAppCreateExecutableZip", _PYTHON_EXE_ZIP_MAKER_SUFFIX), + ("PyZipAppCreateMainPy", _PYTHON_ZIP_MAIN_MAKER_SUFFIX), ("PyZipAppCreateZip", _PYTHON_ZIPPER_SUFFIX), + # keep-sorted end ]: action = env.expect.that_target(target).action_named(mnemonic) action.argv().contains_predicate( diff --git a/tests/zip_main_maker/BUILD.bazel b/tests/zip_main_maker/BUILD.bazel new file mode 100644 index 0000000000..92f37c5e82 --- /dev/null +++ b/tests/zip_main_maker/BUILD.bazel @@ -0,0 +1,16 @@ +load("@rules_rust//rust:defs.bzl", "rust_test") + +package(default_visibility = ["//:__subpackages__"]) + +licenses(["notice"]) + +rust_test( + name = "zip_main_maker_test", + size = "small", + srcs = ["zip_main_maker_test.rs"], + edition = "2024", + deps = [ + "//crates/zip_main_maker:zip_main_maker_lib", + "@crates//:sha2", + ], +) diff --git a/tests/zip_main_maker/zip_main_maker_test.rs b/tests/zip_main_maker/zip_main_maker_test.rs new file mode 100644 index 0000000000..cb844371e4 --- /dev/null +++ b/tests/zip_main_maker/zip_main_maker_test.rs @@ -0,0 +1,495 @@ +use std::collections::HashMap; +use std::env; +use std::ffi::OsString; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use sha2::{Digest, Sha256}; +use zip_main_maker_lib::{ + Cli, Options, compute_inputs_hash, expand_template, generate_zip_main, parse_args, run_cli, +}; + +static TEST_COUNTER: AtomicUsize = AtomicUsize::new(0); + +fn finalize_hex(hasher: Sha256) -> String { + hasher + .finalize() + .iter() + .map(|b| format!("{b:02x}")) + .collect() +} + +fn unwrap_run_options(cli: Cli) -> Options { + match cli { + Cli::Run(opts) => opts, + Cli::Help => panic!("expected Cli::Run, got Cli::Help"), + } +} + +#[cfg(unix)] +fn create_symlink(target: &Path, link: &Path) { + std::os::unix::fs::symlink(target, link).expect("symlink should be creatable"); +} + +#[test] +fn test_creates_zip_main() { + let tmp = TempDir::new("creates_zip_main"); + let template_path = tmp.path().join("template.py"); + fs::write(&template_path, "hash=%APP_HASH%\nfoo=%FOO%\n").expect("template should be writable"); + + let output_path = tmp.path().join("output.py"); + + let file1_path = tmp.path().join("file1.txt"); + fs::write(&file1_path, b"content1").expect("file1 should be writable"); + + let file2_path = tmp.path().join("file2.txt"); + fs::write(&file2_path, b"content2").expect("file2 should be writable"); + + let manifest_path = tmp.path().join("manifest.txt"); + + #[cfg(unix)] + let symlink_path = { + let sp = tmp.path().join("symlink.txt"); + create_symlink(&file1_path, &sp); + sp + }; + + let manifest = [ + format!("rf-file|0|file1.txt|{}", file1_path.display()), + format!("rf-file|0|file2.txt|{}", file2_path.display()), + "rf-empty|empty_file.txt".to_string(), + ] + .join("\n"); + // Rebinding (rather than `Vec::push` under `#[cfg(unix)]`) keeps clippy's + // `useless_vec`/`unused_mut` lints quiet on Windows. + #[cfg(unix)] + let manifest = format!( + "{manifest}\nrf-symlink|1|symlink.txt|{}", + symlink_path.display() + ); + + fs::write(&manifest_path, format!("{manifest}\n")).expect("manifest should be writable"); + + let args: Vec = vec![ + "--template".into(), + template_path.as_os_str().to_owned(), + "--output".into(), + output_path.as_os_str().to_owned(), + "--substitution".into(), + "%FOO%=bar".into(), + "--hash_files_manifest".into(), + manifest_path.as_os_str().to_owned(), + ]; + + let options = unwrap_run_options(parse_args(&args).expect("args should parse")); + generate_zip_main(&options).expect("generate_zip_main should succeed"); + + // Entries sort lexicographically as: rf-empty, rf-file (file1), rf-file (file2), rf-symlink. + let mut hasher = Sha256::new(); + hasher.update(b"empty_file.txt"); + hasher.update(format!("0|file1.txt|{}", file1_path.display()).as_bytes()); + hasher.update(b"content1"); + hasher.update(format!("0|file2.txt|{}", file2_path.display()).as_bytes()); + hasher.update(b"content2"); + #[cfg(unix)] + { + hasher.update(format!("1|symlink.txt|{}", symlink_path.display()).as_bytes()); + hasher.update(file1_path.to_string_lossy().as_bytes()); + } + let expected_hash = finalize_hex(hasher); + + let content = fs::read_to_string(&output_path).expect("output should be readable"); + assert_eq!(content, format!("hash={expected_hash}\nfoo=bar\n")); +} + +#[test] +fn test_unsorted_manifest_determinism() { + let tmp = TempDir::new("unsorted_manifest"); + let f1 = tmp.path().join("a.txt"); + let f2 = tmp.path().join("b.txt"); + fs::write(&f1, b"alpha").expect("f1 should be writable"); + fs::write(&f2, b"beta").expect("f2 should be writable"); + + let l1 = format!("rf-file|0|a.txt|{}", f1.display()); + let l2 = format!("rf-file|0|b.txt|{}", f2.display()); + let l3 = "rf-empty|z_empty.txt".to_string(); + let l4 = "symlink|runfiles_link|/some/target".to_string(); + + let manifest_a = tmp.path().join("manifest_a.txt"); + let manifest_b = tmp.path().join("manifest_b.txt"); + + fs::write(&manifest_a, format!("{l4}\n{l2}\n{l3}\n{l1}\n")) + .expect("manifest_a should be writable"); + fs::write(&manifest_b, format!("{l1}\n{l2}\n{l3}\n{l4}\n")) + .expect("manifest_b should be writable"); + + let hash_a = compute_inputs_hash(&manifest_a).expect("manifest_a hash should succeed"); + let hash_b = compute_inputs_hash(&manifest_b).expect("manifest_b hash should succeed"); + assert_eq!(hash_a, hash_b); + assert_eq!(hash_a.len(), 64); +} + +#[test] +fn test_symlink_type_hashes_rest_only() { + let tmp = TempDir::new("symlink_type"); + // A `symlink||` entry does not open `` on disk; + // it only hashes `rest` (`|`). + let manifest = tmp.path().join("manifest.txt"); + fs::write( + &manifest, + "symlink|pkg/link.py|/nonexistent/target/on/disk.py\nrf-empty|pkg/__init__.py\n", + ) + .expect("manifest should be writable"); + + let actual_hash = compute_inputs_hash(&manifest).expect("manifest hash should succeed"); + + let mut hasher = Sha256::new(); + // "rf-empty|pkg/__init__.py" sorts before "symlink|pkg/link.py|/nonexistent/target/on/disk.py" + hasher.update(b"pkg/__init__.py"); + hasher.update(b"pkg/link.py|/nonexistent/target/on/disk.py"); + let expected_hash = finalize_hex(hasher); + + assert_eq!(actual_hash, expected_hash); +} + +#[test] +fn test_empty_manifest_and_empty_substitution_value() { + let tmp = TempDir::new("empty_manifest"); + let manifest = tmp.path().join("empty.manifest"); + fs::write(&manifest, "").expect("empty manifest should be writable"); + + let tpl = tmp.path().join("tpl.py"); + fs::write(&tpl, "OPT='%OPT%'\nHASH='%APP_HASH%'\n").expect("template should be writable"); + + let out = tmp.path().join("out.py"); + let code = run_cli([ + "--template", + tpl.to_str().expect("tpl path should be UTF-8"), + "--hash_files_manifest", + manifest.to_str().expect("manifest path should be UTF-8"), + "--substitution", + "%OPT%=", + "--output", + out.to_str().expect("out path should be UTF-8"), + ]); + assert_eq!(code, ExitCode::SUCCESS); + + let empty_sha256 = finalize_hex(Sha256::new()); + let rendered = fs::read_to_string(&out).expect("output should be readable"); + assert_eq!(rendered, format!("OPT=''\nHASH='{empty_sha256}'\n")); +} + +#[cfg(unix)] +#[test] +fn test_is_symlink_minus_one_valid_vs_broken() { + let tmp = TempDir::new("symlink_minus_one"); + let real_file = tmp.path().join("real.txt"); + fs::write(&real_file, b"real_file_bytes").expect("real_file should be writable"); + + // Symlink pointing to an existing file: `path.exists()` is true -> `is_symlink` is false -> + // hashes file content. + let valid_symlink = tmp.path().join("valid_sym.txt"); + create_symlink(&real_file, &valid_symlink); + + // Dangling symlink pointing to a non-existent file: `path.exists()` is false -> `is_symlink` + // is true -> hashes readlink target. + let missing_target = tmp.path().join("does_not_exist.txt"); + let broken_symlink = tmp.path().join("broken_sym.txt"); + create_symlink(&missing_target, &broken_symlink); + + let manifest_valid = tmp.path().join("manifest_valid.txt"); + let line_valid = format!("rf-file|-1|valid_sym.txt|{}", valid_symlink.display()); + fs::write(&manifest_valid, format!("{line_valid}\n")) + .expect("manifest_valid should be writable"); + + let hash_valid = compute_inputs_hash(&manifest_valid).expect("valid hash should succeed"); + let mut h_valid = Sha256::new(); + h_valid.update(line_valid.split_once('|').unwrap().1.as_bytes()); + h_valid.update(b"real_file_bytes"); + assert_eq!(hash_valid, finalize_hex(h_valid)); + + let manifest_broken = tmp.path().join("manifest_broken.txt"); + let line_broken = format!("rf-file|-1|broken_sym.txt|{}", broken_symlink.display()); + fs::write(&manifest_broken, format!("{line_broken}\n")) + .expect("manifest_broken should be writable"); + + let hash_broken = compute_inputs_hash(&manifest_broken).expect("broken hash should succeed"); + let mut h_broken = Sha256::new(); + h_broken.update(line_broken.split_once('|').unwrap().1.as_bytes()); + h_broken.update(missing_target.to_string_lossy().as_bytes()); + assert_eq!(hash_broken, finalize_hex(h_broken)); +} + +#[test] +fn test_flagfile_and_equals_flags_and_multiple_substitutions() { + let tmp = TempDir::new("flagfile_and_subst"); + let template_path = tmp.path().join("template.py"); + fs::write( + &template_path, + "hash=%APP_HASH%\ndir=%EXTRACT_DIR%\nbin=%python_binary%\nws=%workspace_name%\n", + ) + .expect("template should be writable"); + + let output_path = tmp.path().join("nested").join("output.py"); + let manifest_path = tmp.path().join("manifest.txt"); + fs::write(&manifest_path, "rf-empty|__init__.py\n").expect("manifest should be writable"); + + let flagfile_path = tmp.path().join("args.params"); + fs::write( + &flagfile_path, + format!( + "--template={}\n--output\n{}\n--substitution=%EXTRACT_DIR%=_main/pkg/app\n--substitution=%python_binary%=old_val\n", + template_path.display(), + output_path.display(), + ), + ) + .expect("flagfile should be writable"); + + let args = vec![ + format!("@{}", flagfile_path.display()), + "--substitution".to_string(), + "%python_binary%=venv/bin/python3".to_string(), + "--substitution=%workspace_name%=my_ws=with_eq".to_string(), + format!("--hash_files_manifest={}", manifest_path.display()), + ]; + + let options = unwrap_run_options(parse_args(&args).expect("flagfile args should parse")); + assert_eq!(options.template, template_path); + assert_eq!(options.output, output_path); + assert_eq!(options.hash_files_manifest, manifest_path); + assert_eq!(run_cli(&args), ExitCode::SUCCESS); + + let expected_hash = compute_inputs_hash(&manifest_path).expect("manifest hash should succeed"); + let content = fs::read_to_string(&output_path).expect("output should be readable"); + assert_eq!( + content, + format!( + "hash={expected_hash}\ndir=_main/pkg/app\nbin=venv/bin/python3\nws=my_ws=with_eq\n" + ) + ); +} + +#[test] +fn test_substitution_overwrite_and_app_hash_override() { + let options = unwrap_run_options( + parse_args([ + "--template=tpl.txt", + "--output=out.txt", + "--hash_files_manifest=manifest.txt", + "--substitution=%FOO%=first", + "--substitution=%BAR%=bar_val", + "--substitution=%FOO%=second", + ]) + .expect("args should parse"), + ); + assert_eq!( + options.substitutions, + HashMap::from([ + ("%BAR%".to_string(), "bar_val".to_string()), + ("%FOO%".to_string(), "second".to_string()), + ]) + ); + + let tmp = TempDir::new("subst_overwrite"); + let tpl = tmp.path().join("tpl.txt"); + let out = tmp.path().join("out.txt"); + fs::write(&tpl, "%APP_HASH% %FOO% %BAR%").expect("tpl should be writable"); + expand_template( + &tpl, + &out, + &HashMap::from([ + ("%APP_HASH%", "deadbeef"), + ("%FOO%", "second"), + ("%BAR%", "bar_val"), + ]), + ) + .expect("expand_template should succeed"); + assert_eq!( + fs::read_to_string(&out).expect("out should be readable"), + "deadbeef second bar_val" + ); +} + +#[cfg(unix)] +#[test] +fn test_is_symlink_zero_vs_one_on_symlink() { + let tmp = TempDir::new("symlink_zero_vs_one"); + let target_file = tmp.path().join("target.txt"); + fs::write(&target_file, b"target_file_payload").expect("target_file should be writable"); + + let sym = tmp.path().join("sym.txt"); + create_symlink(&target_file, &sym); + + // When is_symlink == "0", it opens `sym` (following the symlink) and hashes the target file's + // bytes. + let manifest_0 = tmp.path().join("manifest_0.txt"); + let line_0 = format!("rf-file|0|sym.txt|{}", sym.display()); + fs::write(&manifest_0, format!("{line_0}\n")).expect("manifest_0 should be writable"); + + let hash_0 = compute_inputs_hash(&manifest_0).expect("manifest_0 hash should succeed"); + let mut h0 = Sha256::new(); + h0.update(line_0.split_once('|').unwrap().1.as_bytes()); + h0.update(b"target_file_payload"); + assert_eq!(hash_0, finalize_hex(h0)); + + // When is_symlink == "1", it reads the symlink target path via readlink and hashes the path + // bytes. + let manifest_1 = tmp.path().join("manifest_1.txt"); + let line_1 = format!("rf-symlink|1|sym.txt|{}", sym.display()); + fs::write(&manifest_1, format!("{line_1}\n")).expect("manifest_1 should be writable"); + + let hash_1 = compute_inputs_hash(&manifest_1).expect("manifest_1 hash should succeed"); + let mut h1 = Sha256::new(); + h1.update(line_1.split_once('|').unwrap().1.as_bytes()); + h1.update(target_file.to_string_lossy().as_bytes()); + assert_eq!(hash_1, finalize_hex(h1)); + assert_ne!(hash_0, hash_1); +} + +#[test] +fn test_large_file_streaming_and_crlf_manifest() { + let tmp = TempDir::new("large_file_crlf"); + let large_file = tmp.path().join("large.bin"); + let payload: Vec = (0..(600 * 1024)).map(|i| (i % 251) as u8).collect(); + fs::write(&large_file, &payload).expect("large_file should be writable"); + + let line1 = format!("rf-file|0|large.bin|{}", large_file.display()); + let line2 = "rf-empty|empty.py"; + + let manifest_crlf = tmp.path().join("manifest_crlf.txt"); + fs::write(&manifest_crlf, format!("{line1}\r\n{line2}\r\n")) + .expect("manifest_crlf should be writable"); + + let manifest_lf = tmp.path().join("manifest_lf.txt"); + fs::write(&manifest_lf, format!("{line1}\n{line2}\n")).expect("manifest_lf should be writable"); + + let hash_crlf = compute_inputs_hash(&manifest_crlf).expect("CRLF manifest should hash"); + let hash_lf = compute_inputs_hash(&manifest_lf).expect("LF manifest should hash"); + assert_eq!(hash_crlf, hash_lf); + + let mut expected = Sha256::new(); + // "rf-empty|empty.py" sorts before "rf-file|0|large.bin|..." + expected.update(b"empty.py"); + expected.update(line1.split_once('|').unwrap().1.as_bytes()); + expected.update(&payload); + assert_eq!(hash_lf, finalize_hex(expected)); +} + +#[test] +fn test_help_and_errors() { + assert_eq!( + parse_args(["--help"]).expect("--help should parse"), + Cli::Help + ); + assert_eq!(parse_args(["-h"]).expect("-h should parse"), Cli::Help); + assert_eq!(run_cli(["--help"]), ExitCode::SUCCESS); + + // Missing required flags. + assert!(parse_args::<_, &str>([]).is_err()); + assert!(parse_args(["--template=a", "--output=b"]).is_err()); + + // Bare "@" flagfile fails cleanly via argfile. + assert!(parse_args(["@"]).is_err()); + + // Invalid substitution without '='. + assert!( + parse_args([ + "--template=a", + "--output=b", + "--hash_files_manifest=c", + "--substitution=NO_EQUALS", + ]) + .is_err() + ); + + // Unknown flag. + assert!( + parse_args([ + "--template=a", + "--output=b", + "--hash_files_manifest=c", + "--unknown=1", + ]) + .is_err() + ); + + // Unexpected positional arg. + assert!( + parse_args([ + "--template=a", + "--output=b", + "--hash_files_manifest=c", + "extra", + ]) + .is_err() + ); + + // Malformed manifest lines and missing input files name the offending path/line. + let tmp = TempDir::new("errors"); + let bad_manifest = tmp.path().join("bad_manifest.txt"); + fs::write(&bad_manifest, "no_pipe_delimiter\n").expect("bad_manifest should be writable"); + let err = format!("{:#}", compute_inputs_hash(&bad_manifest).unwrap_err()); + assert!(err.contains("no_pipe_delimiter"), "unexpected err: {err}"); + + let bad_mode = tmp.path().join("bad_mode.txt"); + fs::write(&bad_mode, "rf-file|9|a.py|/nonexistent/a.py\n") + .expect("bad_mode should be writable"); + let err = format!("{:#}", compute_inputs_hash(&bad_mode).unwrap_err()); + assert!( + err.contains("invalid is_symlink value"), + "unexpected err: {err}" + ); + + let unknown_type = tmp.path().join("unknown_type.txt"); + fs::write(&unknown_type, "unknown|0|a.py|/nonexistent/a.py\n") + .expect("unknown_type should be writable"); + let err = format!("{:#}", compute_inputs_hash(&unknown_type).unwrap_err()); + assert!( + err.contains("unknown entry type or invalid format"), + "unexpected err: {err}" + ); + + let truncated_line = tmp.path().join("truncated_line.txt"); + fs::write(&truncated_line, "rf-file|0|a.py\n").expect("truncated_line should be writable"); + let err = format!("{:#}", compute_inputs_hash(&truncated_line).unwrap_err()); + assert!( + err.contains("unknown entry type or invalid format"), + "unexpected err: {err}" + ); + + assert_eq!(run_cli::<_, &str>([]), ExitCode::FAILURE); +} + +struct TempDir { + path: PathBuf, +} + +impl TempDir { + fn new(prefix: &str) -> Self { + let id = TEST_COUNTER.fetch_add(1, Ordering::SeqCst); + let base = env::var_os("TEST_TMPDIR") + .map(PathBuf::from) + .unwrap_or_else(env::temp_dir); + let path = base.join(format!( + "zip_main_maker_test_{}_{}_{}", + prefix, + std::process::id(), + id + )); + let _ = fs::remove_dir_all(&path); + fs::create_dir_all(&path).expect("temp dir should be creatable"); + Self { path } + } + + fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } +} diff --git a/tests/zip_manifest/BUILD.bazel b/tests/zip_manifest/BUILD.bazel new file mode 100644 index 0000000000..42a7e144a5 --- /dev/null +++ b/tests/zip_manifest/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_rust//rust:defs.bzl", "rust_test") + +package(default_visibility = ["//:__subpackages__"]) + +licenses(["notice"]) + +rust_test( + name = "zip_manifest_test", + size = "small", + srcs = ["zip_manifest_test.rs"], + edition = "2024", + deps = [ + "//crates/zip_manifest", + ], +) diff --git a/tests/zip_manifest/zip_manifest_test.rs b/tests/zip_manifest/zip_manifest_test.rs new file mode 100644 index 0000000000..885b8ee335 --- /dev/null +++ b/tests/zip_manifest/zip_manifest_test.rs @@ -0,0 +1,131 @@ +use std::path::Path; + +use zip_manifest::{ + ManifestLine, ManifestParseError, RawManifestEntry, SymlinkMode, parse_manifest_line, + parse_symlink_mode, +}; + +#[test] +fn test_parse_all_valid_entry_kinds() { + let regular = parse_manifest_line("regular|0|__main__.py|bazel-out/main.py") + .expect("regular entry should parse"); + assert_eq!( + regular, + ManifestLine { + rest: "0|__main__.py|bazel-out/main.py", + entry: RawManifestEntry::Regular { + symlink_mode: SymlinkMode::Regular, + zip_path: "__main__.py", + content_path: Path::new("bazel-out/main.py"), + }, + } + ); + assert_eq!( + regular.entry.content_source(), + Some((SymlinkMode::Regular, Path::new("bazel-out/main.py"))) + ); + + let rf_empty = + parse_manifest_line("rf-empty|pkg/__init__.py").expect("rf-empty entry should parse"); + assert_eq!( + rf_empty, + ManifestLine { + rest: "pkg/__init__.py", + entry: RawManifestEntry::RfEmpty { + runfile_path: "pkg/__init__.py", + }, + } + ); + assert_eq!(rf_empty.entry.content_source(), None); + + let rf_file = parse_manifest_line("rf-file|-1|pkg/app.py|src/pkg/app.py") + .expect("rf-file entry should parse"); + assert_eq!( + rf_file.entry, + RawManifestEntry::RfFile { + symlink_mode: SymlinkMode::Infer, + runfile_path: "pkg/app.py", + content_path: Path::new("src/pkg/app.py"), + } + ); + assert_eq!( + rf_file.entry.content_source(), + Some((SymlinkMode::Infer, Path::new("src/pkg/app.py"))) + ); + + let rf_symlink = parse_manifest_line("rf-symlink|1|pkg/link.py|src/pkg/link.py") + .expect("rf-symlink entry should parse"); + assert_eq!( + rf_symlink.entry, + RawManifestEntry::RfSymlink { + symlink_mode: SymlinkMode::Symlink, + runfile_symlink_path: "pkg/link.py", + content_path: Path::new("src/pkg/link.py"), + } + ); + + let rf_root = parse_manifest_line("rf-root-symlink|0|_repo_mapping|bazel-out/_repo_mapping") + .expect("rf-root-symlink entry should parse"); + assert_eq!( + rf_root.entry, + RawManifestEntry::RfRootSymlink { + symlink_mode: SymlinkMode::Regular, + runfile_root_path: "_repo_mapping", + content_path: Path::new("bazel-out/_repo_mapping"), + } + ); + + let symlink = parse_manifest_line("symlink|_main/venv/bin/python|_main/py/bin/python3") + .expect("symlink entry should parse"); + assert_eq!( + symlink, + ManifestLine { + rest: "_main/venv/bin/python|_main/py/bin/python3", + entry: RawManifestEntry::Symlink { + runfile_root_path: "_main/venv/bin/python", + link_to_runfile_root_path: "_main/py/bin/python3", + }, + } + ); + assert_eq!(symlink.entry.content_source(), None); +} + +#[test] +fn test_parse_errors() { + assert_eq!( + parse_symlink_mode("2"), + Err(ManifestParseError::InvalidSymlinkMode { + value: "2".to_string() + }) + ); + assert_eq!( + parse_manifest_line("missing_delimiter"), + Err(ManifestParseError::InvalidFormat { + line: "missing_delimiter".to_string() + }) + ); + assert_eq!( + parse_manifest_line("unknown|0|a|b"), + Err(ManifestParseError::InvalidFormat { + line: "unknown|0|a|b".to_string() + }) + ); + assert_eq!( + parse_manifest_line("rf-empty|too|many"), + Err(ManifestParseError::InvalidFormat { + line: "rf-empty|too|many".to_string() + }) + ); + assert_eq!( + parse_manifest_line("rf-file|0|missing_content_path"), + Err(ManifestParseError::InvalidFormat { + line: "rf-file|0|missing_content_path".to_string() + }) + ); + assert_eq!( + parse_manifest_line("rf-file|9|a|b"), + Err(ManifestParseError::InvalidSymlinkMode { + value: "9".to_string() + }) + ); +}