From aff81dd502cce7a29d78ba323b1d00a4db9fe005 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 15 Nov 2026 20:23:14 -0800 Subject: [PATCH] refactor: convert build_data_writer build tool to Rust Writing target `build_data.txt` stamps uses separate shell and PowerShell scripts across Unix and Windows hosts. Add `//crates/build_data_writer:build_data_writer_lib`, register `build_data_writer` on `shrubbery`, and route `PyWriteBuildData` through `get_shrubbery_tool` with the platform scripts as fallback. --- crates/build_data_writer/BUILD.bazel | 20 ++ crates/build_data_writer/src/lib.rs | 220 +++++++++++++ crates/shrubbery/BUILD.bazel | 1 + crates/shrubbery/src/lib.rs | 3 + python/private/py_executable.bzl | 43 ++- tests/build_data_writer/BUILD.bazel | 15 + .../build_data_writer_test.rs | 294 ++++++++++++++++++ tests/shrubbery/shrubbery_test.rs | 2 +- 8 files changed, 581 insertions(+), 17 deletions(-) create mode 100644 crates/build_data_writer/BUILD.bazel create mode 100644 crates/build_data_writer/src/lib.rs create mode 100644 tests/build_data_writer/BUILD.bazel create mode 100644 tests/build_data_writer/build_data_writer_test.rs diff --git a/crates/build_data_writer/BUILD.bazel b/crates/build_data_writer/BUILD.bazel new file mode 100644 index 0000000000..5eb7f16192 --- /dev/null +++ b/crates/build_data_writer/BUILD.bazel @@ -0,0 +1,20 @@ +load("@rules_rust//rust:defs.bzl", "rust_library") +load("//python/private:distribution_filegroup.bzl", "distribution_filegroup") # buildifier: disable=bzl-visibility + +package(default_visibility = ["//:__subpackages__"]) + +licenses(["notice"]) + +rust_library( + name = "build_data_writer_lib", + srcs = ["src/lib.rs"], + edition = "2024", + deps = [ + "@crates//:anyhow", + "@crates//:lexopt", + ], +) + +distribution_filegroup( + name = "distribution", +) diff --git a/crates/build_data_writer/src/lib.rs b/crates/build_data_writer/src/lib.rs new file mode 100644 index 0000000000..f02da15357 --- /dev/null +++ b/crates/build_data_writer/src/lib.rs @@ -0,0 +1,220 @@ +//! Library for writing Bazel target build data and workspace status stamping info. +//! +//! Configuration is read from the environment variables `OUTPUT`, `TARGET`, `STAMPED`, +//! `VERSION_FILE`, and `INFO_FILE`, matching the `build_data_writer.sh` script it replaces. +//! The command line accepts no arguments other than `-h`/`--help`. + +// Errors must be returned, not panicked; see the `rust-lint` config in .bazelrc. +#![deny(clippy::expect_used, clippy::unwrap_used)] + +use std::env; +use std::ffi::OsString; +use std::fs::{self, File}; +use std::io::{BufWriter, Write}; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; + +use anyhow::{Context, Result, bail}; +use lexopt::prelude::*; + +mod flags { + // keep-sorted start + pub const HELP: &str = "help"; + pub const HELP_SHORT: char = 'h'; + // keep-sorted end +} + +/// Environment variables consumed by the tool. +mod env_vars { + // keep-sorted start + pub const INFO_FILE: &str = "INFO_FILE"; + pub const OUTPUT: &str = "OUTPUT"; + pub const STAMPED: &str = "STAMPED"; + pub const TARGET: &str = "TARGET"; + pub const VERSION_FILE: &str = "VERSION_FILE"; + // keep-sorted end +} + +/// Usage text for `--help`. `lexopt` does not generate help, so this is maintained by hand. +const USAGE: &str = "\ +Usage: build_data_writer + +Write a build_data.txt file describing a target and its workspace status. +All configuration comes from environment variables; no arguments are accepted. + +Environment variables: + OUTPUT Path of the build data file to write (required) + TARGET Label of the target being built (required) + STAMPED TRUE or FALSE; whether workspace status stamping is enabled (required) + VERSION_FILE Path to volatile-status.txt to append; ignored when empty + INFO_FILE Path to stable-status.txt to append; ignored when empty + +Options: + -h, --help Print this help"; + +/// Configuration options for writing a `build_data.txt` file. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Options { + /// Path to the output `build_data.txt` file. + pub output: PathBuf, + /// Target label string (e.g. `//pkg:target`). + pub target: String, + /// Whether stamping is enabled. + pub stamped: bool, + /// Optional path to Bazel's `volatile-status.txt` (`ctx.version_file`). + pub version_file: Option, + /// Optional path to Bazel's `stable-status.txt` (`ctx.info_file`). + pub info_file: Option, +} + +/// Result of parsing the command line and environment. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Cli { + /// `--help` was requested. + Help, + /// Write the build data file with the given options. + Run(Options), +} + +fn parse_stamped(val: &str) -> Result { + match val { + "TRUE" => Ok(true), + "FALSE" => Ok(false), + other => bail!("{} must be TRUE or FALSE, got '{other}'", env_vars::STAMPED), + } +} + +fn required_env(get_env: &F, key: &str) -> Result +where + F: Fn(&str) -> Option, +{ + get_env(key) + .with_context(|| format!("missing required environment variable: {key}"))? + .into_string() + .map_err(|raw| anyhow::anyhow!("{key} environment variable is not valid UTF-8: {raw:?}")) +} + +/// Parses command-line arguments (excluding `argv[0]` and subcommand name) and the environment +/// provided by `get_env` into [`Cli`]. +/// +/// # Errors +/// +/// Returns an error when an argument other than `-h`/`--help` is given, when `OUTPUT`, `TARGET`, +/// or `STAMPED` is unset, or when `STAMPED` is neither `TRUE` nor `FALSE`. +pub fn parse_env_and_args(args: I, get_env: F) -> Result +where + I: IntoIterator, + T: Into, + F: Fn(&str) -> Option, +{ + let mut parser = lexopt::Parser::from_args(args.into_iter().map(Into::into)); + if let Some(arg) = parser.next()? { + match arg { + Short(flags::HELP_SHORT) | Long(flags::HELP) => return Ok(Cli::Help), + _ => bail!("{}", arg.unexpected()), + } + } + + let output = get_env(env_vars::OUTPUT) + .map(PathBuf::from) + .with_context(|| { + format!( + "missing required environment variable: {}", + env_vars::OUTPUT + ) + })?; + let target = required_env(&get_env, env_vars::TARGET)?; + let stamped = parse_stamped(&required_env(&get_env, env_vars::STAMPED)?)?; + let version_file = get_env(env_vars::VERSION_FILE) + .filter(|s| !s.is_empty()) + .map(PathBuf::from); + let info_file = get_env(env_vars::INFO_FILE) + .filter(|s| !s.is_empty()) + .map(PathBuf::from); + + Ok(Cli::Run(Options { + output, + target, + stamped, + version_file, + info_file, + })) +} + +/// Parses command-line arguments (excluding `argv[0]` and subcommand name) and the process +/// environment into [`Cli`]. +/// +/// # Errors +/// +/// See [`parse_env_and_args`]. +pub fn parse_args(args: I) -> Result +where + I: IntoIterator, + T: Into, +{ + parse_env_and_args(args, |key| env::var_os(key)) +} + +/// Parses CLI arguments (excluding `argv[0]` and subcommand name) and the process environment, +/// writes the build data file, and returns the process exit code. +pub fn run_cli(args: I) -> ExitCode +where + I: IntoIterator, + T: Into, +{ + let options = match parse_args(args) { + Ok(Cli::Help) => { + println!("{USAGE}"); + return ExitCode::SUCCESS; + } + Ok(Cli::Run(opts)) => opts, + Err(err) => { + eprintln!("build_data_writer: error: {err:#}"); + return ExitCode::FAILURE; + } + }; + if let Err(err) = write_build_data(&options) { + eprintln!("build_data_writer: error: {err:#}"); + return ExitCode::FAILURE; + } + ExitCode::SUCCESS +} + +/// Writes the build data file according to `options`. +/// +/// # Errors +/// +/// Returns an error when the output file cannot be created or written, or when a configured +/// status file cannot be read. +pub fn write_build_data(options: &Options) -> Result<()> { + let file = File::create(&options.output) + .with_context(|| format!("creating {}", options.output.display()))?; + let mut writer = BufWriter::new(file); + + writeln!(writer, "TARGET {}", options.target)?; + writeln!( + writer, + "STAMPED {}", + if options.stamped { "TRUE" } else { "FALSE" } + )?; + + if let Some(version_file) = &options.version_file { + append_status_file(&mut writer, version_file)?; + } + if let Some(info_file) = &options.info_file { + append_status_file(&mut writer, info_file)?; + } + + writer + .flush() + .with_context(|| format!("writing {}", options.output.display())) +} + +fn append_status_file(writer: &mut W, path: &Path) -> Result<()> { + let contents = fs::read(path).with_context(|| format!("reading {}", path.display()))?; + writer.write_all(&contents)?; + if !contents.is_empty() && !contents.ends_with(b"\n") { + writer.write_all(b"\n")?; + } + Ok(()) +} diff --git a/crates/shrubbery/BUILD.bazel b/crates/shrubbery/BUILD.bazel index e7c8eff447..8123add912 100644 --- a/crates/shrubbery/BUILD.bazel +++ b/crates/shrubbery/BUILD.bazel @@ -10,6 +10,7 @@ rust_library( srcs = ["src/lib.rs"], edition = "2024", deps = [ + "//crates/build_data_writer:build_data_writer_lib", "//crates/exe_zip_maker:exe_zip_maker_lib", "//crates/extract_wheel_files:extract_wheel_files_lib", "//crates/py_console_script_gen:py_console_script_gen_lib", diff --git a/crates/shrubbery/src/lib.rs b/crates/shrubbery/src/lib.rs index feb775a6a8..c598cd98e5 100644 --- a/crates/shrubbery/src/lib.rs +++ b/crates/shrubbery/src/lib.rs @@ -10,6 +10,7 @@ use std::process::ExitCode; /// Subcommand name constants supported by `shrubbery`. pub mod subcommands { // keep-sorted start case=no + pub const BUILD_DATA_WRITER: &str = "build_data_writer"; pub const EXE_ZIP_MAKER: &str = "exe_zip_maker"; pub const EXTRACT_WHEEL_FILES: &str = "extract_wheel_files"; pub const PY_CONSOLE_SCRIPT_GEN: &str = "py_console_script_gen"; @@ -25,6 +26,7 @@ pub mod subcommands { /// Canonical subcommand names supported by `shrubbery`. pub const SUBCOMMANDS: &[&str] = &[ // keep-sorted start case=no + subcommands::BUILD_DATA_WRITER, subcommands::EXE_ZIP_MAKER, subcommands::EXTRACT_WHEEL_FILES, subcommands::PY_CONSOLE_SCRIPT_GEN, @@ -75,6 +77,7 @@ where match sub_raw.as_ref() { // keep-sorted start case=no + subcommands::BUILD_DATA_WRITER => build_data_writer_lib::run_cli(iter), subcommands::EXE_ZIP_MAKER => exe_zip_maker_lib::run_cli(iter), subcommands::EXTRACT_WHEEL_FILES => extract_wheel_files_lib::run_cli(iter), subcommands::PY_CONSOLE_SCRIPT_GEN => py_console_script_gen_lib::run_cli(iter), diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index c1fb7588f8..3f47d8e45e 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -1723,22 +1723,33 @@ def _write_build_data(ctx): ) action_args = ctx.actions.args() - writer_file = ctx.files._build_data_writer[0] - if writer_file.path.endswith(".ps1"): - # powershell.exe is used for broader compatibility - # It is installed by default on most Windows versions - action_exe = "powershell.exe" - action_args.add_all([ - # Bypass execution policy is needed because, - # by default, Windows blocks ps1 scripts. - "-ExecutionPolicy", - "Bypass", - "-File", - writer_file, - ]) - inputs.add(writer_file) + writer = get_shrubbery_tool( + ctx, + "build_data_writer", + ctx.attr._build_data_writer, + ) + if writer.is_rust: + action_exe = writer.files_to_run + action_toolchain = writer.toolchain + action_args.add_all(writer.prefix_args) else: - action_exe = ctx.attr._build_data_writer[DefaultInfo].files_to_run + action_toolchain = None + writer_file = ctx.files._build_data_writer[0] + if writer_file.path.endswith(".ps1"): + # powershell.exe is used for broader compatibility + # It is installed by default on most Windows versions + action_exe = "powershell.exe" + action_args.add_all([ + # Bypass execution policy is needed because, + # by default, Windows blocks ps1 scripts. + "-ExecutionPolicy", + "Bypass", + "-File", + writer_file, + ]) + inputs.add(writer_file) + else: + action_exe = writer.files_to_run ctx.actions.run( executable = action_exe, @@ -1756,7 +1767,7 @@ def _write_build_data(ctx): outputs = [build_data], mnemonic = "PyWriteBuildData", progress_message = "Reticulating %{label} build data", - toolchain = None, + toolchain = action_toolchain, ) return build_data diff --git a/tests/build_data_writer/BUILD.bazel b/tests/build_data_writer/BUILD.bazel new file mode 100644 index 0000000000..62eee0b246 --- /dev/null +++ b/tests/build_data_writer/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_rust//rust:defs.bzl", "rust_test") + +package(default_visibility = ["//:__subpackages__"]) + +licenses(["notice"]) + +rust_test( + name = "build_data_writer_test", + size = "small", + srcs = ["build_data_writer_test.rs"], + edition = "2024", + deps = [ + "//crates/build_data_writer:build_data_writer_lib", + ], +) diff --git a/tests/build_data_writer/build_data_writer_test.rs b/tests/build_data_writer/build_data_writer_test.rs new file mode 100644 index 0000000000..094e13964b --- /dev/null +++ b/tests/build_data_writer/build_data_writer_test.rs @@ -0,0 +1,294 @@ +use std::collections::HashMap; +use std::env; +use std::ffi::OsString; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use build_data_writer_lib::{Cli, Options, parse_env_and_args, run_cli, write_build_data}; + +static TEST_COUNTER: AtomicUsize = AtomicUsize::new(0); + +const NO_ARGS: [&str; 0] = []; + +fn create_temp_dir(prefix: &str) -> PathBuf { + let id = TEST_COUNTER.fetch_add(1, Ordering::SeqCst); + let dir = env::temp_dir().join(format!( + "build_data_writer_test_{}_{}_{}", + prefix, + std::process::id(), + id + )); + fs::create_dir_all(&dir).unwrap(); + dir +} + +fn parse_run(env_map: &HashMap<&str, OsString>) -> Options { + match parse_env_and_args(NO_ARGS, |k| env_map.get(k).cloned()) + .expect("environment should parse") + { + Cli::Run(opts) => opts, + Cli::Help => panic!("expected Cli::Run, got Cli::Help"), + } +} + +fn base_env(output: &Path) -> HashMap<&'static str, OsString> { + HashMap::from([ + ("OUTPUT", output.as_os_str().to_owned()), + ("TARGET", OsString::from("//pkg:bin")), + ("STAMPED", OsString::from("TRUE")), + ]) +} + +#[test] +fn test_env_configuration_both_status_files() { + let temp_dir = create_temp_dir("env_both"); + let output_path = temp_dir.join("build_data.txt"); + let version_file = temp_dir.join("volatile-status.txt"); + let info_file = temp_dir.join("stable-status.txt"); + + fs::write( + &version_file, + "BUILD_TIMESTAMP 1700000000\nFORMATTED_DATE 2023 Nov 14 22 13 20 Tue\n", + ) + .unwrap(); + fs::write(&info_file, "BUILD_HOST build-box\nBUILD_USER builder\n").unwrap(); + + let env_map: HashMap<&str, OsString> = HashMap::from([ + ("OUTPUT", output_path.as_os_str().to_owned()), + ( + "TARGET", + OsString::from("//tests/build_data:build_data_test"), + ), + ("STAMPED", OsString::from("TRUE")), + ("VERSION_FILE", version_file.as_os_str().to_owned()), + ("INFO_FILE", info_file.as_os_str().to_owned()), + ]); + + let opts = parse_run(&env_map); + assert!(opts.stamped); + write_build_data(&opts).expect("build data should be writable"); + + let result = fs::read_to_string(&output_path).unwrap(); + let expected = concat!( + "TARGET //tests/build_data:build_data_test\n", + "STAMPED TRUE\n", + "BUILD_TIMESTAMP 1700000000\n", + "FORMATTED_DATE 2023 Nov 14 22 13 20 Tue\n", + "BUILD_HOST build-box\n", + "BUILD_USER builder\n", + ); + assert_eq!(result, expected); + + let _ = fs::remove_dir_all(&temp_dir); +} + +#[test] +fn test_help_flag() { + for help in ["--help", "-h"] { + let cli = parse_env_and_args([help], |_| None).expect("help flag should parse"); + assert_eq!(cli, Cli::Help, "unexpected result for {help}"); + } + assert_eq!(run_cli(["--help"]), ExitCode::SUCCESS); +} + +#[test] +fn test_run_cli_rejects_arguments() { + assert_eq!(run_cli(["--unknown-flag"]), ExitCode::FAILURE); + assert_eq!(run_cli(["--output=/tmp/out.txt"]), ExitCode::FAILURE); + assert_eq!(run_cli(["positional"]), ExitCode::FAILURE); +} + +#[test] +fn test_unexpected_arguments_error() { + let env_map = base_env(&PathBuf::from("/tmp/out.txt")); + + for (args, token) in [ + (["--unknown-flag"], "--unknown-flag"), + (["--output=/tmp/out.txt"], "--output"), + (["positional"], "positional"), + ] { + let err = parse_env_and_args(args, |k| env_map.get(k).cloned()).unwrap_err(); + assert!( + err.to_string().contains(token), + "unexpected error for {args:?}: {err}" + ); + } +} + +#[test] +fn test_stamped_parsing() { + let output = PathBuf::from("/tmp/out.txt"); + + let mut env_map = base_env(&output); + env_map.insert("STAMPED", OsString::from("TRUE")); + assert!(parse_run(&env_map).stamped); + + env_map.insert("STAMPED", OsString::from("FALSE")); + assert!(!parse_run(&env_map).stamped); + + for invalid in ["true", "false", "1", "0", "", "maybe"] { + env_map.insert("STAMPED", OsString::from(invalid)); + let err = parse_env_and_args(NO_ARGS, |k| env_map.get(k).cloned()).unwrap_err(); + assert!( + err.to_string().contains("STAMPED"), + "unexpected error for {invalid:?}: {err}" + ); + } + + env_map.remove("STAMPED"); + let err = parse_env_and_args(NO_ARGS, |k| env_map.get(k).cloned()).unwrap_err(); + assert!( + err.to_string().contains("STAMPED"), + "unexpected error: {err}" + ); +} + +#[test] +fn test_missing_required_env_vars() { + let err = parse_env_and_args(NO_ARGS, |_| None).unwrap_err(); + assert!( + err.to_string().contains("OUTPUT"), + "unexpected error: {err}" + ); + + let mut env_map = base_env(&PathBuf::from("/tmp/out.txt")); + env_map.remove("TARGET"); + let err = parse_env_and_args(NO_ARGS, |k| env_map.get(k).cloned()).unwrap_err(); + assert!( + err.to_string().contains("TARGET"), + "unexpected error: {err}" + ); +} + +#[test] +fn test_empty_or_unset_version_and_info_files() { + let temp_dir = create_temp_dir("empty_unset"); + let output_path = temp_dir.join("unstamped.txt"); + let empty_file = temp_dir.join("empty.txt"); + fs::write(&empty_file, "").unwrap(); + + // Case 1: Both VERSION_FILE and INFO_FILE unset + let opts = Options { + output: output_path.clone(), + target: "//foo:bar".to_string(), + stamped: false, + version_file: None, + info_file: None, + }; + write_build_data(&opts).expect("build data should be writable"); + assert_eq!( + fs::read_to_string(&output_path).unwrap(), + "TARGET //foo:bar\nSTAMPED FALSE\n" + ); + + // Case 2: VERSION_FILE and INFO_FILE set to empty strings in env + let mut env_map = base_env(&output_path); + env_map.insert("TARGET", OsString::from("//foo:baz")); + env_map.insert("VERSION_FILE", OsString::new()); + env_map.insert("INFO_FILE", OsString::new()); + + let parsed = parse_run(&env_map); + assert_eq!(parsed.version_file, None); + assert_eq!(parsed.info_file, None); + write_build_data(&parsed).expect("build data should be writable"); + assert_eq!( + fs::read_to_string(&output_path).unwrap(), + "TARGET //foo:baz\nSTAMPED TRUE\n" + ); + + // Case 3: VERSION_FILE points to a 0-byte file + let opts_empty_file = Options { + output: output_path.clone(), + target: "//foo:qux".to_string(), + stamped: true, + version_file: Some(empty_file), + info_file: None, + }; + write_build_data(&opts_empty_file).expect("build data should be writable"); + assert_eq!( + fs::read_to_string(&output_path).unwrap(), + "TARGET //foo:qux\nSTAMPED TRUE\n" + ); + + let _ = fs::remove_dir_all(&temp_dir); +} + +#[test] +fn test_status_files_without_trailing_newline() { + let temp_dir = create_temp_dir("no_newline"); + let output_path = temp_dir.join("out.txt"); + let version_file = temp_dir.join("volatile.txt"); + let info_file = temp_dir.join("stable.txt"); + + // Neither file ends with '\n' + fs::write(&version_file, "BUILD_TIMESTAMP 123").unwrap(); + fs::write(&info_file, "BUILD_USER bob").unwrap(); + + let opts = Options { + output: output_path.clone(), + target: "//a:b".to_string(), + stamped: true, + version_file: Some(version_file), + info_file: Some(info_file), + }; + write_build_data(&opts).expect("build data should be writable"); + + let content = fs::read_to_string(&output_path).unwrap(); + assert_eq!( + content, + "TARGET //a:b\nSTAMPED TRUE\nBUILD_TIMESTAMP 123\nBUILD_USER bob\n" + ); + + let _ = fs::remove_dir_all(&temp_dir); +} + +#[test] +fn test_truncates_existing_output() { + let temp_dir = create_temp_dir("truncate"); + let output_path = temp_dir.join("build_data.txt"); + + let opts1 = Options { + output: output_path.clone(), + target: "//first:target_with_long_previous_content".to_string(), + stamped: true, + version_file: None, + info_file: None, + }; + write_build_data(&opts1).expect("build data should be writable"); + + let opts2 = Options { + output: output_path.clone(), + target: "//second:short".to_string(), + stamped: false, + version_file: None, + info_file: None, + }; + write_build_data(&opts2).expect("build data should be writable"); + + let content = fs::read_to_string(&output_path).unwrap(); + assert_eq!(content, "TARGET //second:short\nSTAMPED FALSE\n"); + + let _ = fs::remove_dir_all(&temp_dir); +} + +#[test] +fn test_missing_status_file_errors() { + let temp_dir = create_temp_dir("errors"); + + let opts = Options { + output: temp_dir.join("out.txt"), + target: "//t:t".to_string(), + stamped: true, + version_file: Some(temp_dir.join("does_not_exist.txt")), + info_file: None, + }; + let err = write_build_data(&opts).unwrap_err(); + assert!( + format!("{err:#}").contains("does_not_exist.txt"), + "unexpected error: {err:#}" + ); + + let _ = fs::remove_dir_all(&temp_dir); +} diff --git a/tests/shrubbery/shrubbery_test.rs b/tests/shrubbery/shrubbery_test.rs index 199fbb463e..d1a889279f 100644 --- a/tests/shrubbery/shrubbery_test.rs +++ b/tests/shrubbery/shrubbery_test.rs @@ -17,7 +17,7 @@ fn temp_test_dir(prefix: &str) -> PathBuf { #[test] fn test_subcommands_registry_sorted_and_complete() { - assert_eq!(SUBCOMMANDS.len(), 9); + assert_eq!(SUBCOMMANDS.len(), 10); let mut sorted = SUBCOMMANDS.to_vec(); sorted.sort_unstable(); assert_eq!(SUBCOMMANDS, sorted.as_slice());