diff --git a/.github/workflows/benchmark-build-comparison.yml b/.github/workflows/benchmark-build-comparison.yml index b95d934b..b815ba48 100644 --- a/.github/workflows/benchmark-build-comparison.yml +++ b/.github/workflows/benchmark-build-comparison.yml @@ -151,11 +151,10 @@ jobs: "- fbuild_overhead_ms: \(.fbuild_overhead_ms // "n/a")", "- fbuild_vs_platformio_cold: \(.fbuild_vs_platformio_cold // "n/a")"' "${latest}" echo - echo "| board | tool | phase | cold ms |" - echo "|---|---|---|---:|" - jq -r '.results[]? | select(.tool == "fbuild" or .tool == "platformio") - | . as $r | (.cold_phases_ms // {}) | to_entries[]? - | "| \($r.board) | \($r.tool) | \(.key) | \(.value // "n/a") |"' "${latest}" + echo "| fbuild phase | cold ms |" + echo "|---|---:|" + jq -r '([.results[]? | select(.tool == "fbuild") | .cold_phases_ms // {}] | first // {}) + | to_entries[]? | "| \(.key) | \(.value // "n/a") |"' "${latest}" } >> "${GITHUB_STEP_SUMMARY}" - name: Upload benchmark site snapshot diff --git a/.github/workflows/check-ubuntu.yml b/.github/workflows/check-ubuntu.yml index 2ef2ed0b..cc7e4747 100644 --- a/.github/workflows/check-ubuntu.yml +++ b/.github/workflows/check-ubuntu.yml @@ -42,7 +42,7 @@ jobs: - uses: astral-sh/setup-uv@v3 - name: Setup soldr id: setup-soldr - uses: zackees/setup-soldr@67ed4018aca013f8388050ac9bc264244f9b742c + uses: zackees/setup-soldr@869aa7b888b348632cb7c3c71ef1b738f131ee61 with: cache: true build-cache: true @@ -97,7 +97,7 @@ jobs: with: python-version: "3.10" - name: Setup soldr - uses: zackees/setup-soldr@67ed4018aca013f8388050ac9bc264244f9b742c + uses: zackees/setup-soldr@869aa7b888b348632cb7c3c71ef1b738f131ee61 with: cache: true build-cache: true diff --git a/.github/workflows/check-windows.yml b/.github/workflows/check-windows.yml index 01e0f225..ee315312 100644 --- a/.github/workflows/check-windows.yml +++ b/.github/workflows/check-windows.yml @@ -35,7 +35,7 @@ jobs: ref: ${{ inputs.ref }} - name: Setup soldr id: setup-soldr - uses: zackees/setup-soldr@67ed4018aca013f8388050ac9bc264244f9b742c + uses: zackees/setup-soldr@869aa7b888b348632cb7c3c71ef1b738f131ee61 with: cache: true build-cache: true diff --git a/Cargo.lock b/Cargo.lock index bfbdf512..083129a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1119,7 +1119,6 @@ dependencies = [ "fbuild-test-support", "filetime", "flate2", - "futures", "object 0.36.7", "owo-colors", "regex", @@ -1333,14 +1332,12 @@ version = "2.5.29" dependencies = [ "async-trait", "axum 0.7.9", - "blake3", "bzip2", "fbuild-config", "fbuild-core", "fbuild-packages-fetch", "fbuild-paths", "flate2", - "futures", "prost", "reqwest", "rusqlite", @@ -2297,9 +2294,9 @@ dependencies = [ [[package]] name = "kernal-api" -version = "0.1.23" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54844116729c2a5c64d5b313d79f839323c044cc01867f7e2bf0f192ac9e6903" +checksum = "b423567556c31193c3000b520ff23b82943de1d87c4515ab1428c742c0ce118a" dependencies = [ "blake3", "crash-handler", @@ -5089,7 +5086,7 @@ dependencies = [ [[package]] name = "zccache" version = "1.14.14" -source = "git+https://github.com/zackees/zccache?rev=b385a7bfb308c2d1456613622ad361eb66d31cdd#b385a7bfb308c2d1456613622ad361eb66d31cdd" +source = "git+https://github.com/zackees/zccache?rev=3f9d4e66fe3675cd48060bca03b62b1f5a8112eb#3f9d4e66fe3675cd48060bca03b62b1f5a8112eb" dependencies = [ "arc-swap", "bincode", @@ -5130,7 +5127,7 @@ dependencies = [ [[package]] name = "zccache-artifact" version = "1.14.14" -source = "git+https://github.com/zackees/zccache?rev=b385a7bfb308c2d1456613622ad361eb66d31cdd#b385a7bfb308c2d1456613622ad361eb66d31cdd" +source = "git+https://github.com/zackees/zccache?rev=3f9d4e66fe3675cd48060bca03b62b1f5a8112eb#3f9d4e66fe3675cd48060bca03b62b1f5a8112eb" dependencies = [ "bincode", "dashmap", @@ -5149,7 +5146,7 @@ dependencies = [ [[package]] name = "zccache-audit" version = "1.14.14" -source = "git+https://github.com/zackees/zccache?rev=b385a7bfb308c2d1456613622ad361eb66d31cdd#b385a7bfb308c2d1456613622ad361eb66d31cdd" +source = "git+https://github.com/zackees/zccache?rev=3f9d4e66fe3675cd48060bca03b62b1f5a8112eb#3f9d4e66fe3675cd48060bca03b62b1f5a8112eb" dependencies = [ "serde", "serde_json", @@ -5159,7 +5156,7 @@ dependencies = [ [[package]] name = "zccache-compile-trace" version = "1.14.14" -source = "git+https://github.com/zackees/zccache?rev=b385a7bfb308c2d1456613622ad361eb66d31cdd#b385a7bfb308c2d1456613622ad361eb66d31cdd" +source = "git+https://github.com/zackees/zccache?rev=3f9d4e66fe3675cd48060bca03b62b1f5a8112eb#3f9d4e66fe3675cd48060bca03b62b1f5a8112eb" dependencies = [ "zccache-audit", ] @@ -5167,7 +5164,7 @@ dependencies = [ [[package]] name = "zccache-compiler" version = "1.14.14" -source = "git+https://github.com/zackees/zccache?rev=b385a7bfb308c2d1456613622ad361eb66d31cdd#b385a7bfb308c2d1456613622ad361eb66d31cdd" +source = "git+https://github.com/zackees/zccache?rev=3f9d4e66fe3675cd48060bca03b62b1f5a8112eb#3f9d4e66fe3675cd48060bca03b62b1f5a8112eb" dependencies = [ "clang", "kernal-api", @@ -5181,7 +5178,7 @@ dependencies = [ [[package]] name = "zccache-core" version = "1.14.14" -source = "git+https://github.com/zackees/zccache?rev=b385a7bfb308c2d1456613622ad361eb66d31cdd#b385a7bfb308c2d1456613622ad361eb66d31cdd" +source = "git+https://github.com/zackees/zccache?rev=3f9d4e66fe3675cd48060bca03b62b1f5a8112eb#3f9d4e66fe3675cd48060bca03b62b1f5a8112eb" dependencies = [ "kernal-api", "serde", @@ -5193,7 +5190,7 @@ dependencies = [ [[package]] name = "zccache-daemon-core" version = "1.14.14" -source = "git+https://github.com/zackees/zccache?rev=b385a7bfb308c2d1456613622ad361eb66d31cdd#b385a7bfb308c2d1456613622ad361eb66d31cdd" +source = "git+https://github.com/zackees/zccache?rev=3f9d4e66fe3675cd48060bca03b62b1f5a8112eb#3f9d4e66fe3675cd48060bca03b62b1f5a8112eb" dependencies = [ "arc-swap", "bincode", @@ -5206,7 +5203,6 @@ dependencies = [ "regex", "serde", "serde_json", - "sha2", "tempfile", "thiserror 2.0.20", "tokio", @@ -5229,7 +5225,7 @@ dependencies = [ [[package]] name = "zccache-depgraph" version = "1.14.14" -source = "git+https://github.com/zackees/zccache?rev=b385a7bfb308c2d1456613622ad361eb66d31cdd#b385a7bfb308c2d1456613622ad361eb66d31cdd" +source = "git+https://github.com/zackees/zccache?rev=3f9d4e66fe3675cd48060bca03b62b1f5a8112eb#3f9d4e66fe3675cd48060bca03b62b1f5a8112eb" dependencies = [ "bincode", "dashmap", @@ -5249,7 +5245,7 @@ dependencies = [ [[package]] name = "zccache-fingerprint" version = "1.14.14" -source = "git+https://github.com/zackees/zccache?rev=b385a7bfb308c2d1456613622ad361eb66d31cdd#b385a7bfb308c2d1456613622ad361eb66d31cdd" +source = "git+https://github.com/zackees/zccache?rev=3f9d4e66fe3675cd48060bca03b62b1f5a8112eb#3f9d4e66fe3675cd48060bca03b62b1f5a8112eb" dependencies = [ "kernal-api", "rayon", @@ -5264,7 +5260,7 @@ dependencies = [ [[package]] name = "zccache-fscache" version = "1.14.14" -source = "git+https://github.com/zackees/zccache?rev=b385a7bfb308c2d1456613622ad361eb66d31cdd#b385a7bfb308c2d1456613622ad361eb66d31cdd" +source = "git+https://github.com/zackees/zccache?rev=3f9d4e66fe3675cd48060bca03b62b1f5a8112eb#3f9d4e66fe3675cd48060bca03b62b1f5a8112eb" dependencies = [ "bincode", "dashmap", @@ -5279,7 +5275,7 @@ dependencies = [ [[package]] name = "zccache-hash" version = "1.14.14" -source = "git+https://github.com/zackees/zccache?rev=b385a7bfb308c2d1456613622ad361eb66d31cdd#b385a7bfb308c2d1456613622ad361eb66d31cdd" +source = "git+https://github.com/zackees/zccache?rev=3f9d4e66fe3675cd48060bca03b62b1f5a8112eb#3f9d4e66fe3675cd48060bca03b62b1f5a8112eb" dependencies = [ "kernal-api", "serde", @@ -5288,7 +5284,7 @@ dependencies = [ [[package]] name = "zccache-ipc" version = "1.14.14" -source = "git+https://github.com/zackees/zccache?rev=b385a7bfb308c2d1456613622ad361eb66d31cdd#b385a7bfb308c2d1456613622ad361eb66d31cdd" +source = "git+https://github.com/zackees/zccache?rev=3f9d4e66fe3675cd48060bca03b62b1f5a8112eb#3f9d4e66fe3675cd48060bca03b62b1f5a8112eb" dependencies = [ "bytes", "dashmap", @@ -5306,7 +5302,7 @@ dependencies = [ [[package]] name = "zccache-protocol" version = "1.14.14" -source = "git+https://github.com/zackees/zccache?rev=b385a7bfb308c2d1456613622ad361eb66d31cdd#b385a7bfb308c2d1456613622ad361eb66d31cdd" +source = "git+https://github.com/zackees/zccache?rev=3f9d4e66fe3675cd48060bca03b62b1f5a8112eb#3f9d4e66fe3675cd48060bca03b62b1f5a8112eb" dependencies = [ "bytes", "kernal-api", @@ -5322,7 +5318,7 @@ dependencies = [ [[package]] name = "zccache-watcher" version = "1.14.14" -source = "git+https://github.com/zackees/zccache?rev=b385a7bfb308c2d1456613622ad361eb66d31cdd#b385a7bfb308c2d1456613622ad361eb66d31cdd" +source = "git+https://github.com/zackees/zccache?rev=3f9d4e66fe3675cd48060bca03b62b1f5a8112eb#3f9d4e66fe3675cd48060bca03b62b1f5a8112eb" dependencies = [ "kernal-api", "rayon", diff --git a/bench/fastled-examples/src/README.md b/bench/fastled-examples/src/README.md index ce7eaced..b44992e2 100644 --- a/bench/fastled-examples/src/README.md +++ b/bench/fastled-examples/src/README.md @@ -7,9 +7,6 @@ Sources for the repository's end-to-end benchmark binaries: fbuild Blink build comparison and static-site renderer. - `build_comparison_tests.rs` covers the comparison runner's cold-cache sequencing, output metadata, and renderer behavior. -- `pio_phases.rs` splits a timed `pio run -v` build into `pre-compile`, - `compile`, `link` and `convert-size` from per-line arrival stamps, so - PlatformIO's cold build can be compared with fbuild's perf-log phases. See the parent [`README.md`](../README.md) for the FastLED harness and [`../../blink/README.md`](../../blink/README.md) for the whole-build benchmark. diff --git a/bench/fastled-examples/src/build_comparison.rs b/bench/fastled-examples/src/build_comparison.rs index 0835933e..003476c3 100644 --- a/bench/fastled-examples/src/build_comparison.rs +++ b/bench/fastled-examples/src/build_comparison.rs @@ -3,9 +3,6 @@ //! The harness measures the same Arduino Uno and ESP32-S3 Blink sketch with each real CLI, //! then renders the one-commit benchmark site's JSON, SVG, and HTML artifacts. -#[path = "pio_phases.rs"] -mod pio_phases; - use fbuild_core::path::NormalizedPath; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; @@ -156,19 +153,15 @@ struct ToolResult { speedup: f64, cold_trials_ms: Vec, warm_trials_ms: Vec, - /// Per-phase cold medians: fbuild's perf log, or PlatformIO's `-v` output - /// split by `pio_phases` (empty for other tools). + /// Per-phase medians of fbuild's `avr-orchestrator` perf log (empty for other tools). cold_phases_ms: BTreeMap, - /// Raw per-trial phase timings for each cold build (fbuild and PlatformIO). + /// Raw per-trial phase timings for each cold build (empty for other tools). cold_phase_trials: Vec>, /// Timed fbuild builds that restarted the daemon (FastLED/fbuild#1476); /// each adds ~200 ms, so a non-zero count marks inflated fbuild timings. daemon_restarts: usize, /// Package versions actually selected by this tool, not requested pins. resolved_packages: BTreeMap, - /// Why package identity was unavailable, if a successful build omitted it. - #[serde(skip_serializing_if = "Option::is_none")] - package_metadata_warning: Option, /// Distinct Arduino core sources in the tool's compile database. core_source_count: Option, /// Full compiler invocation for a representative core source. @@ -338,7 +331,6 @@ fn measure_tool( let mut cold_phase_trials = Vec::new(); let mut daemon_restarts = 0; let mut resolved_packages = BTreeMap::new(); - let mut package_metadata_warning = None; let perf_log = output_dir.join(PERF_LOG_FILE); let envs = tool_envs(kind, &perf_log); @@ -376,12 +368,7 @@ fn measure_tool( } MeasurementStep::ColdBuild(trial) => { let offset = perf_line_count(&perf_log); - let TimedBuild { - elapsed_ms: elapsed, - restarted, - packages, - phases: pio_phases, - } = timed_build( + let (elapsed, restarted, packages) = timed_build( kind, board, options, @@ -393,19 +380,10 @@ fn measure_tool( &envs, )?; daemon_restarts += usize::from(restarted); - if matches!(kind, ToolKind::PlatformIo | ToolKind::Fbuild) { - record_package_metadata( - &mut resolved_packages, - &mut package_metadata_warning, - packages, - kind, - board, - ); + if matches!(kind, ToolKind::PlatformIo) && !packages.is_empty() { + resolved_packages = packages; } cold_trials_ms.push(round_millis(elapsed)); - if let Some(phases) = pio_phases { - cold_phase_trials.push(phases); - } if matches!(kind, ToolKind::Fbuild) { let content = fs::read_to_string(&perf_log).unwrap_or_default(); if let Some(phases) = perf_phases_after(&content, offset, PERF_PHASE_LABELS) { @@ -424,11 +402,7 @@ fn measure_tool( } } MeasurementStep::WarmBuild(_) => { - let TimedBuild { - elapsed_ms: elapsed, - restarted, - .. - } = timed_build( + let (elapsed, restarted, _) = timed_build( kind, board, options, @@ -445,20 +419,22 @@ fn measure_tool( } } - if matches!(kind, ToolKind::PlatformIo | ToolKind::Fbuild) - && !package_metadata_is_complete(board.key, &resolved_packages) - { - let warning = package_metadata_warning.get_or_insert_with(|| { - format!( - "{} build output omitted one or more required resolved package identities", - kind.style().label - ) - }); - println!( - "::warning title=benchmark package metadata unavailable::{} ({})", - board.name, warning - ); - writeln!(log, "Package metadata warning: {warning}")?; + if matches!(kind, ToolKind::Fbuild) { + let output = run_logged_env( + fbuild.as_os_str(), + &os_args(&[ + "install", + &project_dir.to_string_lossy(), + "--environment", + board.environment, + "--check", + "--json", + ]), + repo_root, + log, + &[], + )?; + resolved_packages = parse_fbuild_packages(&output.stdout)?; } if matches!(kind, ToolKind::PlatformIo) { run_logged_env( @@ -514,7 +490,6 @@ fn measure_tool( cold_phase_trials, daemon_restarts, resolved_packages, - package_metadata_warning, core_source_count, core_compile_argv, }) @@ -954,7 +929,7 @@ fn timed_build( arduino_build_dir: &Path, log: &mut File, envs: &[(&str, OsString)], -) -> AppResult { +) -> AppResult<(f64, bool, BTreeMap)> { let (program, args) = match kind { ToolKind::Arduino => ( options.arduino_cli.as_os_str(), @@ -975,8 +950,6 @@ fn timed_build( &project_dir.to_string_lossy(), "--environment", board.environment, - // Prints each command as it starts: the phase breakdown's input. - "-v", ]), ), ToolKind::Fbuild => ( @@ -992,180 +965,14 @@ fn timed_build( }; let started = Instant::now(); - let (output, lines) = run_logged_env_stamped(program, &args, repo_root, log, envs)?; + let output = run_logged_env(program, &args, repo_root, log, envs)?; let elapsed_ms = started.elapsed().as_secs_f64() * 1000.0; - let packages = match kind { - ToolKind::PlatformIo => parse_platformio_packages(&output.stdout, board), - ToolKind::Fbuild => parse_fbuild_build_packages(&output.stdout, board), - ToolKind::Arduino => BTreeMap::new(), - }; - let phases = match kind { - ToolKind::PlatformIo => pio_phases::phases(&lines, elapsed_ms), - _ => None, + let packages = if matches!(kind, ToolKind::PlatformIo) { + parse_platformio_packages(&output.stdout, board) + } else { + BTreeMap::new() }; - Ok(TimedBuild { - elapsed_ms, - restarted: restarted_daemon(&output.stderr), - packages, - phases, - }) -} - -struct TimedBuild { - elapsed_ms: f64, - restarted: bool, - packages: BTreeMap, - /// PlatformIO phase breakdown parsed from `-v` output (see `pio_phases`). - phases: Option>, -} - -/// [`run_logged_env`] that also stamps every stdout line with its arrival -/// time (ms since spawn). `PYTHONUNBUFFERED` keeps SCons from batching -/// output, so a stamp is when the command started, not when a buffer filled. -fn run_logged_env_stamped( - program: &OsStr, - args: &[OsString], - cwd: &Path, - log: &mut File, - envs: &[(&str, OsString)], -) -> AppResult<(Output, Vec<(f64, String)>)> { - use std::io::{BufRead, Read}; - writeln!(log, "$ {}", display_command(program, args))?; - log.flush()?; - let started = Instant::now(); - let mut child = Command::new(program) - .args(args) - .current_dir(cwd) - .env("CI", "true") - .env("PLATFORMIO_SETTING_ENABLE_TELEMETRY", "no") - .env("PYTHONUNBUFFERED", "1") - .envs(envs.iter().map(|(key, value)| (*key, value))) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .spawn()?; - let mut stderr_pipe = child.stderr.take().expect("piped stderr"); - let stderr_reader = std::thread::spawn(move || { - let mut bytes = Vec::new(); - let _ = stderr_pipe.read_to_end(&mut bytes); - bytes - }); - let mut stdout = Vec::new(); - let mut lines = Vec::new(); - let mut reader = io::BufReader::new(child.stdout.take().expect("piped stdout")); - let mut line = Vec::new(); - while reader.read_until(b'\n', &mut line)? > 0 { - let at = started.elapsed().as_secs_f64() * 1000.0; - stdout.extend_from_slice(&line); - lines.push((at, String::from_utf8_lossy(&line).trim_end().to_string())); - line.clear(); - } - let status = child.wait()?; - let stderr = stderr_reader.join().unwrap_or_default(); - log.write_all(&stdout)?; - log.write_all(&stderr)?; - log.flush()?; - if !status.success() { - return Err(io::Error::other(format!( - "command failed with {status}: {}; see benchmark-output/benchmark.log", - display_command(program, args) - )) - .into()); - } - Ok(( - Output { - status, - stdout, - stderr, - }, - lines, - )) -} - -fn record_package_metadata( - current: &mut BTreeMap, - warning: &mut Option, - observed: BTreeMap, - kind: ToolKind, - board: Board, -) { - if observed.is_empty() { - current.clear(); - if warning.is_none() { - *warning = Some(format!( - "{} build output omitted resolved package identities on {}", - kind.style().label, - board.name - )); - } - return; - } - // Every timed cold trial must agree. Once any trial is missing or has a - // different identity, later observations cannot restore comparability. - if warning.is_some() { - return; - } - if current.is_empty() { - *current = observed; - } else if *current != observed { - current.clear(); - *warning = Some(format!( - "{} resolved package identities changed across cold trials on {}", - kind.style().label, - board.name - )); - } -} - -fn parse_fbuild_build_packages(stdout: &[u8], board: Board) -> BTreeMap { - let mut packages = BTreeMap::new(); - for line in String::from_utf8_lossy(stdout).lines() { - if board.key == "esp32s3" { - let Some((_, resolved)) = line - .split_once("ESP32 packages:") - .and_then(|(_, packages)| packages.split_once("; resolved ")) - else { - continue; - }; - for (key, marker) in [ - ("platform", "platform="), - ("framework", "framework="), - ("toolchain", "toolchain="), - ("sdk", "ESP-IDF SDK="), - ] { - if let Some(value) = resolved - .split_once(marker) - .map(|(_, value)| value.split([',', ';']).next().unwrap_or("").trim()) - { - let version = value.rsplit_once('@').map_or(value, |(_, version)| version); - if !version.is_empty() { - packages.insert(key.to_string(), version.to_string()); - } - } - } - } else if board.key == "uno" { - if let Some((_, resolved)) = line.split_once("AVR resolved:") { - for (key, package) in [ - ("toolchain", "toolchain-atmelavr"), - ("framework", "framework-"), - ] { - let field = if key == "toolchain" { - resolved.split(';').next().unwrap_or("") - } else { - resolved.split(';').nth(1).unwrap_or("") - }; - if let Some(value) = field.split_once(package).map(|(_, value)| value) { - if let Some((_, version)) = value.split_once('@') { - let version = version.split_whitespace().next().unwrap_or(""); - if !version.is_empty() { - packages.insert(key.to_string(), version.to_string()); - } - } - } - } - } - } - } - packages + Ok((elapsed_ms, restarted_daemon(&output.stderr), packages)) } fn parse_platformio_packages(stdout: &[u8], board: Board) -> BTreeMap { @@ -1210,6 +1017,28 @@ fn parse_platformio_packages(stdout: &[u8], board: Board) -> BTreeMap AppResult> { + let value: Value = serde_json::from_slice(stdout)?; + let packages = value["environments"][0]["packages"] + .as_array() + .ok_or_else(|| io::Error::other("fbuild install --json omitted packages"))?; + let mut resolved = BTreeMap::new(); + for package in packages { + let Some(kind) = package["kind"].as_str() else { + continue; + }; + let key = match kind { + "platform" | "framework" | "toolchain" => kind, + "tool" if package["name"] == "tool-esptoolpy" => "flash_tool", + _ => continue, + }; + if let Some(version) = package["version"].as_str() { + resolved.insert(key.to_string(), version.to_string()); + } + } + Ok(resolved) +} + fn core_compile_metadata(path: &Path) -> AppResult<(Option, Option>)> { let entries: Vec = serde_json::from_slice(&fs::read(path)?)?; let core = entries @@ -1447,30 +1276,6 @@ fn board_cold_ratio(results: &[ToolResult], board: &str) -> Option { } fn board_stack_comparable(results: &[ToolResult], board: &str) -> bool { - board_stack_status(results, board) == StackStatus::Matched -} - -fn stack_package_keys(board: &str) -> &'static [&'static str] { - match board { - "esp32s3" => &["platform", "framework", "toolchain"], - "uno" => &["framework", "toolchain"], - _ => &[], - } -} - -fn package_metadata_is_complete(board: &str, packages: &BTreeMap) -> bool { - let required = stack_package_keys(board); - !required.is_empty() && required.iter().all(|key| packages.contains_key(*key)) -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum StackStatus { - Matched, - Different, - Unverified, -} - -fn board_stack_status(results: &[ToolResult], board: &str) -> StackStatus { results .iter() .find(|result| result.board == board && result.tool == "fbuild") @@ -1479,30 +1284,9 @@ fn board_stack_status(results: &[ToolResult], board: &str) -> StackStatus { .iter() .find(|result| result.board == board && result.tool == "platformio"), ) - .map_or(StackStatus::Unverified, |(fbuild, pio)| { - let keys = stack_package_keys(board); - if keys.is_empty() { - return StackStatus::Unverified; - } - let resolved = keys - .iter() - .map(|key| { - fbuild - .resolved_packages - .get(*key) - .zip(pio.resolved_packages.get(*key)) - }) - .collect::>(); - if resolved.iter().any(Option::is_none) { - StackStatus::Unverified - } else if resolved - .iter() - .all(|pair| pair.is_some_and(|(fbuild, pio)| fbuild == pio)) - { - StackStatus::Matched - } else { - StackStatus::Different - } + .is_some_and(|(fbuild, pio)| { + !fbuild.resolved_packages.is_empty() + && fbuild.resolved_packages == pio.resolved_packages }) } @@ -1515,11 +1299,6 @@ fn board_metrics(metadata: &Metadata, results: &[ToolResult]) -> Value { json!({ "fbuild_vs_platformio_cold": board_cold_ratio(results, board.key), "stack_comparable": board_stack_comparable(results, board.key), - "stack_status": match board_stack_status(results, board.key) { - StackStatus::Matched => "matched", - StackStatus::Different => "different", - StackStatus::Unverified => "unverified", - }, "raw_baseline_ms": metadata.raw_baselines_ms.get(board.key), "fbuild_overhead_ms": cold_of(results, board.key, "fbuild") .zip(metadata.raw_baselines_ms.get(board.key)) @@ -1719,10 +1498,12 @@ fn render_svg(metadata: &Metadata, results: &[ToolResult]) -> String { .max(1.0) }; let mut rows = String::new(); - let stack_note = |board: &str| match board_stack_status(results, board) { - StackStatus::Matched => " | fbuild/PIO stack matched", - StackStatus::Different => " | fbuild/PIO stack differs; ratio excluded", - StackStatus::Unverified => " | fbuild/PIO stack unverified; ratio excluded", + let stack_note = |board: &str| { + if board_stack_comparable(results, board) { + " | fbuild/PIO stack matched" + } else { + " | fbuild/PIO stack differs; ratio excluded" + } }; for (index, result) in results.iter().enumerate() { let kind = match result.tool.as_str() { @@ -1835,10 +1616,10 @@ fn render_html(metadata: &Metadata, results: &[ToolResult]) -> String { let comparison_note = BOARDS .iter() .map(|board| { - let status = match board_stack_status(results, board.key) { - StackStatus::Matched => "matched; fbuild/PlatformIO ratio shown", - StackStatus::Different => "different; fbuild/PlatformIO ratio excluded", - StackStatus::Unverified => "unverified; fbuild/PlatformIO ratio excluded", + let status = if board_stack_comparable(results, board.key) { + "matched; fbuild/PlatformIO ratio shown" + } else { + "different or unverified; fbuild/PlatformIO ratio excluded" }; format!("{}: {}", board.name, status) }) @@ -1861,18 +1642,17 @@ fn render_html(metadata: &Metadata, results: &[ToolResult]) -> String { .join("\n"); let phase_rows = results .iter() - .filter(|result| result.tool == "fbuild" || result.tool == "platformio") + .filter(|result| result.tool == "fbuild") .flat_map(|result| { result .cold_phases_ms .iter() - .map(move |(phase, ms)| (result, phase, ms)) + .map(move |(phase, ms)| (&result.board_name, phase, ms)) }) - .map(|(result, phase, ms)| { + .map(|(board, phase, ms)| { format!( - "{}{}{}{ms:.3} ms", - html_escape(&result.board_name), - html_escape(&result.display_name), + "{}{}{ms:.3} ms", + html_escape(board), html_escape(phase) ) }) @@ -1882,10 +1662,10 @@ fn render_html(metadata: &Metadata, results: &[ToolResult]) -> String { String::new() } else { format!( - r#"

Cold phase breakdown

+ r#"

fbuild cold phase breakdown

- + {phase_rows}
BoardToolPhaseCold median
BoardPhaseCold median
diff --git a/bench/fastled-examples/src/build_comparison_tests.rs b/bench/fastled-examples/src/build_comparison_tests.rs index 79738e0e..be87062f 100644 --- a/bench/fastled-examples/src/build_comparison_tests.rs +++ b/bench/fastled-examples/src/build_comparison_tests.rs @@ -35,11 +35,7 @@ fn sample_results() -> Vec { cold_phases_ms: BTreeMap::new(), cold_phase_trials: Vec::new(), daemon_restarts: 0, - resolved_packages: BTreeMap::from([ - ("framework".into(), "1.8.8".into()), - ("toolchain".into(), "7.3.0".into()), - ]), - package_metadata_warning: None, + resolved_packages: BTreeMap::from([("framework".into(), "1.8.8".into())]), core_source_count: None, core_compile_argv: None, }, @@ -57,11 +53,7 @@ fn sample_results() -> Vec { cold_phases_ms: BTreeMap::new(), cold_phase_trials: Vec::new(), daemon_restarts: 0, - resolved_packages: BTreeMap::from([ - ("framework".into(), "1.8.8".into()), - ("toolchain".into(), "7.3.0".into()), - ]), - package_metadata_warning: None, + resolved_packages: BTreeMap::from([("framework".into(), "1.8.8".into())]), core_source_count: None, core_compile_argv: None, }, @@ -79,11 +71,7 @@ fn sample_results() -> Vec { cold_phases_ms: BTreeMap::from([("compile".to_string(), 400.0)]), cold_phase_trials: vec![BTreeMap::from([("compile".to_string(), 400.0)])], daemon_restarts: 0, - resolved_packages: BTreeMap::from([ - ("framework".into(), "1.8.8".into()), - ("toolchain".into(), "7.3.0".into()), - ]), - package_metadata_warning: None, + resolved_packages: BTreeMap::from([("framework".into(), "1.8.8".into())]), core_source_count: None, core_compile_argv: None, }, @@ -517,15 +505,10 @@ fn esp32_regression_is_not_hidden_by_uno_ratio() { pio.board = "esp32s3".into(); pio.board_name = "ESP32-S3".into(); pio.cold_ms = 6_000.0; - pio.resolved_packages - .insert("platform".into(), "6.13.0".into()); let mut fbuild = results[2].clone(); fbuild.board = "esp32s3".into(); fbuild.board_name = "ESP32-S3".into(); fbuild.cold_ms = 15_000.0; - fbuild - .resolved_packages - .insert("platform".into(), "6.13.0".into()); results.extend([pio, fbuild]); let mut metadata = sample_metadata(); @@ -576,146 +559,16 @@ fn different_resolved_esp32_stacks_cannot_publish_a_ratio() { } #[test] -fn build_output_package_parsers_capture_the_same_esp32_stack() { +fn installed_package_parsers_capture_the_same_esp32_stack() { let board = BOARDS[1]; let pio = parse_platformio_packages( b"Processing esp32s3 (platform: espressif32@6.13.0; board: esp32-s3-devkitc-1; framework: arduino)\nPACKAGES:\n - framework-arduinoespressif32 @ 3.20017.241212+sha.dcc1105b\n - tool-esptoolpy @ 2.41100.260830 (4.11.0)\n - toolchain-riscv32-esp @ 8.4.0+2021r2-patch5\n - toolchain-xtensa-esp32s3 @ 8.4.0+2021r2-patch5\n", board, ); - let fbuild = parse_fbuild_build_packages( - b"ESP32 packages: requested platform=espressif32@6.13.0, framework=old-framework@1.0; resolved platform=espressif32@6.13.0, framework=framework-arduinoespressif32@3.20017.241212+sha.dcc1105b, toolchain=toolchain-xtensa-esp32s3@8.4.0+2021r2-patch5, ESP-IDF SDK=4.4.7\n", - board, - ); - assert_eq!(pio["platform"], fbuild["platform"]); - assert_eq!(pio["framework"], fbuild["framework"]); - assert_eq!(pio["toolchain"], fbuild["toolchain"]); - assert_eq!(fbuild["sdk"], "4.4.7"); - - let mut results = sample_results(); - for result in &mut results { - result.board = "esp32s3".into(); - result.board_name = "ESP32-S3".into(); - } - results[1].resolved_packages = pio; - results[2].resolved_packages = fbuild; - assert_eq!( - board_stack_status(&results, "esp32s3"), - StackStatus::Matched - ); -} - -#[test] -fn fbuild_avr_build_log_reports_selected_package_versions() { - let packages = parse_fbuild_build_packages( - b"AVR resolved: toolchain-atmelavr@1.70300.191015 (https://example.test/toolchain); framework-arduino@1.8.8 (https://example.test/framework),\n", - BOARDS[0], - ); - assert_eq!(packages["toolchain"], "1.70300.191015"); - assert_eq!(packages["framework"], "1.8.8"); -} - -#[test] -fn changed_package_identities_across_trials_become_unverified() { - let mut packages = BTreeMap::new(); - let mut warning = None; - let first = BTreeMap::from([ - ("platform".into(), "6.13.0".into()), - ("framework".into(), "3.20017.241212+sha.dcc1105b".into()), - ("toolchain".into(), "8.4.0+2021r2-patch5".into()), - ]); - record_package_metadata( - &mut packages, - &mut warning, - first.clone(), - ToolKind::Fbuild, - BOARDS[1], - ); - assert!(package_metadata_is_complete("esp32s3", &packages)); - - let mut changed = first.clone(); - changed.insert("toolchain".into(), "14.2.0".into()); - record_package_metadata( - &mut packages, - &mut warning, - changed, - ToolKind::Fbuild, - BOARDS[1], - ); - record_package_metadata( - &mut packages, - &mut warning, - first, - ToolKind::Fbuild, - BOARDS[1], - ); - assert!(packages.is_empty()); - assert!( - warning - .as_deref() - .is_some_and(|message| message.contains("changed across cold trials")) - ); -} - -#[test] -fn missing_package_identity_in_any_cold_trial_stays_unverified() { - let complete = BTreeMap::from([ - ("platform".into(), "6.13.0".into()), - ("framework".into(), "3.20017.241212+sha.dcc1105b".into()), - ("toolchain".into(), "8.4.0+2021r2-patch5".into()), - ]); - for observations in [ - vec![BTreeMap::new(), complete.clone()], - vec![complete.clone(), BTreeMap::new()], - ] { - let mut packages = BTreeMap::new(); - let mut warning = None; - for observed in observations { - record_package_metadata( - &mut packages, - &mut warning, - observed, - ToolKind::Fbuild, - BOARDS[1], - ); - } - assert!(packages.is_empty()); - assert!( - warning - .as_deref() - .is_some_and(|message| message.contains("omitted resolved package identities")) - ); - } -} - -#[test] -fn unavailable_fbuild_package_versions_are_not_reported_as_a_mismatch() { - let mut results = sample_results(); - for result in &mut results { - result.board = "esp32s3".into(); - result.board_name = "ESP32-S3".into(); - result.resolved_packages = BTreeMap::from([ - ("platform".into(), "6.13.0".into()), - ("framework".into(), "3.20017.241212+sha.dcc1105b".into()), - ("toolchain".into(), "8.4.0+2021r2-patch5".into()), - ]); - } - results[2].resolved_packages.clear(); - results[2].package_metadata_warning = Some("build output omitted package identities".into()); - let latest = latest_payload(&sample_metadata(), &results); - assert_eq!( - latest["board_metrics"]["esp32s3"]["stack_status"], - "unverified" - ); - assert!(latest["board_metrics"]["esp32s3"]["fbuild_vs_platformio_cold"].is_null()); - assert_eq!( - latest["results"][2]["package_metadata_warning"], - "build output omitted package identities" - ); - assert!(render_svg(&sample_metadata(), &results).contains("stack unverified; ratio excluded")); - assert!( - render_html(&sample_metadata(), &results) - .contains("unverified; fbuild/PlatformIO ratio excluded") - ); + let fbuild = parse_fbuild_packages( + br#"{"environments":[{"packages":[{"kind":"platform","name":"platform-espressif32","version":"6.13.0"},{"kind":"framework","name":"esp32-arduino","version":"3.20017.241212+sha.dcc1105b"},{"kind":"toolchain","name":"toolchain-xtensa-esp32s3","version":"8.4.0+2021r2-patch5"},{"kind":"tool","name":"tool-esptoolpy","version":"2.41100.260830"}]}]}"#, + ).unwrap(); + assert_eq!(pio, fbuild); } #[test] @@ -801,7 +654,7 @@ fn svg_shows_raw_floor_and_overhead() { assert!(!svg.contains("raw compiler floor")); assert!(svg.contains("fbuild/PIO cold: 0.667")); let html = render_html(&sample_metadata(), &sample_results()); - assert!(html.contains("Cold phase breakdown")); + assert!(html.contains("fbuild cold phase breakdown")); } /// FastLED/fbuild#1467: the raw-compiler baseline must replay the real diff --git a/bench/fastled-examples/src/pio_phases.rs b/bench/fastled-examples/src/pio_phases.rs deleted file mode 100644 index ed9d18a5..00000000 --- a/bench/fastled-examples/src/pio_phases.rs +++ /dev/null @@ -1,135 +0,0 @@ -//! PlatformIO cold-build phase breakdown from `pio run -v` output. -//! -//! PlatformIO has no phase timer, but `-v` makes SCons print every command -//! just before it runs. Stamping each line on arrival gives the start time of -//! every compile, archive, link and image step, which is enough to split a -//! build into the same phases fbuild's perf log reports, so the two tools can -//! be compared phase by phase (FastLED/fbuild#1537). - -use std::collections::BTreeMap; - -/// What a verbose PlatformIO output line starts. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Step { - Compile, - Link, - Image, -} - -/// Classify one `pio run -v` line; `None` for everything that is not a -/// compile, the firmware link, or the firmware image/size step. -pub fn classify(line: &str) -> Option { - let line = line.trim(); - let writes = |suffix: &str| { - line.split_whitespace() - .collect::>() - .windows(2) - .any(|pair| pair[0] == "-o" && pair[1].trim_matches('"').ends_with(suffix)) - }; - if writes("firmware.elf") { - return Some(Step::Link); - } - let image_tool = line.contains("elf2image") || line.contains("objcopy"); - if image_tool && (line.contains("firmware.bin") || line.contains("firmware.hex")) { - return Some(Step::Image); - } - if line.starts_with("Checking size") { - return Some(Step::Image); - } - if line.contains(" -c ") && writes(".o") { - return Some(Step::Compile); - } - None -} - -/// Phase durations (ms) from `(arrival_ms, line)` pairs and the build's total. -/// -/// * `pre-compile`: start until the first compile starts -/// * `compile`: first compile until the firmware link starts (link waits for -/// every compile and archive) -/// * `link`: link start until the image step starts -/// * `convert-size`: image step until the build exits -/// -/// `None` when the output shows no compile or no link (e.g. a no-op build). -pub fn phases(lines: &[(f64, String)], total_ms: f64) -> Option> { - let first = |step: Step, after: f64| { - lines - .iter() - .find(|(at, line)| *at >= after && classify(line) == Some(step)) - .map(|(at, _)| *at) - }; - let compile = first(Step::Compile, 0.0)?; - let link = first(Step::Link, compile)?; - let image = first(Step::Image, link).unwrap_or(total_ms); - let round = |ms: f64| (ms * 1000.0).round() / 1000.0; - Some(BTreeMap::from([ - ("pre-compile".to_string(), round(compile)), - ("compile".to_string(), round(link - compile)), - ("link".to_string(), round(image - link)), - ( - "convert-size".to_string(), - round((total_ms - image).max(0.0)), - ), - ])) -} - -#[cfg(test)] -mod tests { - use super::*; - - const COMPILE: &str = "xtensa-esp32s3-elf-g++ -o .pio/build/esp32s3/FrameworkArduino/Esp.cpp.o -c -std=gnu++11 -Os Esp.cpp"; - const ARCHIVE: &str = "xtensa-esp32s3-elf-ar rc .pio/build/esp32s3/libFrameworkArduino.a a.o"; - const LINK: &str = "xtensa-esp32s3-elf-g++ -o .pio/build/esp32s3/firmware.elf -T memory.ld a.o"; - const IMAGE: &str = "esptool.py --chip esp32s3 elf2image --flash_mode dio -o .pio/build/esp32s3/firmware.bin .pio/build/esp32s3/firmware.elf"; - - #[test] - fn classifies_verbose_platformio_commands() { - assert_eq!(classify(COMPILE), Some(Step::Compile)); - assert_eq!(classify(ARCHIVE), None); - assert_eq!(classify(LINK), Some(Step::Link)); - assert_eq!(classify(IMAGE), Some(Step::Image)); - assert_eq!( - classify( - "avr-objcopy -O ihex -R .eeprom .pio/build/uno/firmware.elf .pio/build/uno/firmware.hex" - ), - Some(Step::Image) - ); - assert_eq!( - classify("Checking size .pio/build/uno/firmware.elf"), - Some(Step::Image) - ); - assert_eq!( - classify("Processing esp32s3 (platform: espressif32@6.13.0)"), - None - ); - } - - #[test] - fn bootloader_elf2image_before_compiles_is_not_the_image_phase() { - let boot = "esptool.py --chip esp32s3 elf2image -o .pio/build/esp32s3/bootloader.bin bootloader.elf"; - assert_eq!(classify(boot), None); - } - - #[test] - fn splits_a_build_into_phases() { - let lines = vec![ - (100.0, "Processing esp32s3".to_string()), - (800.0, COMPILE.to_string()), - (1500.0, COMPILE.to_string()), - (2400.0, ARCHIVE.to_string()), - (2500.0, LINK.to_string()), - (3800.0, IMAGE.to_string()), - ]; - let phases = phases(&lines, 4200.0).unwrap(); - assert_eq!(phases["pre-compile"], 800.0); - assert_eq!(phases["compile"], 1700.0); - assert_eq!(phases["link"], 1300.0); - assert_eq!(phases["convert-size"], 400.0); - } - - #[test] - fn no_link_means_no_breakdown() { - let lines = vec![(10.0, COMPILE.to_string())]; - assert!(phases(&lines, 50.0).is_none()); - } -} diff --git a/ci/platform_boundary_research.tsv b/ci/platform_boundary_research.tsv index 801f40bf..ab0119d7 100644 --- a/ci/platform_boundary_research.tsv +++ b/ci/platform_boundary_research.tsv @@ -12,11 +12,10 @@ crates/fbuild-core/src/platform/linux/device.rs 41 native_path libc:: process ho crates/fbuild-core/src/platform/linux/fs.rs 2 native_path std::os::unix::fs::PermissionsExt fs host_mechanic crates/fbuild-core/src/platform/linux/fs.rs 13 native_path std::os::unix::fs::MetadataExt fs host_mechanic crates/fbuild-core/src/platform/linux/fs.rs 57 native_path std::os::unix::fs::symlink fs host_mechanic -crates/fbuild-core/src/platform/linux/fs.rs 61 native_path std::os::unix::fs::symlink fs host_mechanic -crates/fbuild-core/src/platform/linux/fs.rs 105 native_path std::os::unix::ffi::OsStrExt process host_mechanic -crates/fbuild-core/src/platform/linux/fs.rs 110 native_path libc:: process host_mechanic -crates/fbuild-core/src/platform/linux/fs.rs 112 native_path libc:: process host_mechanic -crates/fbuild-core/src/platform/linux/fs.rs 122 native_path libc:: process host_mechanic +crates/fbuild-core/src/platform/linux/fs.rs 101 native_path std::os::unix::ffi::OsStrExt process host_mechanic +crates/fbuild-core/src/platform/linux/fs.rs 106 native_path libc:: process host_mechanic +crates/fbuild-core/src/platform/linux/fs.rs 108 native_path libc:: process host_mechanic +crates/fbuild-core/src/platform/linux/fs.rs 118 native_path libc:: process host_mechanic crates/fbuild-core/src/platform/linux/ipc.rs 1 native_path interprocess::local_socket ipc host_mechanic crates/fbuild-core/src/platform/linux/ipc.rs 2 native_path interprocess::local_socket ipc host_mechanic crates/fbuild-core/src/platform/linux/ipc.rs 3 native_path interprocess::os::unix ipc host_mechanic @@ -27,11 +26,10 @@ crates/fbuild-core/src/platform/linux/process.rs 1 native_path std::os::unix::pr crates/fbuild-core/src/platform/macos/fs.rs 2 native_path std::os::unix::fs::PermissionsExt fs host_mechanic crates/fbuild-core/src/platform/macos/fs.rs 13 native_path std::os::unix::fs::MetadataExt fs host_mechanic crates/fbuild-core/src/platform/macos/fs.rs 57 native_path std::os::unix::fs::symlink fs host_mechanic -crates/fbuild-core/src/platform/macos/fs.rs 61 native_path std::os::unix::fs::symlink fs host_mechanic -crates/fbuild-core/src/platform/macos/fs.rs 105 native_path std::os::unix::ffi::OsStrExt process host_mechanic -crates/fbuild-core/src/platform/macos/fs.rs 110 native_path libc:: process host_mechanic -crates/fbuild-core/src/platform/macos/fs.rs 112 native_path libc:: process host_mechanic -crates/fbuild-core/src/platform/macos/fs.rs 122 native_path libc:: process host_mechanic +crates/fbuild-core/src/platform/macos/fs.rs 101 native_path std::os::unix::ffi::OsStrExt process host_mechanic +crates/fbuild-core/src/platform/macos/fs.rs 106 native_path libc:: process host_mechanic +crates/fbuild-core/src/platform/macos/fs.rs 108 native_path libc:: process host_mechanic +crates/fbuild-core/src/platform/macos/fs.rs 118 native_path libc:: process host_mechanic crates/fbuild-core/src/platform/macos/ipc.rs 1 native_path interprocess::local_socket ipc host_mechanic crates/fbuild-core/src/platform/macos/ipc.rs 2 native_path interprocess::local_socket ipc host_mechanic crates/fbuild-core/src/platform/macos/ipc.rs 3 native_path socket2:: ipc host_mechanic @@ -48,10 +46,9 @@ crates/fbuild-core/src/platform/windows/fs.rs 2 native_path std::os::windows::ff crates/fbuild-core/src/platform/windows/fs.rs 3 native_path std::os::windows::fs fs host_mechanic crates/fbuild-core/src/platform/windows/fs.rs 4 native_path std::os::windows::io::AsRawHandle process host_mechanic crates/fbuild-core/src/platform/windows/fs.rs 60 native_path std::os::windows::fs::symlink_dir fs host_mechanic -crates/fbuild-core/src/platform/windows/fs.rs 64 native_path std::os::windows::fs::symlink_file fs host_mechanic -crates/fbuild-core/src/platform/windows/fs.rs 92 native_path windows_sys:: process host_mechanic -crates/fbuild-core/src/platform/windows/fs.rs 147 native_path windows_sys:: process host_mechanic -crates/fbuild-core/src/platform/windows/fs.rs 191 native_path windows_sys:: process host_mechanic +crates/fbuild-core/src/platform/windows/fs.rs 88 native_path windows_sys:: process host_mechanic +crates/fbuild-core/src/platform/windows/fs.rs 143 native_path windows_sys:: process host_mechanic +crates/fbuild-core/src/platform/windows/fs.rs 187 native_path windows_sys:: process host_mechanic crates/fbuild-core/src/platform/windows/ipc.rs 1 native_path interprocess::local_socket ipc host_mechanic crates/fbuild-core/src/platform/windows/ipc.rs 2 native_path interprocess::local_socket ipc host_mechanic crates/fbuild-core/src/platform/windows/ipc.rs 3 native_path interprocess::os::windows ipc host_mechanic diff --git a/crates/fbuild-build-arm/Cargo.toml b/crates/fbuild-build-arm/Cargo.toml index 80dbd052..4a39f9d7 100644 --- a/crates/fbuild-build-arm/Cargo.toml +++ b/crates/fbuild-build-arm/Cargo.toml @@ -34,7 +34,7 @@ tar = { workspace = true } zip = { workspace = true } tree-sitter = { workspace = true } tree-sitter-cpp = { workspace = true } -zccache = { git = "https://github.com/zackees/zccache", rev = "b385a7bfb308c2d1456613622ad361eb66d31cdd" } +zccache = { git = "https://github.com/zackees/zccache", rev = "3f9d4e66fe3675cd48060bca03b62b1f5a8112eb" } [dev-dependencies] filetime = { workspace = true } diff --git a/crates/fbuild-build-arm/src/apollo3/orchestrator.rs b/crates/fbuild-build-arm/src/apollo3/orchestrator.rs index ba7f3b9f..6c9f83f9 100644 --- a/crates/fbuild-build-arm/src/apollo3/orchestrator.rs +++ b/crates/fbuild-build-arm/src/apollo3/orchestrator.rs @@ -326,7 +326,7 @@ impl BuildOrchestrator for Apollo3Orchestrator { .config .get_lib_ignore(¶ms.env_name) .unwrap_or_default(); - let lib_deps_plan = if !lib_deps.is_empty() { + let lib_archives = if !lib_deps.is_empty() { let temp_compiler = ArmCompiler::new( toolchain.get_gcc_path(), toolchain.get_gxx_path(), @@ -351,11 +351,12 @@ impl BuildOrchestrator for Apollo3Orchestrator { &crate::compiler::Compiler::cpp_flags(&temp_compiler), &mut include_dirs, params.verbose, + crate::parallel::effective_jobs(params.jobs), None, ) .await? } else { - pipeline::LibDeps::default() + Vec::new() }; let compiler = ArmCompiler::new( @@ -447,8 +448,7 @@ impl BuildOrchestrator for Apollo3Orchestrator { ctx, params, &sources, - &[], - lib_deps_plan, + &lib_archives, Some(&lib_env), TargetArchitecture::Arm, "APOLLO3", diff --git a/crates/fbuild-build-arm/src/nrf52/orchestrator.rs b/crates/fbuild-build-arm/src/nrf52/orchestrator.rs index 29427421..76b70c4d 100644 --- a/crates/fbuild-build-arm/src/nrf52/orchestrator.rs +++ b/crates/fbuild-build-arm/src/nrf52/orchestrator.rs @@ -406,7 +406,7 @@ impl BuildOrchestrator for Nrf52Orchestrator { .config .get_lib_ignore(¶ms.env_name) .unwrap_or_default(); - let lib_deps_plan = if !lib_deps.is_empty() { + let lib_archives = if !lib_deps.is_empty() { let temp_compiler = Nrf52Compiler::new( toolchain.get_gcc_path(), toolchain.get_gxx_path(), @@ -431,11 +431,12 @@ impl BuildOrchestrator for Nrf52Orchestrator { &crate::compiler::Compiler::cpp_flags(&temp_compiler), &mut include_dirs, params.verbose, + crate::parallel::effective_jobs(params.jobs), compiler_cache.as_deref(), ) .await? } else { - pipeline::LibDeps::default() + Vec::new() }; let compiler = Nrf52Compiler::new( @@ -500,8 +501,7 @@ impl BuildOrchestrator for Nrf52Orchestrator { ctx, params, &sources, - &[], - lib_deps_plan, + &lib_archives, Some(&lib_env), TargetArchitecture::Arm, "NRF52", diff --git a/crates/fbuild-build-arm/src/nxplpc/orchestrator.rs b/crates/fbuild-build-arm/src/nxplpc/orchestrator.rs index f552b149..579d5ac3 100644 --- a/crates/fbuild-build-arm/src/nxplpc/orchestrator.rs +++ b/crates/fbuild-build-arm/src/nxplpc/orchestrator.rs @@ -376,7 +376,7 @@ impl BuildOrchestrator for NxpLpcOrchestrator { .config .get_lib_ignore(¶ms.env_name) .unwrap_or_default(); - let lib_deps_plan = if !lib_deps.is_empty() { + let lib_archives = if !lib_deps.is_empty() { let temp_compiler = ArmCompiler::new( toolchain.get_gcc_path(), toolchain.get_gxx_path(), @@ -401,11 +401,12 @@ impl BuildOrchestrator for NxpLpcOrchestrator { &crate::compiler::Compiler::cpp_flags(&temp_compiler), &mut include_dirs, params.verbose, + crate::parallel::effective_jobs(params.jobs), None, ) .await? } else { - pipeline::LibDeps::default() + Vec::new() }; let compiler = ArmCompiler::new( @@ -473,9 +474,10 @@ impl BuildOrchestrator for NxpLpcOrchestrator { jobs: crate::parallel::effective_jobs(params.jobs), compiler_cache: None, }; - let extra_link_inputs = + let mut extra_link_inputs = pipeline::compile_extra_libraries(&extra_library_roots, &ctx.build_dir, &lib_env) .await?; + extra_link_inputs.extend(lib_archives); // 11. Run the shared sequential build pipeline. let result = pipeline::run_sequential_build_with_libs( @@ -485,7 +487,6 @@ impl BuildOrchestrator for NxpLpcOrchestrator { params, &sources, &extra_link_inputs, - lib_deps_plan, Some(&lib_env), TargetArchitecture::Arm, "NXPLPC", diff --git a/crates/fbuild-build-arm/src/renesas/orchestrator.rs b/crates/fbuild-build-arm/src/renesas/orchestrator.rs index 68916b10..c1e52083 100644 --- a/crates/fbuild-build-arm/src/renesas/orchestrator.rs +++ b/crates/fbuild-build-arm/src/renesas/orchestrator.rs @@ -260,7 +260,7 @@ impl BuildOrchestrator for RenesasOrchestrator { .config .get_lib_ignore(¶ms.env_name) .unwrap_or_default(); - let lib_deps_plan = if !lib_deps.is_empty() { + let lib_archives = if !lib_deps.is_empty() { let temp_compiler = RenesasCompiler::new( toolchain.get_gcc_path(), toolchain.get_gxx_path(), @@ -285,11 +285,12 @@ impl BuildOrchestrator for RenesasOrchestrator { &crate::compiler::Compiler::cpp_flags(&temp_compiler), &mut include_dirs, params.verbose, + crate::parallel::effective_jobs(params.jobs), compiler_cache.as_deref(), ) .await? } else { - pipeline::LibDeps::default() + Vec::new() }; let compiler = RenesasCompiler::new( @@ -356,8 +357,7 @@ impl BuildOrchestrator for RenesasOrchestrator { ctx, params, &sources, - &[], - lib_deps_plan, + &lib_archives, Some(&lib_env), TargetArchitecture::Arm, "Renesas RA", diff --git a/crates/fbuild-build-arm/src/rp2040/orchestrator.rs b/crates/fbuild-build-arm/src/rp2040/orchestrator.rs index 8d3ead85..9fcabc38 100644 --- a/crates/fbuild-build-arm/src/rp2040/orchestrator.rs +++ b/crates/fbuild-build-arm/src/rp2040/orchestrator.rs @@ -407,7 +407,7 @@ impl BuildOrchestrator for Rp2040Orchestrator { .unwrap_or_default(); let external_lib_deps = fbuild_library_select::external_declared_deps(&lib_deps, &framework_libs); - let lib_deps_plan = if !external_lib_deps.is_empty() { + let lib_archives = if !external_lib_deps.is_empty() { let temp_compiler = ArmCompiler::new( toolchain.get_gcc_path(), toolchain.get_gxx_path(), @@ -432,11 +432,12 @@ impl BuildOrchestrator for Rp2040Orchestrator { &temp_compiler.cpp_flags(), &mut include_dirs, params.verbose, + crate::parallel::effective_jobs(params.jobs), compiler_cache.as_deref(), ) .await? } else { - pipeline::LibDeps::default() + Vec::new() }; let compiler = ArmCompiler::new( @@ -507,6 +508,7 @@ impl BuildOrchestrator for Rp2040Orchestrator { let mut support_link_inputs = rp_support_objects(&framework_dir, &ctx.board.mcu, &board_props); support_link_inputs.push(boot2_object); + support_link_inputs.extend(lib_archives); // 9. Run shared sequential build pipeline let board_mcu = ctx.board.mcu.clone(); @@ -517,7 +519,6 @@ impl BuildOrchestrator for Rp2040Orchestrator { params, &sources, &support_link_inputs, - lib_deps_plan, Some(&lib_env), TargetArchitecture::Arm, "RP2040", diff --git a/crates/fbuild-build-arm/src/sam/orchestrator.rs b/crates/fbuild-build-arm/src/sam/orchestrator.rs index dcd57db7..9210a220 100644 --- a/crates/fbuild-build-arm/src/sam/orchestrator.rs +++ b/crates/fbuild-build-arm/src/sam/orchestrator.rs @@ -493,7 +493,7 @@ impl BuildOrchestrator for SamOrchestrator { .config .get_lib_ignore(¶ms.env_name) .unwrap_or_default(); - let lib_deps_plan = if !lib_deps.is_empty() { + let lib_archives = if !lib_deps.is_empty() { let temp_compiler = SamCompiler::new( toolchain.get_gcc_path(), toolchain.get_gxx_path(), @@ -518,11 +518,12 @@ impl BuildOrchestrator for SamOrchestrator { &crate::compiler::Compiler::cpp_flags(&temp_compiler), &mut include_dirs, params.verbose, + crate::parallel::effective_jobs(params.jobs), compiler_cache.as_deref(), ) .await? } else { - pipeline::LibDeps::default() + Vec::new() }; let compiler = SamCompiler::new( @@ -582,8 +583,7 @@ impl BuildOrchestrator for SamOrchestrator { ctx, params, &sources, - &[], - lib_deps_plan, + &lib_archives, Some(&lib_env), TargetArchitecture::Arm, "SAM", diff --git a/crates/fbuild-build-arm/src/silabs/orchestrator.rs b/crates/fbuild-build-arm/src/silabs/orchestrator.rs index ffb6bdff..54c71478 100644 --- a/crates/fbuild-build-arm/src/silabs/orchestrator.rs +++ b/crates/fbuild-build-arm/src/silabs/orchestrator.rs @@ -236,7 +236,7 @@ impl BuildOrchestrator for SilabsOrchestrator { .config .get_lib_ignore(¶ms.env_name) .unwrap_or_default(); - let lib_deps_plan = if !lib_deps.is_empty() { + let lib_archives = if !lib_deps.is_empty() { let temp_compiler = SilabsCompiler::new( toolchain.get_gcc_path(), toolchain.get_gxx_path(), @@ -261,11 +261,12 @@ impl BuildOrchestrator for SilabsOrchestrator { &crate::compiler::Compiler::cpp_flags(&temp_compiler), &mut include_dirs, params.verbose, + crate::parallel::effective_jobs(params.jobs), None, ) .await? } else { - pipeline::LibDeps::default() + Vec::new() }; let compiler = SilabsCompiler::new( @@ -328,8 +329,7 @@ impl BuildOrchestrator for SilabsOrchestrator { ctx, params, &sources, - &[], - lib_deps_plan, + &lib_archives, Some(&lib_env), TargetArchitecture::Arm, "Silicon Labs", diff --git a/crates/fbuild-build-arm/src/stm32/orchestrator/arduino_mbed.rs b/crates/fbuild-build-arm/src/stm32/orchestrator/arduino_mbed.rs index a359814f..423e27da 100644 --- a/crates/fbuild-build-arm/src/stm32/orchestrator/arduino_mbed.rs +++ b/crates/fbuild-build-arm/src/stm32/orchestrator/arduino_mbed.rs @@ -134,7 +134,7 @@ pub(super) async fn build_arduino_mbed_stm32( .config .get_lib_ignore(¶ms.env_name) .unwrap_or_default(); - let lib_deps_plan = if !lib_deps.is_empty() { + let lib_archives = if !lib_deps.is_empty() { let temp_compiler = ArmCompiler::new( toolchain.get_gcc_path(), toolchain.get_gxx_path(), @@ -159,11 +159,12 @@ pub(super) async fn build_arduino_mbed_stm32( &crate::compiler::Compiler::cpp_flags(&temp_compiler), &mut include_dirs, params.verbose, + crate::parallel::effective_jobs(params.jobs), None, ) .await? } else { - pipeline::LibDeps::default() + Vec::new() }; let compiler = ArmCompiler::new( @@ -218,8 +219,7 @@ pub(super) async fn build_arduino_mbed_stm32( ctx, params, &sources, - &[], - lib_deps_plan, + &lib_archives, Some(&lib_env), TargetArchitecture::Arm, "STM32", diff --git a/crates/fbuild-build-arm/src/stm32/orchestrator/mod.rs b/crates/fbuild-build-arm/src/stm32/orchestrator/mod.rs index 7b329b62..bd649955 100644 --- a/crates/fbuild-build-arm/src/stm32/orchestrator/mod.rs +++ b/crates/fbuild-build-arm/src/stm32/orchestrator/mod.rs @@ -487,7 +487,7 @@ impl BuildOrchestrator for Stm32Orchestrator { .config .get_lib_ignore(¶ms.env_name) .unwrap_or_default(); - let lib_deps_plan = if !lib_deps.is_empty() { + let lib_archives = if !lib_deps.is_empty() { let temp_compiler = ArmCompiler::new( toolchain.get_gcc_path(), toolchain.get_gxx_path(), @@ -512,11 +512,12 @@ impl BuildOrchestrator for Stm32Orchestrator { &crate::compiler::Compiler::cpp_flags(&temp_compiler), &mut include_dirs, params.verbose, + crate::parallel::effective_jobs(params.jobs), None, ) .await? } else { - pipeline::LibDeps::default() + Vec::new() }; let compiler = ArmCompiler::new( @@ -579,8 +580,7 @@ impl BuildOrchestrator for Stm32Orchestrator { ctx, params, &sources, - &[], - lib_deps_plan, + &lib_archives, Some(&lib_env), TargetArchitecture::Arm, "STM32", diff --git a/crates/fbuild-build-arm/src/teensy/orchestrator.rs b/crates/fbuild-build-arm/src/teensy/orchestrator.rs index 72a17db8..5d96e5a6 100644 --- a/crates/fbuild-build-arm/src/teensy/orchestrator.rs +++ b/crates/fbuild-build-arm/src/teensy/orchestrator.rs @@ -318,7 +318,7 @@ impl BuildOrchestrator for TeensyOrchestrator { .config .get_lib_ignore(¶ms.env_name) .unwrap_or_default(); - let lib_deps_plan = if !lib_deps.is_empty() { + let lib_archives = if !lib_deps.is_empty() { let temp_compiler = TeensyCompiler::new( toolchain.get_gcc_path(), toolchain.get_gxx_path(), @@ -343,11 +343,12 @@ impl BuildOrchestrator for TeensyOrchestrator { &temp_compiler.cpp_flags(), &mut include_dirs, params.verbose, + crate::parallel::effective_jobs(params.jobs), compiler_cache.as_deref(), ) .await? } else { - pipeline::LibDeps::default() + Vec::new() }; let compiler = TeensyCompiler::new( @@ -418,8 +419,7 @@ impl BuildOrchestrator for TeensyOrchestrator { ctx, params, &sources, - &[], - lib_deps_plan, + &lib_archives, Some(&lib_env), TargetArchitecture::Arm, "Teensy", diff --git a/crates/fbuild-build-engine/Cargo.toml b/crates/fbuild-build-engine/Cargo.toml index 376f2035..c38a570e 100644 --- a/crates/fbuild-build-engine/Cargo.toml +++ b/crates/fbuild-build-engine/Cargo.toml @@ -34,7 +34,7 @@ tar = { workspace = true } zip = { workspace = true } tree-sitter = { workspace = true } tree-sitter-cpp = { workspace = true } -zccache = { git = "https://github.com/zackees/zccache", rev = "b385a7bfb308c2d1456613622ad361eb66d31cdd" } +zccache = { git = "https://github.com/zackees/zccache", rev = "3f9d4e66fe3675cd48060bca03b62b1f5a8112eb" } [dev-dependencies] filetime = { workspace = true } diff --git a/crates/fbuild-build-engine/README.md b/crates/fbuild-build-engine/README.md index 67a0bfa1..980ec22e 100644 --- a/crates/fbuild-build-engine/README.md +++ b/crates/fbuild-build-engine/README.md @@ -6,7 +6,6 @@ parallelism (FastLED/fbuild#1008). Holds every shared engine module — `pipeline`, `compiler`, `compile_many`, `source_scanner`, `linker`, `build_fingerprint`, `compile_database`, `symbol_analyzer`, `shrink`, `framework_libs`, `framework_core_cache`, -`include_farm`, `script_runtime`, `flag_overlay`, `build_info`, `build_output`, `eh_frame_policy`, `zccache`/`zccache_embedded`, `arduino_props`, `compile_backend`, `parallel`, `perf_log`, `package_override`, `resolution`, diff --git a/crates/fbuild-build-engine/src/include_farm.rs b/crates/fbuild-build-engine/src/include_farm.rs deleted file mode 100644 index f8163b4e..00000000 --- a/crates/fbuild-build-engine/src/include_farm.rs +++ /dev/null @@ -1,552 +0,0 @@ -//! Collapse a framework's block of `-I` directories into one merged directory. -//! -//! GCC resolves every `#include` by probing each `-I` directory in order, so a -//! toolchain header such as `` first fails one `open` per framework -//! directory. On ESP32-S3 that is ~31k failed opens per translation unit and -//! about a third of all compile CPU (FastLED/fbuild#1537). A header farm is a -//! single directory whose entries symlink into the original directories, so a -//! lookup costs one probe instead of ~200. -//! -//! The farm resolves every include to the same file the original list did: -//! -//! * Only header paths held by exactly one directory are farmed. A directory -//! holding a duplicated path keeps its own `-I`, in its original order, right -//! after the farm, so first-match-wins and `#include_next` are unchanged. -//! * A subtree owned by one directory is a single directory symlink, so quoted -//! and `..`-relative includes inside it resolve physically, exactly as before. -//! Only directories that several sources contribute to are real. A header in -//! one of those whose quoted include would resolve differently there keeps -//! its directory out of the farm. -//! -//! Anything unexpected yields no farm; callers then keep the original list. -//! Callers skip farms on Windows, where symlinks usually need elevated rights. - -use std::collections::{BTreeMap, HashMap}; -use std::io; -use std::path::{Component, Path}; - -use fbuild_core::path::NormalizedPath; -use serde::{Deserialize, Serialize}; - -/// Bumped whenever the farm layout or its planning rules change. -const FORMAT_VERSION: u32 = 2; -const MANIFEST: &str = "farm.json"; - -/// A materialized farm plus the block directories that stay separate. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct IncludeFarm { - /// The merged directory that replaces the farmed block entries. - pub dir: NormalizedPath, - /// Block directories kept as their own `-I`, in original order, after `dir`. - pub kept: Vec, -} - -impl IncludeFarm { - /// The `-I` directories that replace the block, in search order. - pub fn replacement(&self) -> Vec { - std::iter::once(self.dir.clone()) - .chain(self.kept.iter().cloned()) - .collect() - } - - /// Maps the farm's prefix out of `__FILE__`, so headers reached through it - /// do not embed the cache path in flash. - pub fn macro_prefix_map(&self, label: &str) -> String { - format!("-fmacro-prefix-map={}={label}", self.dir.display()) - } -} - -#[derive(Serialize, Deserialize)] -struct Manifest { - version: u32, - before: Vec, - block: Vec, - kept: Vec, -} - -/// Return the farm for `block`, creating it under `farms_root` on first use. -/// -/// `before` lists the `-I` directories that precede the block; they can shadow -/// a header a farmed file would otherwise find beside itself, so they take part -/// in the equivalence check and in the farm's key. The farm directory is a pure -/// function of both lists, which keeps depfiles and cache keys that name it -/// valid when it is deleted and rebuilt. -pub fn ensure_farm( - farms_root: &Path, - before: &[NormalizedPath], - block: &[NormalizedPath], -) -> io::Result { - let before_s = path_strings(before)?; - let block_s = path_strings(block)?; - let dir = NormalizedPath::new(farms_root.join(farm_key(&before_s, &block_s))); - if let Some(farm) = read_manifest(dir.as_path(), &before_s, &block_s) { - return Ok(farm); - } - let plan = plan_farm(before, block)?; - std::fs::create_dir_all(farms_root)?; - let nanos = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |d| d.as_nanos()); - let staging = NormalizedPath::new(farms_root.join(format!( - "tmp-{}-{}-{nanos}", - dir.file_name().map_or_else(String::new, |n| n.to_string_lossy().into_owned()), - std::process::id() - ))); - let built = materialize(&plan, staging.as_path()).and_then(|()| { - let manifest = Manifest { - version: FORMAT_VERSION, - before: before_s.clone(), - block: block_s.clone(), - kept: plan.kept.iter().map(|&i| block_s[i].clone()).collect(), - }; - std::fs::write( - staging.join(MANIFEST).as_path(), - serde_json::to_vec(&manifest).map_err(io::Error::other)?, - ) - }); - if let Err(error) = built { - let _ = std::fs::remove_dir_all(staging.as_path()); - return Err(error); - } - if std::fs::rename(staging.as_path(), dir.as_path()).is_err() { - // Another build published the same farm first; theirs is identical. - let _ = std::fs::remove_dir_all(staging.as_path()); - } - read_manifest(dir.as_path(), &before_s, &block_s) - .ok_or_else(|| io::Error::other(format!("include farm {} is unreadable", dir.display()))) -} - -fn path_strings(dirs: &[NormalizedPath]) -> io::Result> { - dirs.iter() - .map(|d| { - d.to_str().map(str::to_string).ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidData, "non-UTF-8 include directory") - }) - }) - .collect() -} - -fn farm_key(before: &[String], block: &[String]) -> String { - let mut hasher = blake3::Hasher::new(); - hasher.update(format!("include-farm-v{FORMAT_VERSION}\n").as_bytes()); - for dir in before { - hasher.update(dir.as_bytes()); - hasher.update(b"\n"); - } - hasher.update(b"--\n"); - for dir in block { - hasher.update(dir.as_bytes()); - hasher.update(b"\n"); - } - hasher.finalize().to_hex()[..32].to_string() -} - -fn read_manifest(dir: &Path, before: &[String], block: &[String]) -> Option { - let bytes = std::fs::read(dir.join(MANIFEST)).ok()?; - let manifest: Manifest = serde_json::from_slice(&bytes).ok()?; - (manifest.version == FORMAT_VERSION && manifest.before == before && manifest.block == block) - .then(|| IncludeFarm { - dir: NormalizedPath::new(dir), - kept: manifest.kept.iter().map(NormalizedPath::from).collect(), - }) -} - -/// What to build: symlinks (farm-relative path, target) and the block indices -/// that stay as their own `-I`. -#[derive(Debug)] -struct Plan { - links: Vec<(String, NormalizedPath)>, - real_dirs: Vec, - kept: Vec, -} - -#[derive(Default, Debug)] -struct Node { - children: BTreeMap, - /// Owning block index when this node is a file. - file: Option, - /// The single block index every file below this node comes from, if any. - owner: Option, -} - -impl Node { - fn sole_owner(&self) -> Option { - self.owner - } - - fn annotate(&mut self) -> Option { - self.owner = match self.file { - Some(owner) => Some(owner), - None => { - let mut owners = self.children.values_mut().map(Node::annotate); - let first = owners.next().flatten(); - // Drain the iterator so every child is annotated. - let all_same = owners.fold(first.is_some(), |same, o| same && o == first); - first.filter(|_| all_same) - } - }; - self.owner - } -} - -/// Every block and `before` directory's files, for lookups without stats. -struct Index<'a> { - before: &'a [NormalizedPath], - before_files: Vec>, - block: &'a [NormalizedPath], - block_files: Vec>, -} - -impl Index<'_> { - /// `dir/rel`, as a path string, if that file exists. - fn file_in(&self, dir: &Path, has: impl Fn(&str) -> bool, rel: &str) -> Option { - match normalize_rel(rel) { - Some(rel) => has(&rel).then(|| format!("{}/{rel}", dir.display())), - // Escapes the directory: rare, so ask the filesystem. - None => lexical_join(dir, rel).filter(|p| Path::new(p).is_file()), - } - } - - fn in_block(&self, index: usize, rel: &str) -> Option { - self.file_in( - self.block[index].as_path(), - |r| self.block_files[index].contains(r), - rel, - ) - } - - fn in_before(&self, index: usize, rel: &str) -> Option { - self.file_in( - self.before[index].as_path(), - |r| self.before_files[index].contains(r), - rel, - ) - } -} - -/// `rel` with `.`/`..` applied, or `None` if it climbs out of its root. -fn normalize_rel(rel: &str) -> Option { - let mut parts: Vec<&str> = Vec::new(); - for part in rel.split('/') { - match part { - "" | "." => {} - ".." => { - parts.pop()?; - } - part => parts.push(part), - } - } - Some(parts.join("/")) -} - -fn plan_farm(before: &[NormalizedPath], block: &[NormalizedPath]) -> io::Result { - let files: Vec> = block - .iter() - .map(|dir| list_files(dir.as_path())) - .collect::>()?; - let mut farmed = vec![true; block.len()]; - let mut owners: HashMap<&str, Vec> = HashMap::new(); - for (index, rels) in files.iter().enumerate() { - for rel in rels { - owners.entry(rel.as_str()).or_default().push(index); - } - } - for dirs in owners.values().filter(|dirs| dirs.len() > 1) { - for &index in dirs { - farmed[index] = false; - } - } - // `#include_next` continues after the directory a header was found in, - // but GCC restarts it when the header was found beside a quoted includer. - // In a merged farm directory that can bring the header back to itself - // (and `#pragma once` then drops the next one), so a directory holding any - // `#include_next` header keeps its own `-I` (FastLED/fbuild#1566). - for (index, rels) in files.iter().enumerate() { - if farmed[index] - && rels - .iter() - .any(|rel| uses_include_next(&block[index].join(rel))) - { - farmed[index] = false; - } - } - // A path that is a file in one directory and a directory in another cannot - // be merged; keep both directories out. - for rel in owners.keys() { - let mut prefix = String::new(); - for part in rel.split('/').take(rel.split('/').count() - 1) { - if !prefix.is_empty() { - prefix.push('/'); - } - prefix.push_str(part); - if let Some(file_owners) = owners.get(prefix.as_str()) { - for &index in file_owners.iter().chain(&owners[rel]) { - farmed[index] = false; - } - } - } - } - let index = Index { - before, - before_files: before - .iter() - .map(|dir| list_files(dir.as_path()).map(|rels| rels.into_iter().collect())) - .collect::>()?, - block, - block_files: files - .iter() - .map(|rels| rels.iter().map(String::as_str).collect()) - .collect(), - }; - loop { - let mut root = build_tree(&files, &farmed); - root.annotate(); - let conflicts = quoted_include_conflicts(&root, &index, &files, &farmed); - if conflicts.is_empty() { - let mut plan = Plan { - links: Vec::new(), - real_dirs: Vec::new(), - kept: (0..block.len()).filter(|&i| !farmed[i]).collect(), - }; - collect_links(&root, "", block, &mut plan); - return Ok(plan); - } - for index in conflicts { - farmed[index] = false; - } - } -} - -fn list_files(dir: &Path) -> io::Result> { - let mut rels = Vec::new(); - // GCC ignores a missing `-I`; it contributes nothing to the farm. - if dir.is_dir() { - collect_files(dir, "", 0, &mut rels)?; - } - Ok(rels) -} - -/// Relative (`/`-joined) paths of the regular files under `dir`, following -/// symlinks as `open` does. Dangling links are skipped, as GCC skips them. -fn collect_files(dir: &Path, prefix: &str, depth: usize, out: &mut Vec) -> io::Result<()> { - if depth > 64 { - return Err(io::Error::other(format!( - "include tree too deep (symlink loop?) at {}", - dir.display() - ))); - } - for entry in std::fs::read_dir(dir)? { - let entry = entry?; - let name = entry - .file_name() - .into_string() - .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "non-UTF-8 header path"))?; - let rel = if prefix.is_empty() { - name - } else { - format!("{prefix}/{name}") - }; - let Ok(meta) = std::fs::metadata(entry.path()) else { - continue; - }; - if meta.is_dir() { - collect_files(&entry.path(), &rel, depth + 1, out)?; - } else if meta.is_file() { - out.push(rel); - } - } - Ok(()) -} - -fn build_tree(files: &[Vec], farmed: &[bool]) -> Node { - let mut root = Node::default(); - for (index, rels) in files.iter().enumerate() { - if !farmed[index] { - continue; - } - for rel in rels { - let mut node = &mut root; - for part in rel.split('/') { - node = node.children.entry(part.to_string()).or_default(); - } - node.file = Some(index); - } - } - root -} - -fn collect_links(node: &Node, rel: &str, block: &[NormalizedPath], plan: &mut Plan) { - for (name, child) in &node.children { - let child_rel = if rel.is_empty() { - name.clone() - } else { - format!("{rel}/{name}") - }; - match child.sole_owner() { - Some(owner) => plan - .links - .push((child_rel.clone(), block[owner].join(&child_rel))), - None => { - plan.real_dirs.push(child_rel.clone()); - collect_links(child, &child_rel, block, plan); - } - } - } -} - -/// Block indices whose farmed headers would resolve a quoted include to a -/// different file from inside a merged farm directory. -fn quoted_include_conflicts( - root: &Node, - index: &Index<'_>, - files: &[Vec], - farmed: &[bool], -) -> Vec { - let block = index.block; - let mut conflicts = Vec::new(); - for (owner, rels) in files.iter().enumerate() { - if !farmed[owner] { - continue; - } - let conflicting = rels.iter().any(|rel| { - let parent = rel.rsplit_once('/').map_or("", |(p, _)| p); - // Inside a symlinked subtree, lookups resolve physically in the - // owning directory, exactly as before. - if !in_merged_dir(root, parent) { - return false; - } - quoted_includes(&block[owner].join(rel)).iter().any(|name| { - // A miss beside the file falls through to the same `-I` chain - // both ways; the farm cannot change the chain's answer - // because farmed paths are unique. - let chain = || chain_lookup(index, name); - let beside = format!("{parent}/{name}"); - let original = index.in_block(owner, &beside).or_else(chain); - let farm = beside_farm(root, index, &beside).or_else(chain); - original != farm - }) - }); - if conflicting { - conflicts.push(owner); - } - } - conflicts -} - -/// Whether the farm directory at `rel` is real (merged) rather than a symlink. -fn in_merged_dir(root: &Node, rel: &str) -> bool { - let mut node = root; - if rel.is_empty() { - return true; - } - for part in rel.split('/') { - match node.children.get(part) { - Some(child) if child.sole_owner().is_none() => node = child, - _ => return false, - } - } - true -} - -fn uses_include_next(file: &Path) -> bool { - std::fs::read(file).is_ok_and(|bytes| { - String::from_utf8_lossy(&bytes).lines().any(|line| { - line.trim_start() - .strip_prefix('#') - .is_some_and(|rest| rest.trim_start().starts_with("include_next")) - }) - }) -} - -fn quoted_includes(file: &Path) -> Vec { - let Ok(bytes) = std::fs::read(file) else { - return Vec::new(); - }; - let text = String::from_utf8_lossy(&bytes); - text.lines() - .filter_map(|line| { - let rest = line.trim_start().strip_prefix('#')?.trim_start(); - let rest = rest.strip_prefix("include")?.trim_start(); - let rest = rest.strip_prefix('"')?; - let (name, _) = rest.split_once('"')?; - (!name.is_empty() && !Path::new(name).is_absolute()).then(|| name.to_string()) - }) - .collect() -} - -/// The first directory before or inside the block that holds `name`. -fn chain_lookup(index: &Index<'_>, name: &str) -> Option { - (0..index.before.len()) - .find_map(|i| index.in_before(i, name)) - .or_else(|| (0..index.block.len()).find_map(|i| index.in_block(i, name))) -} - -/// `beside` (the including file's directory joined with the include name) -/// looked up in the farm, where that directory is merged. -fn beside_farm(root: &Node, index: &Index<'_>, beside: &str) -> Option { - let mut stack: Vec<&Node> = vec![root]; - let mut rel: Vec<&str> = Vec::new(); - let parts: Vec<&str> = beside - .split('/') - .filter(|p| !p.is_empty() && *p != ".") - .collect(); - for (i, part) in parts.iter().enumerate() { - let node = *stack.last()?; - if *part == ".." { - stack.pop(); - rel.pop(); - if stack.is_empty() { - // Escapes the farm root into the cache directory. - return None; - } - continue; - } - let child = node.children.get(*part)?; - rel.push(part); - if let Some(owner) = child.sole_owner() { - // A symlink: the remainder resolves physically in the owner. - let physical = format!("{}/{}", rel.join("/"), parts[i + 1..].join("/")); - return index.in_block(owner, &physical); - } - stack.push(child); - } - None -} - -/// `base` joined with `rel`, `..` applied lexically; `None` if not UTF-8. -fn lexical_join(base: &Path, rel: &str) -> Option { - let mut out: Vec = Vec::new(); - let mut prefix = String::new(); - for component in base.join(rel).components() { - match component { - Component::Prefix(p) => prefix = p.as_os_str().to_str()?.to_string(), - Component::RootDir => prefix.push('/'), - Component::CurDir => {} - Component::ParentDir => { - out.pop(); - } - Component::Normal(part) => out.push(part.to_str()?.to_string()), - } - } - Some(format!("{prefix}{}", out.join("/"))) -} - -fn materialize(plan: &Plan, staging: &Path) -> io::Result<()> { - std::fs::create_dir_all(staging)?; - for dir in &plan.real_dirs { - std::fs::create_dir_all(staging.join(dir))?; - } - for (rel, target) in &plan.links { - let link = staging.join(rel); - if target.as_path().is_dir() { - fbuild_core::platform::fs::symlink_dir(target.as_path(), &link)?; - } else { - fbuild_core::platform::fs::symlink_file(target.as_path(), &link)?; - } - } - Ok(()) -} - -#[cfg(test)] -#[path = "include_farm_tests.rs"] -mod tests; diff --git a/crates/fbuild-build-engine/src/include_farm_tests.rs b/crates/fbuild-build-engine/src/include_farm_tests.rs deleted file mode 100644 index 4b9ceb68..00000000 --- a/crates/fbuild-build-engine/src/include_farm_tests.rs +++ /dev/null @@ -1,248 +0,0 @@ -use super::*; - -fn write(root: &Path, rel: &str, body: &str) { - let path = root.join(rel); - std::fs::create_dir_all(path.parent().unwrap()).unwrap(); - std::fs::write(path, body).unwrap(); -} - -/// Block directories `d0..dN` under a fresh temp root. -fn block_dirs(tmp: &tempfile::TempDir, n: usize) -> Vec { - (0..n) - .map(|i| { - let dir = NormalizedPath::new(tmp.path().join(format!("d{i}"))); - std::fs::create_dir_all(dir.as_path()).unwrap(); - dir - }) - .collect() -} - -fn kept_indices(plan: &Plan) -> Vec { - plan.kept.clone() -} - -#[test] -fn unique_headers_are_all_farmed() { - let tmp = tempfile::TempDir::new().unwrap(); - let block = block_dirs(&tmp, 3); - write(block[0].as_path(), "a.h", ""); - write(block[1].as_path(), "sub/b.h", ""); - write(block[2].as_path(), "c.h", ""); - - let plan = plan_farm(&[], &block).unwrap(); - - assert!(kept_indices(&plan).is_empty()); - let rels: Vec<&str> = plan.links.iter().map(|(rel, _)| rel.as_str()).collect(); - assert_eq!(rels, ["a.h", "c.h", "sub"]); -} - -#[test] -fn duplicated_header_keeps_both_directories_in_order() { - let tmp = tempfile::TempDir::new().unwrap(); - let block = block_dirs(&tmp, 4); - write(block[0].as_path(), "only0.h", ""); - write(block[1].as_path(), "endian.h", ""); - write(block[2].as_path(), "only2.h", ""); - write(block[3].as_path(), "endian.h", ""); - - let plan = plan_farm(&[], &block).unwrap(); - - assert_eq!(kept_indices(&plan), [1, 3]); - let rels: Vec<&str> = plan.links.iter().map(|(rel, _)| rel.as_str()).collect(); - assert_eq!(rels, ["only0.h", "only2.h"]); -} - -#[test] -fn single_owner_subtree_is_one_directory_link() { - let tmp = tempfile::TempDir::new().unwrap(); - let block = block_dirs(&tmp, 2); - write(block[0].as_path(), "freertos/FreeRTOS.h", ""); - write(block[0].as_path(), "freertos/task.h", ""); - write(block[1].as_path(), "hal/gpio.h", ""); - write(block[0].as_path(), "hal/uart.h", ""); - - let plan = plan_farm(&[], &block).unwrap(); - - let links: Vec<(&str, NormalizedPath)> = plan - .links - .iter() - .map(|(rel, target)| (rel.as_str(), target.clone())) - .collect(); - assert!(links.contains(&("freertos", block[0].join("freertos")))); - assert!(links.contains(&("hal/gpio.h", block[1].join("hal/gpio.h")))); - assert!(links.contains(&("hal/uart.h", block[0].join("hal/uart.h")))); - assert_eq!(plan.real_dirs, ["hal"]); -} - -#[test] -fn quoted_include_that_would_change_inside_a_merged_dir_keeps_its_dir() { - let tmp = tempfile::TempDir::new().unwrap(); - let block = block_dirs(&tmp, 3); - // d0/hal/a.h includes "b.h": originally not beside it, so the chain finds - // d2/b.h. In the merged farm `hal/`, d1's hal/b.h would sit beside it. - write(block[0].as_path(), "hal/a.h", "#include \"b.h\"\n"); - write(block[1].as_path(), "hal/b.h", ""); - write(block[2].as_path(), "b.h", ""); - - let plan = plan_farm(&[], &block).unwrap(); - - assert_eq!(kept_indices(&plan), [0]); -} - -#[test] -fn sibling_supplied_by_another_dir_in_a_merged_dir_is_a_conflict() { - let tmp = tempfile::TempDir::new().unwrap(); - let block = block_dirs(&tmp, 2); - write(block[0].as_path(), "hal/a.h", "#include \"a_impl.h\"\n"); - write(block[0].as_path(), "hal/a_impl.h", ""); - write(block[1].as_path(), "hal/b.h", "#include \"a.h\"\n"); - - let plan = plan_farm(&[], &block).unwrap(); - - // d1/hal/b.h's "a.h" is not beside it originally and the chain has no - // top-level a.h, but the merged farm hal/ holds d0's a.h: a conflict. - assert_eq!(kept_indices(&plan), [1]); -} - -#[test] -fn directory_before_the_block_shadowing_a_sibling_include_is_a_conflict() { - let tmp = tempfile::TempDir::new().unwrap(); - let before = vec![NormalizedPath::new(tmp.path().join("core"))]; - write(before[0].as_path(), "config.h", ""); - let block = block_dirs(&tmp, 2); - // Originally "config.h" from d0 resolves to core/config.h (d0 lacks it); - // in the farm root, d1's config.h would sit beside it. - write(block[0].as_path(), "a.h", "#include \"config.h\"\n"); - write(block[1].as_path(), "config.h", ""); - - let plan = plan_farm(&before, &block).unwrap(); - - assert_eq!(kept_indices(&plan), [0]); -} - -#[test] -fn file_in_one_dir_and_directory_in_another_keeps_both() { - let tmp = tempfile::TempDir::new().unwrap(); - let block = block_dirs(&tmp, 3); - write(block[0].as_path(), "port", ""); - write(block[1].as_path(), "port/x.h", ""); - write(block[2].as_path(), "fine.h", ""); - - let plan = plan_farm(&[], &block).unwrap(); - - assert_eq!(kept_indices(&plan), [0, 1]); -} - -#[test] -fn farm_resolves_every_header_to_the_original_file() { - if fbuild_core::platform::host::is_windows() { - return; - } - let tmp = tempfile::TempDir::new().unwrap(); - let block = block_dirs(&tmp, 3); - write(block[0].as_path(), "freertos/task.h", "task"); - write(block[1].as_path(), "hal/gpio.h", "gpio"); - write(block[2].as_path(), "hal/uart.h", "uart"); - write(block[2].as_path(), "top.h", "top"); - - let farm = ensure_farm(&tmp.path().join("farms"), &[], &block).unwrap(); - - for (rel, body) in [ - ("freertos/task.h", "task"), - ("hal/gpio.h", "gpio"), - ("hal/uart.h", "uart"), - ("top.h", "top"), - ] { - assert_eq!(std::fs::read_to_string(farm.dir.join(rel)).unwrap(), body); - } - assert!(farm.kept.is_empty()); - assert!( - farm.dir - .join("freertos") - .symlink_metadata() - .unwrap() - .is_symlink() - ); - assert!( - !farm - .dir - .join("hal") - .symlink_metadata() - .unwrap() - .is_symlink() - ); -} - -#[test] -fn farm_path_is_deterministic_and_rebuilt_in_place() { - if fbuild_core::platform::host::is_windows() { - return; - } - let tmp = tempfile::TempDir::new().unwrap(); - let block = block_dirs(&tmp, 2); - write(block[0].as_path(), "a.h", ""); - write(block[1].as_path(), "a.h", ""); - write(block[1].as_path(), "b.h", ""); - let farms = tmp.path().join("farms"); - - let first = ensure_farm(&farms, &[], &block).unwrap(); - assert_eq!(first.kept, block, "a duplicated a.h keeps both dirs"); - let again = ensure_farm(&farms, &[], &block).unwrap(); - assert_eq!(first, again); - - std::fs::remove_dir_all(first.dir.as_path()).unwrap(); - let rebuilt = ensure_farm(&farms, &[], &block).unwrap(); - assert_eq!(first, rebuilt, "depfiles naming the farm stay valid"); - - let other = ensure_farm(&farms, &[], &block[..1]).unwrap(); - assert_ne!(first.dir, other.dir); -} - -#[test] -fn replacement_puts_the_farm_first_then_kept_dirs() { - let farm = IncludeFarm { - dir: NormalizedPath::from("/farm"), - kept: vec![NormalizedPath::from("/d1"), NormalizedPath::from("/d3")], - }; - assert_eq!( - farm.replacement(), - [ - NormalizedPath::from("/farm"), - NormalizedPath::from("/d1"), - NormalizedPath::from("/d3") - ] - ); - assert_eq!( - farm.macro_prefix_map("fw"), - format!( - "-fmacro-prefix-map={}=fw", - NormalizedPath::from("/farm").display() - ) - ); -} - -#[test] -fn include_next_header_beside_a_quoted_includer_keeps_its_directory() { - let tmp = tempfile::TempDir::new().unwrap(); - let block = block_dirs(&tmp, 3); - // d0/assert.h chains to the toolchain's; d1/esp_assert.h includes it - // quoted. In the merged farm root it would sit beside its includer, and - // GCC restarts `#include_next` for such a header (FastLED/fbuild#1566). - write( - block[0].as_path(), - "assert.h", - "#pragma once\n#include_next \n", - ); - write( - block[1].as_path(), - "esp_assert.h", - "#include \"assert.h\"\n", - ); - write(block[2].as_path(), "other.h", ""); - - let plan = plan_farm(&[], &block).unwrap(); - - assert_eq!(kept_indices(&plan), [0]); - let rels: Vec<&str> = plan.links.iter().map(|(rel, _)| rel.as_str()).collect(); - assert_eq!(rels, ["esp_assert.h", "other.h"]); -} diff --git a/crates/fbuild-build-engine/src/lib.rs b/crates/fbuild-build-engine/src/lib.rs index 64ad6332..cd6a99b3 100644 --- a/crates/fbuild-build-engine/src/lib.rs +++ b/crates/fbuild-build-engine/src/lib.rs @@ -22,7 +22,6 @@ pub mod eh_frame_policy_compute; pub mod flag_overlay; pub mod framework_core_cache; pub mod framework_libs; -pub mod include_farm; pub mod linker; pub mod mcu_config; pub mod package_override; diff --git a/crates/fbuild-build-engine/src/parallel.rs b/crates/fbuild-build-engine/src/parallel.rs index 41bf23c4..3cdd3b37 100644 --- a/crates/fbuild-build-engine/src/parallel.rs +++ b/crates/fbuild-build-engine/src/parallel.rs @@ -66,7 +66,6 @@ pub async fn compile_sources_parallel( extra_flags, &Arc::new(Semaphore::new(permits)), build_log, - None, ) .await } @@ -83,15 +82,9 @@ pub async fn compile_sources_parallel_shared( extra_flags: &LanguageExtraFlags, semaphore: &Arc, build_log: Option<&std::sync::Mutex>, - // `Some` overrides the TU-duration history root (tests only); `None` - // uses the real `fbuild_paths::get_cache_root()` (FastLED/fbuild#1564). - history_root: Option<&Path>, ) -> Result { - let history_root_owned = history_root - .map(Path::to_path_buf) - .unwrap_or_else(fbuild_paths::get_cache_root); - // Build work list: (source, object, signature) triples needing rebuild - let mut work: Vec<(PathBuf, PathBuf, String)> = Vec::new(); + // Build work list: (source, object) pairs needing rebuild + let mut work: Vec<(PathBuf, PathBuf)> = Vec::new(); let mut objects: Vec = Vec::new(); for source in sources { @@ -102,25 +95,11 @@ pub async fn compile_sources_parallel_shared( // match this check (stage-2 seeding, FastLED/fbuild#1346). let signature = compiler.rebuild_signature(source, &source_flags, &obj); if CompilerBase::needs_rebuild_with_signature(source, &obj, Some(&signature)) { - work.push((source.clone(), obj.clone(), signature)); + work.push((source.clone(), obj.clone())); } objects.push(obj); } - // Long poles first, using recorded compile durations where available - // (FastLED/fbuild#1564); `objects` keeps source order, so link order is - // unchanged. - fbuild_packages::library::tu_history::dispatch_order_in( - &history_root_owned, - &mut work, - |(source, _, signature)| { - ( - fbuild_core::path::NormalizedPath::new(source), - signature.clone(), - ) - }, - ); - if work.is_empty() { return Ok(ParallelCompileResult { objects, @@ -157,47 +136,19 @@ pub async fn compile_sources_parallel_shared( let build_log_ptr: Option<&'static std::sync::Mutex> = unsafe { std::mem::transmute(build_log) }; - let failed = Arc::new(std::sync::atomic::AtomicBool::new(false)); - let history_root_arc = Arc::new(history_root_owned); - for (source, obj, signature) in work.into_iter() { - if failed.load(std::sync::atomic::Ordering::Relaxed) { - break; - } - // Take the permit here, in dispatch order: on a multi-thread runtime - // spawned tasks first run in no particular order, so a permit taken - // inside the task would not honor the dispatch order. A closed - // semaphore (shutdown) stops spawning; the drain below still awaits - // every task already spawned, which the `'static` borrows above rely on. - let permit = match semaphore.clone().acquire_owned().await { - Ok(permit) => permit, - Err(e) => { - first_error = Some(format!("semaphore closed: {e}")); - break; - } - }; - // A compile may have failed while this loop waited for the permit. - if failed.load(std::sync::atomic::Ordering::Relaxed) { - break; - } + for (source, obj) in work.into_iter() { + let sem = semaphore.clone(); let counter = compiled_count.clone(); - let failed = failed.clone(); - let history_root_t = history_root_arc.clone(); tasks.spawn(async move { - let _permit = permit; - let started = std::time::Instant::now(); + // Acquire permit; if Acquired returns Err, semaphore was closed + // (only happens on shutdown — propagate as immediate-error). + let _permit = sem + .acquire() + .await + .map_err(|e| format!("semaphore closed: {e}"))?; + let source_flags = extra_flags_ptr.for_source(&source); - let outcome = compiler_ptr.compile(&source, &obj, &source_flags).await; - if !matches!(&outcome, Ok(result) if result.success) { - failed.store(true, std::sync::atomic::Ordering::Relaxed); - } else { - fbuild_packages::library::tu_history::record_in( - &history_root_t, - &source, - &signature, - started.elapsed(), - ); - } - match outcome { + match compiler_ptr.compile(&source, &obj, &source_flags).await { Ok(result) if result.success => { let stderr = result.stderr.trim().to_string(); let count = counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1; @@ -383,7 +334,6 @@ mod tests { &LanguageExtraFlags::default(), &slots, None, - Some(tmp.path()), ) .await; assert!(result.is_err()); @@ -422,23 +372,14 @@ mod tests { let (core_build, sketch_build) = (tmp.path().join("core"), tmp.path().join("src")); let (a, b) = tokio::join!( - compile_sources_parallel_shared( - &compiler, - &core, - &core_build, - &flags, - &slots, - None, - Some(tmp.path()) - ), + compile_sources_parallel_shared(&compiler, &core, &core_build, &flags, &slots, None), compile_sources_parallel_shared( &compiler, &sketch, &sketch_build, &flags, &slots, - None, - Some(tmp.path()) + None ), ); assert_eq!(a.unwrap().objects.len(), 3); @@ -475,135 +416,4 @@ mod tests { "the sketch phase must start while core compiles are still running" ); } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn cpp_units_are_dispatched_before_c_and_assembly() { - let tmp = tempfile::tempdir().unwrap(); - let names = ["a.c", "b.S", "c.cpp", "d.c", "e.cpp"]; - let sources: Vec = names - .iter() - .map(|name| { - let path = tmp.path().join(name); - std::fs::write(&path, "int x;\n").unwrap(); - path - }) - .collect(); - let compiler = TracingCompiler { - gcc: PathBuf::from("/toolchain/bin/gcc"), - in_flight: Default::default(), - max_in_flight: Default::default(), - spans: Default::default(), - }; - let result = compile_sources_parallel_shared( - &compiler, - &sources, - &tmp.path().join("obj"), - &LanguageExtraFlags::default(), - &Arc::new(Semaphore::new(1)), - None, - Some(tmp.path()), - ) - .await - .unwrap(); - - let mut spans = compiler.spans.lock().unwrap().clone(); - spans.sort_by_key(|span| span.1); - let started: Vec = spans - .iter() - .map(|span| span.0.file_name().unwrap().to_string_lossy().into_owned()) - .collect(); - assert_eq!(started, ["c.cpp", "e.cpp", "a.c", "d.c", "b.S"]); - let objects: Vec = result - .objects - .iter() - .map(|o| o.file_name().unwrap().to_string_lossy().into_owned()) - .collect(); - let expected: Vec = sources - .iter() - .map(|src| { - CompilerBase::object_path(src, &tmp.path().join("obj")) - .file_name() - .unwrap() - .to_string_lossy() - .into_owned() - }) - .collect(); - assert_eq!(objects, expected, "objects stay in source order"); - } - - /// FastLED/fbuild#1564: with recorded history showing `z_slow.cpp` far - /// outweighs the others, it must start first even though it sorts last - /// in source order. The history root is threaded explicitly - /// (`compile_sources_parallel_shared`'s last argument) rather than via - /// `FBUILD_CACHE_DIR`, so this test stays hermetic under `cargo test`'s - /// parallel test threads. - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn recorded_history_dispatches_the_longest_tu_first() { - let cache_dir = tempfile::tempdir().unwrap(); - - let tmp = tempfile::tempdir().unwrap(); - let names = ["a_fast.cpp", "b_fast.cpp", "z_slow.cpp"]; - let sources: Vec = names - .iter() - .map(|name| { - let path = tmp.path().join(name); - std::fs::write(&path, "int x;\n").unwrap(); - path - }) - .collect(); - - let z_slow = sources.iter().find(|s| s.ends_with("z_slow.cpp")).unwrap(); - // TracingCompiler::rebuild_signature is just the source path string. - let z_slow_signature = z_slow.to_string_lossy().into_owned(); - fbuild_packages::library::tu_history::record_in( - cache_dir.path(), - z_slow, - &z_slow_signature, - std::time::Duration::from_secs(20), - ); - - let compiler = TracingCompiler { - gcc: PathBuf::from("/toolchain/bin/gcc"), - in_flight: Default::default(), - max_in_flight: Default::default(), - spans: Default::default(), - }; - compile_sources_parallel_shared( - &compiler, - &sources, - &tmp.path().join("obj"), - &LanguageExtraFlags::default(), - &Arc::new(Semaphore::new(1)), - None, - Some(cache_dir.path()), - ) - .await - .unwrap(); - - let mut spans = compiler.spans.lock().unwrap().clone(); - spans.sort_by_key(|span| span.1); - let started: Vec = spans - .iter() - .map(|span| span.0.file_name().unwrap().to_string_lossy().into_owned()) - .collect(); - assert_eq!( - started[0], "z_slow.cpp", - "the longest recorded TU must start first, got {started:?}" - ); - - // A history record now exists for every compiled TU (recorded after - // its own successful compile). - for source in &sources { - let signature = source.to_string_lossy().into_owned(); - assert!( - fbuild_packages::library::tu_history::lookup_in( - cache_dir.path(), - source, - &signature - ) - .is_some(), - "expected a recorded duration for {source:?}" - ); - } - } } diff --git a/crates/fbuild-build-engine/src/pipeline/compile.rs b/crates/fbuild-build-engine/src/pipeline/compile.rs index a12c4063..d86e09fd 100644 --- a/crates/fbuild-build-engine/src/pipeline/compile.rs +++ b/crates/fbuild-build-engine/src/pipeline/compile.rs @@ -33,7 +33,6 @@ pub async fn compile_sources( extra_flags, jobs, Some(build_log), - None, ) .await?; if !result.warnings.is_empty() { @@ -84,7 +83,6 @@ pub async fn compile_local_libraries( extra_flags, jobs, Some(build_log), - None, ) .await .map_err(|e| { @@ -263,38 +261,4 @@ mod tests { // only the IDE database. assert!(!project.join(CompileDatabase::RAW_FILE_NAME).exists()); } - - /// FastLED/fbuild#1559: the per-build toolchain version line must not - /// spawn the compiler on every build; the probe is memoized per path. - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn log_toolchain_version_probes_the_compiler_once() { - // The fake compiler is a POSIX shell script. - if fbuild_core::platform::host::is_windows() { - return; - } - let tmp = tempfile::tempdir().unwrap(); - let probes = tmp.path().join("probes.log"); - let gcc = tmp.path().join("xtensa-gcc"); - let staging = tmp.path().join("xtensa-gcc.staging"); - std::fs::write( - &staging, - format!( - "#!/bin/sh\necho \"$*\" >> '{}'\necho 14.2.0\n", - probes.display() - ), - ) - .unwrap(); - fbuild_core::platform::fs::set_executable(&staging).unwrap(); - std::fs::rename(&staging, &gcc).unwrap(); - - let mut log = BuildLog::new(); - log_toolchain_version(&gcc, "xtensa-gcc", &mut log).await; - log_toolchain_version(&gcc, "xtensa-gcc", &mut log).await; - - let lines = log.into_lines(); - assert_eq!(lines.len(), 2, "{lines:?}"); - assert!(lines.iter().all(|l| l == "Toolchain: xtensa-gcc 14.2.0")); - let spawned = std::fs::read_to_string(&probes).unwrap().lines().count(); - assert_eq!(spawned, 1, "second build respawned the compiler"); - } } diff --git a/crates/fbuild-build-engine/src/pipeline/library.rs b/crates/fbuild-build-engine/src/pipeline/library.rs index 17a09c99..e1cd6509 100644 --- a/crates/fbuild-build-engine/src/pipeline/library.rs +++ b/crates/fbuild-build-engine/src/pipeline/library.rs @@ -1,8 +1,7 @@ //! Library compilation helpers: `LibraryBuildEnv`, archiver selection, //! extra library roots, the "project-as-library" compile path, and -//! `resolve_lib_deps`, which downloads registry/remote `lib_deps` entries -//! before the compiler is created and defers their compile to the build's -//! shared job gate. +//! `ensure_lib_deps` for downloading registry/remote `lib_deps` entries +//! before the compiler is created. use std::path::{Path, PathBuf}; @@ -207,22 +206,60 @@ pub async fn compile_extra_libraries( /// library archive was produced. /// /// Matches PlatformIO's project-as-library convention; see ISSUES.md Issue 1. -/// Its compiles draw from `gate`, the build's shared job budget -/// (FastLED/fbuild#1559). pub async fn compile_project_as_library( project_dir: &Path, src_dir: &Path, build_dir: &Path, env: &LibraryBuildEnv<'_>, existing_lib_names: &std::collections::HashSet, - gate: &fbuild_packages::library::library_compiler::JobGate, ) -> Result> { - let Some((lib_name, sources)) = - project_library_candidate(project_dir, src_dir, existing_lib_names, true) - else { + // Guard 1: must be a library project (library.json or library.properties at root). + if !is_project_a_library(project_dir) { return Ok(None); - }; + } + + // Guard 2: project must have a src/ dir. let project_src = project_dir.join("src"); + if !project_src.is_dir() { + return Ok(None); + } + + // Guard 3: must be building an example. If src_dir IS the project's own + // src/, we're doing a normal library self-build and the sketch scanner + // is already compiling these sources — don't double-compile. + // Also guard the BuildContext fallback where src_dir collapses to + // project_dir (would cause the scanner to recursively pick up library + // sources, leading to multiply-defined symbols). + if src_dir == project_src || src_dir == project_dir { + return Ok(None); + } + + // Compute lib name from project dir basename. + let lib_name = project_dir + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("project") + .to_lowercase(); + + // Guard 4: collision with a lib// subdirectory — lib/ wins + // (matches PlatformIO behavior). + if existing_lib_names.contains(&lib_name) { + tracing::warn!( + "project-as-library '{}' collides with lib/{} — skipping project root", + lib_name, + lib_name + ); + return Ok(None); + } + + // Discover sources via the same helper used for installed libraries. + let lib_info = + fbuild_packages::library::library_info::InstalledLibrary::new(project_dir, &lib_name); + let sources = lib_info.get_source_files(); + if sources.is_empty() { + tracing::info!("project-as-library '{}' is header-only", lib_name); + return Ok(None); + } tracing::info!( "compiling project-as-library: {} ({} sources from {})", @@ -234,7 +271,7 @@ pub async fn compile_project_as_library( let project_libs_dir = build_dir.join("project_lib"); std::fs::create_dir_all(&project_libs_dir)?; - match fbuild_packages::library::library_compiler::compile_library_gated( + match fbuild_packages::library::library_compiler::compile_library_with_jobs( &lib_name, &sources, env.include_dirs, @@ -245,11 +282,10 @@ pub async fn compile_project_as_library( env.cpp_flags, &project_libs_dir, env.verbose, - gate, + env.jobs, env.compiler_cache, None, None, - None, ) .await { @@ -269,106 +305,90 @@ pub async fn compile_project_as_library( } } -/// Name of the archive [`compile_project_as_library`] produces -/// (`lib{name}.a`), or `None` when it produces none. Known before anything -/// compiles, so a build can plan around it (FastLED/fbuild#1559). -pub fn project_library_name( - project_dir: &Path, - src_dir: &Path, - existing_lib_names: &std::collections::HashSet, -) -> Option { - project_library_candidate(project_dir, src_dir, existing_lib_names, false).map(|(name, _)| name) -} - -/// The project-as-library name and sources, when every guard passes and the -/// library has sources. `log` reports the collision / header-only skips. -fn project_library_candidate( +/// Download and compile `lib_deps` entries from `platformio.ini`. +/// +/// Called by build orchestrators **before** the compiler is created, so the +/// returned include directories can be folded into the compiler's search path. +/// The caller is responsible for: +/// +/// 1. Adding the returned `include_dirs` to the compiler's include list. +/// 2. Passing the returned `archives` to the linker (via +/// `run_sequential_build_with_libs`' `extra_link_inputs` parameter). +/// +/// `base_includes` is the set of include directories available before any +/// `lib_deps` are downloaded (core, variant, sketch, toolchain sysroot). +/// `libs_dir` is where downloaded libraries will be staged — callers typically +/// use `/libs`. +/// +/// When `lib_deps` is empty this returns `(vec![], vec![])` immediately with +/// no I/O or network access. +/// +/// FastLED/fbuild#1276: registry dependencies (`fastled/FastLED@^3.10.3`) +/// were classified by `fbuild sync` but never downloaded by the build +/// orchestrator, so the compile step couldn't find their headers. +#[allow(clippy::too_many_arguments)] +pub async fn ensure_lib_deps( + lib_deps: &[String], + lib_ignore: &[String], + gcc_path: &Path, + gxx_path: &Path, + ar_path: &Path, + c_flags: &[String], + cpp_flags: &[String], + base_includes: &[PathBuf], project_dir: &Path, - src_dir: &Path, - existing_lib_names: &std::collections::HashSet, - log: bool, -) -> Option<(String, Vec)> { - // Guard 1: must be a library project (library.json or library.properties at root). - if !is_project_a_library(project_dir) { - return None; - } - - // Guard 2: project must have a src/ dir. - let project_src = project_dir.join("src"); - if !project_src.is_dir() { - return None; - } - - // Guard 3: must be building an example. If src_dir IS the project's own - // src/, we're doing a normal library self-build and the sketch scanner - // is already compiling these sources — don't double-compile. - // Also guard the BuildContext fallback where src_dir collapses to - // project_dir (would cause the scanner to recursively pick up library - // sources, leading to multiply-defined symbols). - if src_dir == project_src || src_dir == project_dir { - return None; + libs_dir: &Path, + verbose: bool, + jobs: usize, + compiler_cache: Option<&Path>, +) -> Result<(Vec, Vec)> { + if lib_deps.is_empty() { + return Ok((vec![], vec![])); } - // Compute lib name from project dir basename. - let lib_name = project_dir - .file_name() - .and_then(|n| n.to_str()) - .unwrap_or("project") - .to_lowercase(); + tracing::info!( + "downloading {} lib_deps to {}", + lib_deps.len(), + libs_dir.display() + ); - // Guard 4: collision with a lib// subdirectory — lib/ wins - // (matches PlatformIO behavior). - if existing_lib_names.contains(&lib_name) { - if log { - tracing::warn!( - "project-as-library '{}' collides with lib/{} — skipping project root", - lib_name, - lib_name - ); - } - return None; - } + let lib_result = fbuild_packages::library::library_manager::ensure_libraries( + lib_deps, + lib_ignore, + gcc_path, + gxx_path, + ar_path, + c_flags, + cpp_flags, + base_includes, + project_dir, + libs_dir, + verbose, + jobs, + compiler_cache, + ) + .await?; - // Discover sources via the same helper used for installed libraries. - let lib_info = - fbuild_packages::library::library_info::InstalledLibrary::new(project_dir, &lib_name); - let sources = lib_info.get_source_files(); - if sources.is_empty() { - if log { - tracing::info!("project-as-library '{}' is header-only", lib_name); - } - return None; - } - Some((lib_name, sources)) -} + // FastLED/fbuild#966: sort include dirs so `-I` flags are deterministic + // across checkouts (read_dir order varies by filesystem). + let mut lib_include_dirs = lib_result.include_dirs; + lib_include_dirs.sort(); -/// `lib_deps` resolved and installed, compiled later on the build's shared -/// job gate so libraries overlap the core, variant and sketch -/// (FastLED/fbuild#1559). `Default` is "no lib_deps". -#[derive(Default)] -pub struct LibDeps { - plan: Option, -} + tracing::info!( + "lib_deps: {} include dirs, {} archives", + lib_include_dirs.len(), + lib_result.archives.len() + ); -impl LibDeps { - /// Compile every library on `gate`; archives come back in link order. - pub async fn compile( - self, - gate: &fbuild_packages::library::library_compiler::JobGate, - ) -> Result> { - match self.plan { - Some(plan) => plan.compile(gate).await, - None => Ok(Vec::new()), - } - } + Ok((lib_include_dirs, lib_result.archives)) } /// Resolve `lib_deps` from platformio.ini: pick the LTO-aware archiver, -/// download the libraries, and fold their include dirs into the caller's list. +/// download/compile libraries, and fold the returned include dirs into the +/// caller's list. /// -/// Nothing is compiled here: the returned [`LibDeps`] is compiled by the build -/// pipeline alongside everything else. When `lib_deps` is empty this returns -/// immediately with no I/O or network access. +/// Returns the compiled library archives for link-time. When `lib_deps` is +/// empty this returns an empty vec immediately with no I/O or network access. /// /// This is the single choke-point that replaced a ~55-line copy-paste block /// duplicated across every build orchestrator (FastLED/fbuild#1292). Each @@ -388,19 +408,15 @@ pub async fn resolve_lib_deps( cpp_flags: &[String], include_dirs: &mut Vec, verbose: bool, + jobs: usize, compiler_cache: Option<&Path>, -) -> Result { +) -> Result> { if lib_deps.is_empty() { - return Ok(LibDeps::default()); + return Ok(Vec::new()); } let dep_lib_ar_path = pick_archiver(ar_path, gcc_ar_path, c_flags, cpp_flags); let libs_dir = build_dir.join("libs"); - tracing::info!( - "downloading {} lib_deps to {}", - lib_deps.len(), - libs_dir.display() - ); - let resolved = fbuild_packages::library::library_manager::resolve_libraries( + let (lib_include_dirs, archives) = ensure_lib_deps( lib_deps, lib_ignore, gcc_path, @@ -412,27 +428,14 @@ pub async fn resolve_lib_deps( project_dir, &libs_dir, verbose, + jobs, compiler_cache, ) .await?; - // FastLED/fbuild#966: sort include dirs so `-I` flags are deterministic - // across checkouts (read_dir order varies by filesystem). - let mut lib_include_dirs = resolved.include_dirs; - lib_include_dirs.sort(); - tracing::info!("lib_deps: {} include dirs", lib_include_dirs.len()); include_dirs.extend(lib_include_dirs); - Ok(LibDeps { - plan: Some(resolved.plan), - }) + Ok(archives) } -// See `library_shared_gate_tests.rs` (FastLED/fbuild#1559): proves -// `LibDeps::compile` and `compile_sources_parallel_shared` share one job -// budget when run concurrently against the same `JobGate`. -#[cfg(test)] -#[path = "library_shared_gate_tests.rs"] -mod shared_gate_tests; - #[cfg(test)] mod pick_archiver_tests { use super::*; @@ -562,7 +565,6 @@ mod project_as_library_tests { &project_dir.join("build"), &env, &HashSet::new(), - &fbuild_packages::library::library_compiler::job_gate(1), ) .await; assert!(matches!(result, Ok(None))); @@ -590,7 +592,6 @@ mod project_as_library_tests { &project_dir.join("build"), &env, &HashSet::new(), - &fbuild_packages::library::library_compiler::job_gate(1), ) .await; assert!(matches!(result, Ok(None))); @@ -619,7 +620,6 @@ mod project_as_library_tests { &project_dir.join("build"), &env, &HashSet::new(), - &fbuild_packages::library::library_compiler::job_gate(1), ) .await; assert!(matches!(result, Ok(None))); @@ -644,7 +644,6 @@ mod project_as_library_tests { &project_dir.join("build"), &env, &HashSet::new(), - &fbuild_packages::library::library_compiler::job_gate(1), ) .await; assert!(matches!(result, Ok(None))); @@ -674,7 +673,6 @@ mod project_as_library_tests { &project_dir.join("build"), &env, &HashSet::new(), - &fbuild_packages::library::library_compiler::job_gate(1), ) .await; assert!(matches!(result, Ok(None))); @@ -710,7 +708,6 @@ mod project_as_library_tests { &project_dir.join("build"), &env, &existing, - &fbuild_packages::library::library_compiler::job_gate(1), ) .await; assert!(matches!(result, Ok(None))); @@ -745,7 +742,6 @@ mod project_as_library_tests { &project_dir.join("build"), &env, &HashSet::new(), - &fbuild_packages::library::library_compiler::job_gate(1), ) .await; // Must NOT be Ok(None) — that would mean a guard skipped compile. @@ -755,46 +751,4 @@ mod project_as_library_tests { panic!("expected compile to be attempted, but a guard returned Ok(None)"); } } - - /// FastLED/fbuild#1559: `project_library_name` predicts, before anything - /// compiles, exactly when `compile_project_as_library` yields an archive. - #[test] - fn test_project_library_name_matches_the_compile_guards() { - let tmp = tempfile::TempDir::new().unwrap(); - let project_dir = tmp.path().join("FastLED"); - let project_src = project_dir.join("src"); - let example = project_dir.join("examples").join("Blink"); - std::fs::create_dir_all(&project_src).unwrap(); - std::fs::create_dir_all(&example).unwrap(); - let none = HashSet::new(); - - // Not a library project yet. - std::fs::write(project_src.join("FastLED.cpp"), "").unwrap(); - assert_eq!(project_library_name(&project_dir, &example, &none), None); - - std::fs::write(project_dir.join("library.json"), "{}").unwrap(); - assert_eq!( - project_library_name(&project_dir, &example, &none).as_deref(), - Some("fastled") - ); - // A normal self-build, or a src dir collapsed to the project root. - assert_eq!( - project_library_name(&project_dir, &project_src, &none), - None - ); - assert_eq!( - project_library_name(&project_dir, &project_dir, &none), - None - ); - // lib/fastled wins. - let collision: HashSet = ["fastled".to_string()].into(); - assert_eq!( - project_library_name(&project_dir, &example, &collision), - None - ); - // Header-only. - std::fs::remove_file(project_src.join("FastLED.cpp")).unwrap(); - std::fs::write(project_src.join("FastLED.h"), "").unwrap(); - assert_eq!(project_library_name(&project_dir, &example, &none), None); - } } diff --git a/crates/fbuild-build-engine/src/pipeline/library_shared_gate_tests.rs b/crates/fbuild-build-engine/src/pipeline/library_shared_gate_tests.rs deleted file mode 100644 index dacf7c7a..00000000 --- a/crates/fbuild-build-engine/src/pipeline/library_shared_gate_tests.rs +++ /dev/null @@ -1,330 +0,0 @@ -//! Proves `LibDeps::compile` and `compile_sources_parallel_shared` share ONE -//! job budget (FastLED/fbuild#1559): a fake in-process core compile and a -//! resolved `LibDeps` (two symlinked local libraries, compiled via real -//! fake-shell-script gcc/g++/ar) run in `tokio::join!` against the same -//! `JobGate`. Overlap between the two phases (never possible before -//! FastLED/fbuild#1468/#1559 unified the gate) plus a total in-flight count -//! that never exceeds the gate size proves the sharing. - -use std::sync::Arc; -use std::time::{SystemTime, UNIX_EPOCH}; - -use fbuild_core::Result as FbuildResult; -use fbuild_core::path::NormalizedPath; - -use super::resolve_lib_deps; -use crate::compiler::{CompileResult, Compiler}; -use crate::flag_overlay::LanguageExtraFlags; -use crate::parallel::compile_sources_parallel_shared; -use std::path::Path; - -/// Fake in-process "core" compiler: records a (label, phase, ns) event to -/// the shared log for every compile, sleeping long enough that at least -/// one lib_deps archive compile is guaranteed to overlap it. -struct FakeCoreCompiler { - gcc: NormalizedPath, - log: Arc>>, - sleep_ms: u64, -} - -fn now_ns() -> u128 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_nanos() -} - -#[async_trait::async_trait] -impl Compiler for FakeCoreCompiler { - async fn compile_one( - &self, - _compiler_path: &Path, - _source: &Path, - output: &Path, - _flags: &[String], - _extra_flags: &[String], - ) -> FbuildResult { - self.log - .lock() - .unwrap() - .push(("core".to_string(), "start".to_string(), now_ns())); - tokio::time::sleep(std::time::Duration::from_millis(self.sleep_ms)).await; - self.log - .lock() - .unwrap() - .push(("core".to_string(), "end".to_string(), now_ns())); - if let Some(parent) = output.parent() { - std::fs::create_dir_all(parent).ok(); - } - std::fs::write(output, "").unwrap(); - Ok(CompileResult { - success: true, - object_file: output.to_path_buf(), - stdout: String::new(), - stderr: String::new(), - exit_code: 0, - }) - } - - fn gcc_path(&self) -> &Path { - self.gcc.as_path() - } - - fn gxx_path(&self) -> &Path { - self.gcc.as_path() - } - - fn c_flags(&self) -> Vec { - Vec::new() - } - - fn cpp_flags(&self) -> Vec { - Vec::new() - } - - fn rebuild_signature(&self, source: &Path, _extra: &[String], _out: &Path) -> String { - source.to_string_lossy().into_owned() - } -} - -/// Installs a fake gcc/g++ that logs " start " / -/// " end " to `log` and sleeps ~0.1s before touching `-o`. -/// Staged via `.staging` + `set_executable` + `rename` to avoid ETXTBSY -/// (FastLED's `symlink://` local libraries recompile on every resolve). -fn install_timed_fake_compiler(path: &Path, log: &Path) { - let script = format!( - "#!/bin/sh\n\ - obj=\"\"\n\ - src=\"\"\n\ - prev=\"\"\n\ - for arg in \"$@\"; do\n\ - \x20 if [ \"$prev\" = \"-o\" ]; then obj=\"$arg\"; fi\n\ - \x20 if [ \"$prev\" = \"-c\" ]; then src=\"$arg\"; fi\n\ - \x20 prev=\"$arg\"\n\ - done\n\ - name=$(basename \"$src\")\n\ - echo \"$name start $(date +%s%N)\" >> \"{log}\"\n\ - sleep 0.1\n\ - echo \"$name end $(date +%s%N)\" >> \"{log}\"\n\ - mkdir -p \"$(dirname \"$obj\")\"\n\ - touch \"$obj\"\n", - log = log.display() - ); - let staging = path.with_extension("staging"); - std::fs::write(&staging, script).unwrap(); - fbuild_core::platform::fs::set_executable(&staging).unwrap(); - std::fs::rename(&staging, path).unwrap(); -} - -fn install_fake_ar(path: &Path) { - let script = "#!/bin/sh\ntouch \"$2\"\n"; - let staging = path.with_extension("staging"); - std::fs::write(&staging, script).unwrap(); - fbuild_core::platform::fs::set_executable(&staging).unwrap(); - std::fs::rename(&staging, path).unwrap(); -} - -fn make_local_library(project: &Path, rel: &str, name: &str) { - let src = project.join(rel).join("src"); - std::fs::create_dir_all(&src).unwrap(); - std::fs::write(src.join(format!("{name}.cpp")), "int f() { return 1; }\n").unwrap(); - std::fs::write(src.join(format!("{name}.h")), "int f();\n").unwrap(); -} - -/// Parses " start|end " lines from a fake-compiler log into -/// `(name, start_ns, end_ns)` intervals. -fn parse_intervals(log_text: &str) -> Vec<(String, u128, u128)> { - let mut starts: std::collections::HashMap = std::collections::HashMap::new(); - let mut out = Vec::new(); - for line in log_text.lines() { - let parts: Vec<&str> = line.split_whitespace().collect(); - if parts.len() != 3 { - continue; - } - let (name, phase, ts) = (parts[0], parts[1], parts[2]); - let ts: u128 = ts.parse().unwrap(); - match phase { - "start" => { - starts.insert(name.to_string(), ts); - } - "end" => { - if let Some(start) = starts.remove(name) { - out.push((name.to_string(), start, ts)); - } - } - _ => {} - } - } - out -} - -fn intervals_overlap(a: (u128, u128), b: (u128, u128)) -> bool { - a.0 < b.1 && b.0 < a.1 -} - -/// Sweep line over every interval's start/end events; returns the peak -/// number of simultaneously in-flight compiles. -fn max_concurrency(intervals: &[(u128, u128)]) -> usize { - let mut events: Vec<(u128, i32)> = Vec::new(); - for &(start, end) in intervals { - events.push((start, 1)); - events.push((end, -1)); - } - events.sort(); - let (mut cur, mut max) = (0i32, 0i32); - for (_, delta) in events { - cur += delta; - max = max.max(cur); - } - max.max(0) as usize -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn lib_deps_and_core_share_one_job_gate() { - if fbuild_core::platform::host::is_windows() { - return; - } - for _ in 0..5 { - let tmp = tempfile::TempDir::new().unwrap(); - let project = tmp.path().join("project"); - make_local_library(&project, "liba", "liba"); - make_local_library(&project, "libb", "libb"); - let build_dir = tmp.path().join("build"); - std::fs::create_dir_all(&build_dir).unwrap(); - - let tools_dir = tmp.path().join("tools"); - std::fs::create_dir_all(&tools_dir).unwrap(); - let lib_log = tmp.path().join("lib_compile.log"); - let gcc = tools_dir.join("gcc"); - let gxx = tools_dir.join("g++"); - let ar = tools_dir.join("ar"); - install_timed_fake_compiler(&gcc, &lib_log); - install_timed_fake_compiler(&gxx, &lib_log); - install_fake_ar(&ar); - - let mut include_dirs = Vec::new(); - let lib_deps = resolve_lib_deps( - &[ - "LibA=symlink://liba".to_string(), - "LibB=symlink://libb".to_string(), - ], - &[], - &project, - &build_dir, - &gcc, - &gxx, - &ar, - &ar, - &[], - &[], - &mut include_dirs, - false, - None, - ) - .await - .unwrap(); - - let core_log: Arc>> = - Arc::new(std::sync::Mutex::new(Vec::new())); - let core_compiler = FakeCoreCompiler { - gcc: NormalizedPath::new(&gcc), - log: core_log.clone(), - // Long enough that the two ~0.1s library compiles (which - // must serialize behind each other once the gate's second - // slot is claimed by this core compile) are guaranteed to - // land inside this compile's [start, end) window. - sleep_ms: 300, - }; - let core_source = project.join("core.cpp"); - std::fs::write(&core_source, "int g();\n").unwrap(); - let core_build_dir = build_dir.join("core"); - - let gate = fbuild_packages::library::library_compiler::job_gate(2); - let core_sources = [core_source.clone()]; - let core_flags = LanguageExtraFlags::default(); - let (core_result, lib_archives) = tokio::join!( - compile_sources_parallel_shared( - &core_compiler, - &core_sources, - &core_build_dir, - &core_flags, - &gate, - None, - None, - ), - lib_deps.compile(&gate), - ); - - let core_result = core_result.unwrap(); - assert_eq!(core_result.objects.len(), 1); - let lib_archives = lib_archives.unwrap(); - assert_eq!( - lib_archives.len(), - 2, - "both lib_deps archives must be produced" - ); - for archive in &lib_archives { - assert!(archive.exists(), "{} must exist", archive.display()); - } - - // Merge the in-process core events with the subprocess lib - // events recorded by the fake gcc/g++ scripts. - let core_events = core_log.lock().unwrap().clone(); - let core_intervals: Vec<(u128, u128)> = { - let text = core_events - .iter() - .map(|(label, phase, ts)| format!("{label} {phase} {ts}")) - .collect::>() - .join("\n"); - parse_intervals(&text) - .into_iter() - .map(|(_, s, e)| (s, e)) - .collect() - }; - assert_eq!(core_intervals.len(), 1, "exactly one core compile ran"); - let core_interval = core_intervals[0]; - - let lib_log_text = std::fs::read_to_string(&lib_log).unwrap(); - let lib_intervals: Vec<(u128, u128)> = parse_intervals(&lib_log_text) - .into_iter() - .map(|(_, s, e)| (s, e)) - .collect(); - assert_eq!( - lib_intervals.len(), - 2, - "both library TU compiles must be logged" - ); - - // (b) the library compile overlaps the core compile — not - // strictly sequential. - assert!( - lib_intervals - .iter() - .any(|&lib_interval| intervals_overlap(core_interval, lib_interval)), - "expected at least one lib_deps compile to overlap the core compile: \ - core={core_interval:?} libs={lib_intervals:?}" - ); - - // (a) total in-flight across both never exceeds the gate size. - let mut all_intervals = lib_intervals.clone(); - all_intervals.push(core_interval); - let peak = max_concurrency(&all_intervals); - assert!( - peak <= 2, - "gate of size 2 must never be exceeded, saw peak={peak}" - ); - - // The two lib compiles must not run concurrently with each - // other AND the core compile at once (only one gate, size 2: - // core takes one slot for its whole 300ms, so the two lib - // compiles must themselves serialize behind the gate's other - // slot). - assert!( - !intervals_overlap(lib_intervals[0], lib_intervals[1]), - "with only one non-core slot, the two lib_deps TUs must not overlap \ - each other: {:?} vs {:?}", - lib_intervals[0], - lib_intervals[1] - ); - } -} diff --git a/crates/fbuild-build-engine/src/pipeline/mod.rs b/crates/fbuild-build-engine/src/pipeline/mod.rs index a9ca78dc..f4e85d2c 100644 --- a/crates/fbuild-build-engine/src/pipeline/mod.rs +++ b/crates/fbuild-build-engine/src/pipeline/mod.rs @@ -21,8 +21,8 @@ pub use compile::{ }; pub use context::BuildContext; pub use library::{ - LibDeps, LibraryBuildEnv, add_extra_library_include_dirs, compile_extra_libraries, - compile_project_as_library, discover_extra_library_roots, pick_archiver, project_library_name, + LibraryBuildEnv, add_extra_library_include_dirs, compile_extra_libraries, + compile_project_as_library, discover_extra_library_roots, ensure_lib_deps, pick_archiver, resolve_lib_deps, }; pub use link::{assemble_build_result, handle_link_result}; diff --git a/crates/fbuild-build-engine/src/pipeline/sequential.rs b/crates/fbuild-build-engine/src/pipeline/sequential.rs index 5b841491..b4f2c0fd 100644 --- a/crates/fbuild-build-engine/src/pipeline/sequential.rs +++ b/crates/fbuild-build-engine/src/pipeline/sequential.rs @@ -1,7 +1,7 @@ //! The sequential compile → link → result pipeline used by AVR, Teensy, //! RP2040, STM32, ESP8266, CH32V, NRF52, SAM, Renesas, and Apollo3. -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::time::Instant; use fbuild_core::Result; @@ -39,7 +39,6 @@ pub async fn run_sequential_build_with_libs( params: &BuildParams, sources: &SourceCollection, extra_link_inputs: &[PathBuf], - lib_deps: super::library::LibDeps, lib_env: Option<&LibraryBuildEnv<'_>>, arch: TargetArchitecture, platform_label: &str, @@ -268,56 +267,19 @@ pub async fn run_sequential_build_with_libs( &build_log_mutex, )); - // `lib_deps` and the project-as-library archive compile in the same pool - // (FastLED/fbuild#1559): nothing but the link needs their archives. - let lib_deps_compile = timed(lib_deps.compile(&compile_slots)); - let existing_lib_names = local_lib_dir_names(¶ms.project_dir); - let project_as_lib = timed(async { - match lib_env { - Some(env) => { - compile_project_as_library( - ¶ms.project_dir, - &ctx.src_dir, - &ctx.build_dir, - env, - &existing_lib_names, - &compile_slots, - ) - .await - } - None => Ok(None), - } - }); - let ( ((mut core_objects, core_time), (variant_objects, variant_time), store_time), (sketch_objects, sketch_time), (library_objects, library_time), - (lib_deps_archives, lib_deps_time), - (project_as_lib_archive, project_as_lib_time), ) = { - let (framework, sketch, libraries, lib_deps_compile, project_as_lib) = tokio::join!( - framework, - sketch, - libraries, - lib_deps_compile, - project_as_lib - ); - ( - framework?, - sketch?, - libraries?, - lib_deps_compile?, - project_as_lib?, - ) + let (framework, sketch, libraries) = tokio::join!(framework, sketch, libraries); + (framework?, sketch?, libraries?) }; perf.record("compile-core", core_time); perf.record("compile-variant", variant_time); perf.record("core-cache-store", store_time); perf.record("compile-sketch", sketch_time); perf.record("compile-local-libs", library_time); - perf.record("compile-lib-deps", lib_deps_time); - perf.record("project-as-lib", project_as_lib_time); core_objects.extend(variant_objects); // Unwrap the build log Mutex back into the context for the remaining @@ -326,6 +288,40 @@ pub async fn run_sequential_build_with_libs( .into_inner() .unwrap_or_else(|e| e.into_inner()); + // Project-as-library: compile project root's src/ as an archive when + // building an example sketch from a library project (e.g. FastLED examples). + // Only runs when caller provided a LibraryBuildEnv with toolchain paths. + let project_as_lib_archive: Option = { + let _g = perf.phase("project-as-lib"); + if let Some(env) = lib_env { + // Collect existing lib/* names so the helper can detect collisions. + let mut existing_lib_names = std::collections::HashSet::new(); + let local_lib_dir = params.project_dir.join("lib"); + if local_lib_dir.is_dir() { + if let Ok(entries) = std::fs::read_dir(&local_lib_dir) { + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + if let Some(name) = path.file_name().and_then(|n| n.to_str()) { + existing_lib_names.insert(name.to_lowercase()); + } + } + } + } + } + compile_project_as_library( + ¶ms.project_dir, + &ctx.src_dir, + &ctx.build_dir, + env, + &existing_lib_names, + ) + .await? + } else { + None + } + }; + // Generate compile_commands.json let compile_database_path = { let _g = perf.phase("compile-db"); @@ -352,7 +348,6 @@ pub async fn run_sequential_build_with_libs( crate::build_output::log_linking(&mut ctx.build_log, "Linking firmware.elf"); core_objects.extend(library_objects); core_objects.extend(extra_link_inputs.iter().cloned()); - core_objects.extend(lib_deps_archives); if let Some(archive) = project_as_lib_archive { // GCC accepts .a in the same positional slot as .o files. core_objects.push(archive); @@ -439,20 +434,3 @@ async fn timed( let value = future.await?; Ok((value, started.elapsed())) } - -/// Lower-cased names of the project's `lib/*` directories; the -/// project-as-library archive yields to a `lib/` entry of the same name. -fn local_lib_dir_names(project_dir: &Path) -> std::collections::HashSet { - let mut names = std::collections::HashSet::new(); - if let Ok(entries) = std::fs::read_dir(project_dir.join("lib")) { - for entry in entries.flatten() { - let path = entry.path(); - if path.is_dir() { - if let Some(name) = path.file_name().and_then(|n| n.to_str()) { - names.insert(name.to_lowercase()); - } - } - } - } - names -} diff --git a/crates/fbuild-build-engine/src/zccache_embedded.rs b/crates/fbuild-build-engine/src/zccache_embedded.rs index efd225d0..4e25e102 100644 --- a/crates/fbuild-build-engine/src/zccache_embedded.rs +++ b/crates/fbuild-build-engine/src/zccache_embedded.rs @@ -90,36 +90,6 @@ pub struct EmbeddedCompileOutcome { pub cached: bool, } -/// zccache's process-wide cap on concurrently running compilers. It is read -/// once, when the embedded service starts. -pub const ZCCACHE_MAX_PARALLEL_COMPILES_ENV: &str = "ZCCACHE_MAX_PARALLEL_COMPILES"; - -/// fbuild's spelling of that cap. `FBUILD_*` variables survive the scrubbed -/// environment the CLI spawns the daemon with; `ZCCACHE_*` ones do not. -pub const FBUILD_MAX_PARALLEL_COMPILES_ENV: &str = "FBUILD_MAX_PARALLEL_COMPILES"; - -/// Value to export as [`ZCCACHE_MAX_PARALLEL_COMPILES_ENV`] before the -/// embedded service starts, or `None` to leave an explicit value alone. -/// -/// Unset, zccache caps compiles at `cores - 1` unless it sees a CI marker, -/// and the daemon's scrubbed environment never carries one — so every build, -/// CI included, left a core idle (3 of 4 on a GitHub runner). The cap is the -/// one global governor across concurrent builds; it defaults to every core -/// because zccache already runs compiles at low priority on interactive hosts. -pub fn compile_cap_env_value( - zccache_value: Option<&str>, - fbuild_value: Option<&str>, - cores: usize, -) -> Option { - if zccache_value.is_some() { - return None; - } - let requested = fbuild_value - .map(str::trim) - .filter(|value| value.eq_ignore_ascii_case("unlimited") || value.parse::().is_ok()); - Some(requested.map_or_else(|| cores.max(1).to_string(), str::to_string)) -} - impl FbuildZccacheService { /// Start the embedded service on the caller's tokio runtime. /// @@ -420,51 +390,3 @@ fn synthetic_audit_id() -> String { } hex } - -#[cfg(test)] -mod compile_cap_tests { - use super::compile_cap_env_value; - - #[test] - fn defaults_to_every_core() { - assert_eq!(compile_cap_env_value(None, None, 4).as_deref(), Some("4")); - } - - #[test] - fn never_exports_zero_cores() { - assert_eq!(compile_cap_env_value(None, None, 0).as_deref(), Some("1")); - } - - #[test] - fn explicit_zccache_value_is_left_alone() { - assert_eq!(compile_cap_env_value(Some("2"), Some("6"), 4), None); - } - - #[test] - fn fbuild_override_wins_over_the_default() { - assert_eq!( - compile_cap_env_value(None, Some(" 6 "), 4).as_deref(), - Some("6") - ); - assert_eq!( - compile_cap_env_value(None, Some("Unlimited"), 4).as_deref(), - Some("Unlimited") - ); - assert_eq!( - compile_cap_env_value(None, Some("0"), 4).as_deref(), - Some("0") - ); - } - - #[test] - fn invalid_fbuild_override_falls_back_to_every_core() { - assert_eq!( - compile_cap_env_value(None, Some("lots"), 4).as_deref(), - Some("4") - ); - assert_eq!( - compile_cap_env_value(None, Some(""), 4).as_deref(), - Some("4") - ); - } -} diff --git a/crates/fbuild-build-esp/Cargo.toml b/crates/fbuild-build-esp/Cargo.toml index 1b84bb59..2d41c67d 100644 --- a/crates/fbuild-build-esp/Cargo.toml +++ b/crates/fbuild-build-esp/Cargo.toml @@ -21,7 +21,6 @@ serde_json = { workspace = true } thiserror = { workspace = true } tracing = { workspace = true } async-trait = { workspace = true } -futures = { workspace = true } regex = { workspace = true } walkdir = { workspace = true } sha2 = { workspace = true } @@ -35,7 +34,7 @@ tar = { workspace = true } zip = { workspace = true } tree-sitter = { workspace = true } tree-sitter-cpp = { workspace = true } -zccache = { git = "https://github.com/zackees/zccache", rev = "b385a7bfb308c2d1456613622ad361eb66d31cdd" } +zccache = { git = "https://github.com/zackees/zccache", rev = "3f9d4e66fe3675cd48060bca03b62b1f5a8112eb" } [dev-dependencies] filetime = { workspace = true } diff --git a/crates/fbuild-build-esp/src/esp32/esp32_linker.rs b/crates/fbuild-build-esp/src/esp32/esp32_linker.rs index b685d6b6..66a26b0b 100644 --- a/crates/fbuild-build-esp/src/esp32/esp32_linker.rs +++ b/crates/fbuild-build-esp/src/esp32/esp32_linker.rs @@ -49,12 +49,6 @@ pub fn f_flash_to_esptool_freq(f_flash: Option<&str>, default_freq: &str) -> Str } } -/// GNU ld matches the SDK linker scripts' section wildcards with `fnmatch`, -/// which under a UTF-8 locale converts every pattern and name to wide -/// characters: a third of an ESP32-S3 link. The C locale matches the same -/// (ASCII) names and produces a byte-identical ELF (FastLED/fbuild#1537). -pub(crate) const LINK_ENV: &[(&str, &str)] = &[("LC_ALL", "C")]; - /// Build the argv for an esptool `elf2image` invocation. /// /// When esptool was provisioned, the standalone binary is invoked directly; @@ -492,10 +486,10 @@ impl Linker for Esp32Linker { ) .await?; let rsp_args = [link_args[0].as_str(), &format!("@{}", rsp_path.display())]; - run_command(&rsp_args, None, Some(LINK_ENV), link_timeout).await? + run_command(&rsp_args, None, None, link_timeout).await? } else { let args_ref: Vec<&str> = link_args.iter().map(|s| s.as_str()).collect(); - run_command(&args_ref, None, Some(LINK_ENV), link_timeout).await? + run_command(&args_ref, None, None, link_timeout).await? }; if !result.success() { diff --git a/crates/fbuild-build-esp/src/esp32/esp32_linker_tests.rs b/crates/fbuild-build-esp/src/esp32/esp32_linker_tests.rs index 9cf29594..695e6d6f 100644 --- a/crates/fbuild-build-esp/src/esp32/esp32_linker_tests.rs +++ b/crates/fbuild-build-esp/src/esp32/esp32_linker_tests.rs @@ -399,36 +399,3 @@ fn test_esp32h2_flash_freq_not_64m() { let freq = f_flash_to_esptool_freq(Some("64000000L"), config.default_flash_freq()); assert_eq!(freq, "48m"); } - -/// The link runs under the C locale; see [`super::LINK_ENV`]. -#[tokio::test] -async fn link_runs_the_linker_under_the_c_locale() { - if fbuild_core::platform::host::is_windows() { - return; - } - let tmp = tempfile::TempDir::new().unwrap(); - let recorded = tmp.path().join("lc_all"); - let fake_gcc = tmp.path().join("fake-gcc"); - // Staged and renamed into place: exec'ing a file another thread's fork - // may still hold open for writing fails with ETXTBSY. - let staging = tmp.path().join("fake-gcc.staging"); - std::fs::write( - &staging, - format!( - "#!/bin/sh\nprintf '%s' \"$LC_ALL\" > '{}'\n", - recorded.display() - ), - ) - .unwrap(); - fbuild_core::platform::fs::set_executable(&staging).unwrap(); - std::fs::rename(&staging, &fake_gcc).unwrap(); - let mut linker = test_linker("esp32s3"); - linker.gcc_path = fake_gcc; - - linker - .link(&[], &[], &tmp.path().join("out"), &LinkExtraArgs::default()) - .await - .unwrap(); - - assert_eq!(std::fs::read_to_string(&recorded).unwrap(), "C"); -} diff --git a/crates/fbuild-build-esp/src/esp32/orchestrator/README.md b/crates/fbuild-build-esp/src/esp32/orchestrator/README.md index fe176d8c..daca72b0 100644 --- a/crates/fbuild-build-esp/src/esp32/orchestrator/README.md +++ b/crates/fbuild-build-esp/src/esp32/orchestrator/README.md @@ -8,10 +8,8 @@ exceeds the 1000-LOC gate. | `mod.rs` | Module root; exposes `Esp32Orchestrator` and the small public helpers (`create`, `is_esp32_project`, `cdc_on_boot_enabled`, `warn_if_cdc_on_boot`). | | `build.rs` | `impl BuildOrchestrator for Esp32Orchestrator`. Top-level phase wiring. | | `packages.rs` | pioarduino platform / framework / toolchain resolution. | -| `job_pool.rs` | The build's single compile job pool (FastLED/fbuild#1559): one `tokio::join!` of lib_deps, framework libs, project-as-library, core + sketch, local libs and boot artifacts on one job gate; `link_order` keeps the archive order. Tests in `job_pool_tests.rs`. | -| `compile_phases.rs` | Core + sketch compile on the caller's shared job gate. | -| `framework_libs.rs` | Built-in Arduino libraries shipped with the framework: sync prepare (cache hits, failure skips), concurrent compile on the shared gate, in-order finish. Tests in `framework_libs_tests.rs`. | -| `local_libs.rs` | Libraries from the project's `lib/` directory, and the project-as-library. | +| `framework_libs.rs` | Compiles built-in Arduino libraries shipped with the framework. | +| `local_libs.rs` | Compiles libraries from the project's `lib/` directory. | | `embed.rs` | `objcopy --input-target binary` conversion of embedded files. | | `embed_stage.rs` | `.lnk` resolution and target selection wrapper around `embed`. | | `boot_artifacts.rs` | Produces `bootloader.bin`, `partitions.bin`, `boot_app0.bin`. | diff --git a/crates/fbuild-build-esp/src/esp32/orchestrator/boot_artifacts.rs b/crates/fbuild-build-esp/src/esp32/orchestrator/boot_artifacts.rs index 30fe2447..48300e0a 100644 --- a/crates/fbuild-build-esp/src/esp32/orchestrator/boot_artifacts.rs +++ b/crates/fbuild-build-esp/src/esp32/orchestrator/boot_artifacts.rs @@ -24,12 +24,14 @@ pub(super) async fn prepare_boot_artifacts( flash_freq: &str, esptool_bin: Option<&Path>, caller_path: Option<&str>, -) -> Result { + perf: &mut crate::perf_log::PerfTimer, +) -> Result<()> { // FastLED/fbuild#1219: bare-name tool spawns below (`esptool` fallback, // python interpreter) resolve against the CLI caller's PATH when it was // forwarded; `None` keeps the daemon's spawn-time env. let spawn_env: Option> = caller_path.map(|p| vec![("PATH", p)]); let boot_artifacts_started = Instant::now(); + perf.checkpoint("boot-artifacts-start"); // SDK directory selector matching the chip's ROM revision (e.g. `esp32p4_es` // for ESP32-P4 eco0–eco2). The bootloader ELF must come from the same SDK // variant the app is linked against, or the ROM jumps into an illegal @@ -147,12 +149,9 @@ pub(super) async fn prepare_boot_artifacts( // `python` doesn't exist on modern distros (ubuntu 24.04 ships // only `python3`); resolve the interpreter the same way the // extra_scripts runtime does. - let python = match managed_python(esptool_bin) { - Some(python) => vec![python.to_string_lossy().into_owned()], - None => crate::script_runtime::find_python_with_path(caller_path) - .await - .unwrap_or_else(|| vec!["python".to_string()]), - }; + let python = crate::script_runtime::find_python_with_path(caller_path) + .await + .unwrap_or_else(|| vec!["python".to_string()]); let mut args: Vec<&str> = python.iter().map(|s| s.as_str()).collect(); args.extend([ gen_tool_str.as_ref(), @@ -193,20 +192,9 @@ pub(super) async fn prepare_boot_artifacts( std::fs::copy(&boot_app0_src, &boot_app0_dst)?; tracing::info!("copied boot_app0.bin"); } - Ok(boot_artifacts_started.elapsed()) -} - -/// The interpreter inside the managed esptool's environment. fbuild installed -/// and verified it, and `gen_esp32part.py` needs only the standard library, so -/// it wins over PATH's `python`: a broken alias there (the Windows Store stub, -/// a session-bound shim) silently left the build without `partitions.bin`, -/// which kept the warm fast path from ever persisting (FastLED/fbuild#1542). -fn managed_python(esptool_bin: Option<&Path>) -> Option { - let dir = esptool_bin?.parent()?; - ["python3", "python", "python.exe"] - .into_iter() - .map(|name| NormalizedPath::new(dir.join(name))) - .find(|candidate| candidate.as_path().is_file()) + perf.record("boot-artifacts", boot_artifacts_started.elapsed()); + perf.checkpoint("boot-artifacts-finish"); + Ok(()) } /// Size of the app partition the firmware is flashed into, which is what an @@ -303,25 +291,6 @@ fn resolve_partitions_csv( mod tests { use super::*; - #[test] - fn managed_python_is_the_interpreter_beside_esptool() { - let tmp = tempfile::TempDir::new().unwrap(); - let esptool = tmp.path().join("esptool.py"); - std::fs::write(&esptool, "").unwrap(); - assert_eq!(managed_python(Some(&esptool)), None); - - std::fs::write(tmp.path().join("python3"), "").unwrap(); - assert_eq!( - managed_python(Some(&esptool)), - Some(NormalizedPath::new(tmp.path().join("python3"))) - ); - } - - #[test] - fn no_managed_esptool_means_no_managed_python() { - assert_eq!(managed_python(None), None); - } - /// Arduino-ESP32's `default.csv`: two OTA slots, no factory. const DEFAULT_CSV: &str = "# Name, Type, SubType, Offset, Size, Flags nvs, data, nvs, 0x9000, 0x5000, diff --git a/crates/fbuild-build-esp/src/esp32/orchestrator/build.rs b/crates/fbuild-build-esp/src/esp32/orchestrator/build.rs index e212b3ce..dcedf4e0 100644 --- a/crates/fbuild-build-esp/src/esp32/orchestrator/build.rs +++ b/crates/fbuild-build-esp/src/esp32/orchestrator/build.rs @@ -17,10 +17,10 @@ use super::boot_artifacts::prepare_boot_artifacts; use super::cdc::warn_if_cdc_on_boot; use super::embed_stage::stage_embed_files; use super::fingerprint::Esp32FingerprintMetadata; -use super::framework_libs::prepare_framework_libs; +use super::framework_libs::compile_framework_builtin_libs; use super::helpers::{ - SdkIncludeFarm, apply_effective_define_flags, compile_db_is_current, compile_include_dirs, - framework_macro_prefix_map, profile_label, reject_unsupported_sdkconfig_overlay, + apply_effective_define_flags, compile_db_is_current, framework_macro_prefix_map, profile_label, + reject_unsupported_sdkconfig_overlay, }; use super::local_libs::compile_local_libraries; use super::packages::resolve_pioarduino_packages; @@ -30,6 +30,7 @@ use crate::build_fingerprint::{ expected_fast_path_artifacts, stable_hash_with_build_config, }; use crate::compiler::Compiler as _; +use crate::flag_overlay::apply_overlay_flags; use crate::linker::LinkerScripts; use crate::{BuildOrchestrator, BuildParams, BuildResult, SourceScanner}; @@ -123,6 +124,9 @@ impl BuildOrchestrator for Esp32Orchestrator { mcu_config.disable_lto(); } + let mut user_flags = sdk_defines; + let user_build_flags = ctx.config.get_build_flags(¶ms.env_name)?; + user_flags.extend(user_build_flags.clone()); let embed_files = ctx.config.get_embed_files(¶ms.env_name)?; let embed_txtfiles = ctx.config.get_embed_txtfiles(¶ms.env_name)?; @@ -159,7 +163,6 @@ impl BuildOrchestrator for Esp32Orchestrator { crate::eh_frame_policy::EhFramePolicy::Strip => "strip", crate::eh_frame_policy::EhFramePolicy::Preserve => "preserve", }, - sdk_defines, }, &ctx, )?; @@ -225,12 +228,7 @@ impl BuildOrchestrator for Esp32Orchestrator { // FastLED/fbuild#800: the `zccache start` daemon-spawn was deleted. // The embedded service is part of fbuild-daemon's own lifecycle. - // Toolchain and framework install independently (FastLED/fbuild#1559). - let (toolchain_dir, framework_dir) = tokio::join!( - fbuild_packages::Package::ensure_installed(&toolchain), - fbuild_packages::Package::ensure_installed(&framework) - ); - let toolchain_dir = toolchain_dir?; + let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain).await?; tracing::info!( "ESP32 {} toolchain at {}", if mcu_config.is_riscv() { @@ -241,7 +239,7 @@ impl BuildOrchestrator for Esp32Orchestrator { toolchain_dir.display() ); - let framework_dir = framework_dir?; + let framework_dir = fbuild_packages::Package::ensure_installed(&framework).await?; tracing::info!("ESP32 framework at {}", framework_dir.display()); let tc_label = format!("{}gcc", mcu_config.toolchain_prefix()); @@ -263,11 +261,8 @@ impl BuildOrchestrator for Esp32Orchestrator { include_dirs.push(variant_dir.clone()); } // Add SDK include paths (294+ paths from ESP-IDF) - let sdk_block_start = include_dirs.len(); include_dirs .extend(framework.get_sdk_include_dirs(&sdk_variant, sdk_memory_type.as_deref())); - let sdk_farm = - SdkIncludeFarm::build(&include_dirs, sdk_block_start..include_dirs.len()).await; let builtin_libs_dir = framework.get_libraries_dir(); @@ -315,13 +310,13 @@ impl BuildOrchestrator for Esp32Orchestrator { let lib_ignore = ctx.config.get_lib_ignore(¶ms.env_name)?; use fbuild_packages::Toolchain; + let mut library_archives = Vec::new(); // Read user build_flags early — needed for both library and sketch compilation. // SDK defines (from flags/defines) are prepended so user flags can override them. let mut user_flags = sdk_defines.clone(); // Before the user's build_flags, so their own prefix maps still win. user_flags.extend(framework_macro_prefix_map(&core_dir)); - user_flags.extend(sdk_farm.as_ref().map(SdkIncludeFarm::macro_prefix_map)); let mut user_build_flags = ctx.config.get_build_flags(¶ms.env_name)?; user_build_flags.extend(params.extra_build_flags.clone()); user_flags.extend(user_build_flags.clone()); @@ -338,35 +333,67 @@ impl BuildOrchestrator for Esp32Orchestrator { ); crate::warn_debug_build_flags(&user_build_flags); - // External libraries are planned before their framework dependencies - // are selected. Give their compilation all bundled include roots, then - // seed the later LDF pass with their source files to select only the - // archives that external code actually reaches. Nothing compiles here: - // the plan joins the build's one job pool below (FastLED/fbuild#1559). + // External libraries compile before their framework dependencies are + // selected. Give that compilation all bundled include roots, then seed + // the later LDF pass with its source files to select only the archives + // that external code actually reaches. let mut external_base_includes = include_dirs.clone(); external_base_includes.extend( framework_libraries .iter() .flat_map(|library| library.include_dirs.iter().cloned()), ); - let external_compile_includes = - compile_include_dirs(sdk_farm.as_ref(), &external_base_includes); let mut external_library_sources = Vec::new(); - let mut lib_deps_plan = None; if !lib_deps.is_empty() { - let resolved = super::helpers::resolve_lib_deps( - params, + let libs_dir = build_dir.join("libs"); + + // Build compiler to get flags for library compilation + let mut defines = ctx.board.get_defines(); + defines.extend(mcu_config.defines_map()); + + let temp_compiler = Esp32Compiler::with_temp_dir( + toolchain.get_gcc_path(), + toolchain.get_gxx_path(), + mcu_config.clone(), + &ctx.board.f_cpu, + defines.clone(), + external_base_includes.clone(), + params.profile, + params.verbose, + build_dir.join("tmp"), + ) + .with_build_unflags(ctx.build_unflags.clone()) + .with_eh_frame_policy(eh_frame_policy); + // Apply user build_flags to library compilation (matching PlatformIO behavior). + // User flags like -std=gnu++2a replace the MCU config's -std=gnu++2b. + let c_flags = apply_overlay_flags(&temp_compiler.c_flags(), &user_overlay, "dummy.c"); + let cpp_flags = + apply_overlay_flags(&temp_compiler.cpp_flags(), &user_overlay, "dummy.cpp"); + + let jobs = crate::parallel::effective_jobs(params.jobs); + // Use gcc-ar for LTO archives so the linker-plugin index is written. + let dep_ar_path = toolchain.get_ar_path(); + let dep_gcc_ar_path = toolchain.get_gcc_ar_path(); + let dep_lib_ar_path = crate::pipeline::pick_archiver( + &dep_ar_path, + &dep_gcc_ar_path, + &c_flags, + &cpp_flags, + ); + let lib_result = fbuild_packages::library::library_manager::ensure_libraries( &lib_deps, &lib_ignore, - &toolchain, - &mcu_config, - &ctx.board, - &ctx.build_unflags, - eh_frame_policy, - &external_compile_includes, - &user_overlay, - build_dir, + &toolchain.get_gcc_path(), + &toolchain.get_gxx_path(), + dep_lib_ar_path, + &c_flags, + &cpp_flags, + &external_base_includes, + ¶ms.project_dir, + &libs_dir, + params.verbose, + jobs, compiler_cache.as_deref(), ) .await?; @@ -378,16 +405,17 @@ impl BuildOrchestrator for Esp32Orchestrator { // `-I` flags and changes each TU's zccache context key, defeating // cross-project cache hits. Library includes are same-tier, so a // stable sort is safe for include resolution. - external_library_sources = resolved.source_files; - let mut lib_include_dirs = resolved.include_dirs; + external_library_sources = lib_result.source_files; + let mut lib_include_dirs = lib_result.include_dirs; lib_include_dirs.sort(); include_dirs.extend(lib_include_dirs); + library_archives = lib_result.archives; + tracing::info!( - "libraries: {} planned archives, {} include dirs", - resolved.plan.archive_names().len(), + "libraries: {} archives, {} new include dirs", + library_archives.len(), include_dirs.len() ); - lib_deps_plan = Some(resolved.plan); } // Unlike section GC, Arduino's Matter archive can retain global roots @@ -427,10 +455,107 @@ impl BuildOrchestrator for Esp32Orchestrator { selected_framework_libraries.len() ); include_dirs.extend(framework_selection.include_dirs); - let compile_includes = compile_include_dirs(sdk_farm.as_ref(), &include_dirs); + + // 8.5b. Project-as-library compilation — shared with sequential pipeline. + // When the project root contains library.json or library.properties (e.g., FastLED), + // the project's own src/ directory is compiled as a library archive so that example + // sketches can link against it. Centralized in pipeline::compile_project_as_library + // so every orchestrator gets this behavior architecturally. + if !params.compiledb_only { + // Build temp compiler to get the actual c_flags/cpp_flags ESP32 uses for + // library compilation. SDK defines + user flags must be applied so the + // archive matches what sketch sources see. + let mut p_defines = ctx.board.get_defines(); + p_defines.extend(mcu_config.defines_map()); + let p_compiler = Esp32Compiler::with_temp_dir( + toolchain.get_gcc_path(), + toolchain.get_gxx_path(), + mcu_config.clone(), + &ctx.board.f_cpu, + p_defines, + include_dirs.clone(), + params.profile, + params.verbose, + build_dir.join("tmp"), + ) + .with_build_unflags(ctx.build_unflags.clone()) + .with_eh_frame_policy(eh_frame_policy); + let p_c_flags = apply_overlay_flags(&p_compiler.c_flags(), &src_overlay, "dummy.c"); + let p_cpp_flags = + apply_overlay_flags(&p_compiler.cpp_flags(), &src_overlay, "dummy.cpp"); + + // Collect lib/* names so the helper can detect collisions with project-as-library. + let mut existing_lib_names = std::collections::HashSet::new(); + let local_lib_dir = params.project_dir.join("lib"); + if local_lib_dir.is_dir() { + if let Ok(entries) = std::fs::read_dir(&local_lib_dir) { + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + if let Some(name) = path.file_name().and_then(|n| n.to_str()) { + existing_lib_names.insert(name.to_lowercase()); + } + } + } + } + } + + let gcc_path = toolchain.get_gcc_path(); + let gxx_path = toolchain.get_gxx_path(); + let ar_path = toolchain.get_ar_path(); + let gcc_ar_path = toolchain.get_gcc_ar_path(); + // Use gcc-ar for LTO archives so the linker-plugin index is written. + let lib_ar_path = + crate::pipeline::pick_archiver(&ar_path, &gcc_ar_path, &p_c_flags, &p_cpp_flags); + let lib_env = crate::pipeline::LibraryBuildEnv { + gcc_path: &gcc_path, + gxx_path: &gxx_path, + ar_path: lib_ar_path, + c_flags: &p_c_flags, + cpp_flags: &p_cpp_flags, + include_dirs: &include_dirs, + verbose: params.verbose, + jobs: crate::parallel::effective_jobs(params.jobs), + compiler_cache: compiler_cache.as_deref(), + }; + if let Some(archive) = crate::pipeline::compile_project_as_library( + ¶ms.project_dir, + &ctx.src_dir, + build_dir, + &lib_env, + &existing_lib_names, + ) + .await? + { + library_archives.push(archive); + } + } tracing::info!("include paths: {} total", include_dirs.len()); + // 8.6. Compile the active LDF-selected framework libraries. + // Skip when only generating compile_commands.json. + if !params.compiledb_only { + compile_framework_builtin_libs( + params, + &mut perf, + &framework, + &toolchain, + &mcu_config, + &ctx.board, + &ctx.build_unflags, + eh_frame_policy, + &include_dirs, + &user_overlay, + build_dir, + compiler_cache.as_deref(), + &selected_framework_libraries, + &mut library_archives, + &mut ctx.build_log, + ) + .await?; + } + // 9. Scan sources let sources = { let _g = perf.phase("scan-sources"); @@ -473,7 +598,7 @@ impl BuildOrchestrator for Esp32Orchestrator { mcu_config.clone(), &ctx.board.f_cpu, defines, - compile_includes.clone(), + include_dirs.clone(), params.profile, params.verbose, build_dir.join("tmp"), @@ -535,50 +660,6 @@ impl BuildOrchestrator for Esp32Orchestrator { }); } - // 8.5b. Project-as-library: when the project root carries library.json - // or library.properties (e.g. FastLED), its src/ is compiled as an - // archive that example sketches link against. Shared with the - // sequential pipeline via pipeline::compile_project_as_library. - let project_library = super::local_libs::ProjectLibrary::prepare( - params, - &toolchain, - &mcu_config, - &ctx.board, - &ctx.build_unflags, - eh_frame_policy, - &compile_includes, - &src_overlay, - build_dir, - ); - - // 8.6. The active LDF-selected framework libraries: cache hits now, - // compiles in the job pool. One a lib_deps or project library already - // provides is skipped. - let already_compiled = super::job_pool::already_compiled( - lib_deps_plan - .as_ref() - .map(|plan| plan.archive_names()) - .unwrap_or_default(), - project_library.name(¶ms.project_dir, &ctx.src_dir), - ); - let fw_libs = prepare_framework_libs( - params, - &mut perf, - &framework, - &toolchain, - &mcu_config, - &ctx.board, - &ctx.build_unflags, - eh_frame_policy, - &compile_includes, - &user_overlay, - build_dir, - compiler_cache.as_deref(), - &selected_framework_libraries, - &already_compiled, - &mut ctx.build_log, - )?; - // Compile core + variant sources in parallel // `--clean` resets only the project build directory. Framework core // objects are content-addressed global artifacts, so hydrate them even @@ -657,76 +738,18 @@ impl BuildOrchestrator for Esp32Orchestrator { ), } } - // Every compile of the build runs now, in one job pool: lib_deps, - // framework libraries, project-as-library, core + sketch and local - // lib/ libraries all draw from one gate (FastLED/fbuild#1559; see - // `job_pool`). Boot artifacts depend on nothing compiled, so they are - // staged alongside. Only the link waits. - let gate = fbuild_packages::library::library_compiler::job_gate(jobs); - let local_libraries = crate::framework_libs::select_local_libraries( - ¶ms.project_dir, - &ctx.src_dir, - &declared_lib_deps, - ); - let library_jobs = super::job_pool::LibraryJobs { - lib_deps: async { - match lib_deps_plan { - Some(plan) => plan.compile(&gate).await, - None => Ok(Vec::new()), - } - }, - project_library: project_library.compile( - ¶ms.project_dir, - &ctx.src_dir, - build_dir, - &toolchain, - &compile_includes, - params.verbose, - compiler_cache.as_deref(), - &gate, - ), - local_libs: compile_local_libraries( - &local_libraries, - build_dir, + let core_result = { + let _g = perf.phase("compile-core-variant"); + crate::parallel::compile_sources_parallel( &compiler, - &toolchain, - &compile_includes, - &src_overlay, - &gate, - params.verbose, - compiler_cache.as_deref(), - ), - boot_artifacts: prepare_boot_artifacts( - build_dir, - ¶ms.project_dir, - &framework, - &ctx.board, - &mcu_config, - &flash_freq, - esptool_bin.as_ref().map(|path| path.as_path()), - params.caller_path.as_deref(), - ), + &all_core_sources, + core_build_dir, + &user_overlay, + jobs, + Some(&build_log_mutex), + ) + .await? }; - let pool = super::job_pool::run( - &mut perf, - &gate, - &compiler, - super::compile_phases::CompileTarget { - sources: &all_core_sources, - build_dir: core_build_dir, - overlay: &user_overlay, - }, - super::compile_phases::CompileTarget { - sources: &sources.sketch_sources, - build_dir: src_build_dir, - overlay: &src_overlay, - }, - &build_log_mutex, - &fw_libs, - library_jobs, - ) - .await?; - let (core_result, sketch_result) = (pool.core, pool.sketch); { let _g = perf.phase("core-cache-store"); let outcome = core_cache.store(core_build_dir); @@ -754,6 +777,20 @@ impl BuildOrchestrator for Esp32Orchestrator { } } + // Compile sketch sources in parallel + let sketch_result = { + let _g = perf.phase("compile-sketch"); + crate::parallel::compile_sources_parallel( + &compiler, + &sources.sketch_sources, + src_build_dir, + &src_overlay, + jobs, + Some(&build_log_mutex), + ) + .await? + }; + // Unwrap build log and flush collected warnings let mut build_log = build_log_mutex .into_inner() @@ -761,17 +798,34 @@ impl BuildOrchestrator for Esp32Orchestrator { for w in core_result.warnings.iter().chain(&sketch_result.warnings) { crate::build_output::collect_warnings(w, &mut build_log); } - let framework_archives = fw_libs.finish(pool.framework_libs, &mut perf, &mut build_log); - let library_archives = super::job_pool::link_order( - pool.lib_deps, - pool.project_library, - framework_archives, - pool.local_libs, - ); let core_objects = core_result.objects; let mut sketch_objects = sketch_result.objects; + // Compile local libraries from the project's lib/ directory. + // PlatformIO discovers and compiles these automatically. + { + let _g = perf.phase("compile-local-libs"); + let local_libraries = crate::framework_libs::select_local_libraries( + ¶ms.project_dir, + &ctx.src_dir, + &declared_lib_deps, + ); + compile_local_libraries( + &local_libraries, + build_dir, + &compiler, + &toolchain, + &include_dirs, + &src_overlay, + jobs, + params.verbose, + compiler_cache.as_deref(), + &mut library_archives, + ) + .await?; + } + // 11.5. Process embedded files (board_build.embed_files + embed_txtfiles) // // `.lnk` entries are pre-resolved: each `.lnk` is parsed, its blob is @@ -886,6 +940,20 @@ impl BuildOrchestrator for Esp32Orchestrator { .await? }; + // 14. Prepare boot artifacts for deployment / emulation + prepare_boot_artifacts( + build_dir, + ¶ms.project_dir, + &framework, + &ctx.board, + &mcu_config, + &flash_freq, + esptool_bin.as_ref().map(|path| path.as_path()), + params.caller_path.as_deref(), + &mut perf, + ) + .await?; + // 15. Size reporting + result assembly let fingerprint_started = Instant::now(); perf.checkpoint("fingerprint-save-start"); diff --git a/crates/fbuild-build-esp/src/esp32/orchestrator/compile_phases.rs b/crates/fbuild-build-esp/src/esp32/orchestrator/compile_phases.rs deleted file mode 100644 index 285536c4..00000000 --- a/crates/fbuild-build-esp/src/esp32/orchestrator/compile_phases.rs +++ /dev/null @@ -1,108 +0,0 @@ -//! Concurrent compile phases for the ESP32 orchestrator. -//! -//! The framework core and the sketch are independent source sets that link -//! together, so nothing about one depends on the other having finished. They -//! are compiled against the build's single shared job gate here rather than -//! one after the other, which is what this module exists to hold -//! (FastLED/fbuild#1537, FastLED/fbuild#1559). - -use std::path::Path; -use std::sync::{Arc, Mutex}; -use std::time::Instant; - -use fbuild_core::{BuildLog, Result}; -use tokio::sync::Semaphore; - -use crate::compiler::Compiler; -use crate::flag_overlay::LanguageExtraFlags; -use crate::parallel::{ParallelCompileResult, compile_sources_parallel_shared}; -use crate::perf_log::PerfTimer; - -/// One source set to compile, with the build dir and flag overlay it needs. -/// -/// Generic over the source element so callers pass their existing `PathBuf` -/// vectors straight through. `dylints/ban_std_pathbuf` denies naming -/// `std::path::PathBuf` in a new file while the compile engine's API takes -/// exactly that, so the slices are materialised at the call below (where the -/// element type is inferred) instead of in this signature. -pub(super) struct CompileTarget<'a, S: AsRef> { - /// Sources to compile. - pub sources: &'a [S], - /// Directory the objects are written to. - pub build_dir: &'a Path, - /// Per-language extra flags for this source set. - pub overlay: &'a LanguageExtraFlags, -} - -/// Compile the framework core sources and the sketch sources concurrently, -/// every translation unit drawing a permit from the caller's `gate`. -/// -/// Returns `(core, sketch)`. The sketch is submitted first, so its single -/// translation unit starts alongside the core fan-out instead of being -/// scheduled after all of it. The gate is the build's one job budget, shared -/// with every other compile running at the same time (libraries included), so -/// the whole region stays at the gate's permit count. -/// -/// One [`PerfTimer::phase`] guard covers the region both phases share, so -/// `compile-core-variant` spans it; the sketch's own span is recorded -/// separately, so the two entries deliberately overlap. -pub(super) async fn compile_core_and_sketch( - compiler: &(dyn Compiler + Send + Sync), - perf: &mut PerfTimer, - gate: &Arc, - core: CompileTarget<'_, S>, - sketch: CompileTarget<'_, T>, - build_log: &Mutex, -) -> Result<(ParallelCompileResult, ParallelCompileResult)> -where - S: AsRef + Send + Sync, - T: AsRef + Send + Sync, -{ - let sketch_started = Instant::now(); - - let sketch_fut = async { - let paths: Vec<_> = sketch - .sources - .iter() - .map(|s| s.as_ref().to_path_buf()) - .collect(); - let result = compile_sources_parallel_shared( - compiler, - &paths, - sketch.build_dir, - sketch.overlay, - gate, - Some(build_log), - None, - ) - .await; - (result, sketch_started.elapsed()) - }; - let core_fut = async { - let paths: Vec<_> = core - .sources - .iter() - .map(|s| s.as_ref().to_path_buf()) - .collect(); - compile_sources_parallel_shared( - compiler, - &paths, - core.build_dir, - core.overlay, - gate, - Some(build_log), - None, - ) - .await - }; - - // `join!`, not `try_join!`: `compile_sources_parallel_shared` extends its - // borrows to `'static` and is sound only when awaited to completion. - let ((sketch_result, sketch_elapsed), core_result) = { - let _region = perf.phase("compile-core-variant"); - tokio::join!(sketch_fut, core_fut) - }; - perf.record("compile-sketch", sketch_elapsed); - - Ok((core_result?, sketch_result?)) -} diff --git a/crates/fbuild-build-esp/src/esp32/orchestrator/fingerprint.rs b/crates/fbuild-build-esp/src/esp32/orchestrator/fingerprint.rs index be101050..89825155 100644 --- a/crates/fbuild-build-esp/src/esp32/orchestrator/fingerprint.rs +++ b/crates/fbuild-build-esp/src/esp32/orchestrator/fingerprint.rs @@ -30,9 +30,6 @@ pub(super) struct Esp32FingerprintMetadata { pub max_flash: Option, pub max_ram: Option, pub eh_frame_policy: &'static str, - /// SDK `-D` flags reach every TU but come from the framework, not the - /// project config, so a change there must not replay a stale fast path. - pub sdk_defines: Vec, } #[cfg(test)] @@ -67,7 +64,6 @@ mod tests { eh_frame_policy: "preserve", toolchain_name: toolchain_name.into(), toolchain_version: toolchain_version.into(), - sdk_defines: vec!["-DESP_PLATFORM".into()], } } @@ -84,15 +80,4 @@ mod tests { assert_ne!(original, different_name); assert_ne!(original, different_version); } - - #[test] - fn sdk_defines_change_fast_path_metadata() { - let original = metadata("toolchain-xtensa-esp32s3", "8.4.0+2021r2-patch5"); - let mut without = metadata("toolchain-xtensa-esp32s3", "8.4.0+2021r2-patch5"); - without.sdk_defines.clear(); - assert_ne!( - serde_json::to_vec(&original).unwrap(), - serde_json::to_vec(&without).unwrap() - ); - } } diff --git a/crates/fbuild-build-esp/src/esp32/orchestrator/framework_library_cache.rs b/crates/fbuild-build-esp/src/esp32/orchestrator/framework_library_cache.rs index 0bc07c62..026b897f 100644 --- a/crates/fbuild-build-esp/src/esp32/orchestrator/framework_library_cache.rs +++ b/crates/fbuild-build-esp/src/esp32/orchestrator/framework_library_cache.rs @@ -34,7 +34,7 @@ impl FrameworkLibraryCache { } #[cfg(test)] - pub(super) fn with_cache_root( + fn with_cache_root( project_dir: &Path, cache_root: &Path, profile: BuildProfile, diff --git a/crates/fbuild-build-esp/src/esp32/orchestrator/framework_libs.rs b/crates/fbuild-build-esp/src/esp32/orchestrator/framework_libs.rs index e83065be..dc56e4f4 100644 --- a/crates/fbuild-build-esp/src/esp32/orchestrator/framework_libs.rs +++ b/crates/fbuild-build-esp/src/esp32/orchestrator/framework_libs.rs @@ -5,25 +5,12 @@ //! library: Matter carries global roots that pull its full networking stack into //! otherwise empty sketches. Callers must therefore pass only the libraries //! selected by the active-branch LDF resolver (FastLED/fbuild#1449). -//! -//! The work is split in three so the compiles can share the build's one job -//! pool with everything else (FastLED/fbuild#1559): -//! 1. [`prepare_framework_libs`] (sync): cache eviction/hydration, cache hits -//! and failure-marker skips; decides what still needs compiling. -//! 2. [`FwLibsPlan::compile`] (async): compiles every pending library -//! concurrently on a shared [`JobGate`]. Touches no perf/build-log state. -//! 3. [`FwLibsPlan::finish`] (sync): applies the outcomes in selected-library -//! order — cache stores, failure markers, perf checkpoints, build log — and -//! returns the archives in exactly the order the old serial loop produced. -use std::collections::HashSet; use std::path::{Path, PathBuf}; -use std::sync::Arc; -use std::time::{Duration, Instant}; +use std::time::Instant; use fbuild_core::Result; use fbuild_packages::Framework; -use fbuild_packages::library::library_compiler::{JobGate, LibCompileBackend}; use super::super::esp32_compiler::Esp32Compiler; use super::super::mcu_config::Esp32McuConfig; @@ -35,62 +22,13 @@ use super::helpers::{ use crate::BuildParams; use crate::compiler::Compiler as _; use crate::flag_overlay::{LanguageExtraFlags, apply_overlay_flags}; -use crate::perf_log::PerfTimer; - -/// Everything a framework-library compile needs, fixed before compiling. -pub(super) struct FwLibsContext { - pub build_dir: PathBuf, - pub include_dirs: Vec, - pub gcc_path: PathBuf, - pub gxx_path: PathBuf, - pub ar_path: PathBuf, - pub c_flags: Vec, - pub cpp_flags: Vec, - /// Failure-marker signature (include dirs + flags). - pub signature: String, - pub verbose: bool, - pub compiler_cache: Option, - pub compile_cwd: Option, - pub backend: Arc, - pub cache: FrameworkLibraryCache, -} - -/// One selected library that yields an archive (or may), in selection order. -enum FwLibEntry { - /// The archive is already in the build dir (built earlier or hydrated). - Cached { archive: PathBuf }, - /// Needs compiling. - Compile { - name: String, - index: usize, - sources: Vec, - marker: PathBuf, - }, -} -/// Framework libraries resolved against the cache; see the module docs. -pub(super) struct FwLibsPlan { - /// `None` when the framework ships no libraries dir (or `clean_only`). - ctx: Option, - entries: Vec, - /// Time spent in [`prepare_framework_libs`]. - prepare_elapsed: Duration, -} - -/// Result of [`FwLibsPlan::compile`]: one outcome per [`FwLibEntry::Compile`] -/// (in entry order), plus the compile's own wall time. -pub(super) struct FwLibsCompiled { - outcomes: Vec>>, - elapsed: Duration, -} - -/// Resolve the LDF-selected framework libraries for this build: flags, -/// cache hydrate, cache hits and failure skips. `already_compiled` names the -/// archives a `lib_deps` / project library provides, which are skipped. +/// Compile LDF-selected Arduino framework libraries for ESP32. +/// Library archives are appended to `library_archives`. #[allow(clippy::too_many_arguments)] -pub(super) fn prepare_framework_libs( +pub(super) async fn compile_framework_builtin_libs( params: &BuildParams, - perf: &mut PerfTimer, + perf: &mut crate::perf_log::PerfTimer, framework: &fbuild_packages::library::Esp32Framework, toolchain: &fbuild_packages::toolchain::Esp32Toolchain, mcu_config: &Esp32McuConfig, @@ -102,16 +40,18 @@ pub(super) fn prepare_framework_libs( build_dir: &Path, compiler_cache: Option<&Path>, selected_libraries: &[fbuild_packages::library::FrameworkLibrary], - already_compiled: &HashSet, + library_archives: &mut Vec, build_log: &mut fbuild_core::BuildLog, -) -> Result { +) -> Result<()> { use fbuild_packages::Toolchain; - let started = Instant::now(); + let fw_libs_started = Instant::now(); perf.checkpoint("fw-libs-start"); let builtin_libs_dir = framework.get_libraries_dir(); if !builtin_libs_dir.is_dir() { - return Ok(FwLibsPlan::empty(started.elapsed())); + perf.record("fw-libs", fw_libs_started.elapsed()); + perf.checkpoint("fw-libs-finish"); + return Ok(()); } let fw_libs_build_dir = build_dir.join("fw_libs"); @@ -125,6 +65,23 @@ pub(super) fn prepare_framework_libs( let fw_compile_cwd = fbuild_core::path::compile_cwd_from_output(&fw_libs_build_dir.join("obj").join("_probe.o")); + // FastLED/fbuild#986: route framework-lib TU compiles through the in-process + // embedded zccache service (the same path sketch/core use) so they are + // cached and hit cross-project via #985's project-independent keys, instead + // of the uncached direct-gcc fallback (`compiler_cache` is `None`). + let lib_backend: std::sync::Arc< + dyn fbuild_packages::library::library_compiler::LibCompileBackend, + > = std::sync::Arc::new(crate::compile_backend::EmbeddedLibBackend); + + // Build set of already-compiled library names + let already_compiled: std::collections::HashSet = library_archives + .iter() + .filter_map(|p| p.file_stem()) + .filter_map(|s| s.to_str()) + .filter_map(|s| s.strip_prefix("lib")) + .map(|s| s.to_string()) + .collect(); + // Get compiler flags for framework library compilation let mut fw_defines = board.get_defines(); fw_defines.extend(mcu_config.defines_map()); @@ -145,306 +102,179 @@ pub(super) fn prepare_framework_libs( let fw_c_flags = apply_overlay_flags(&fw_compiler.c_flags(), user_overlay, "dummy.c"); let fw_cpp_flags = apply_overlay_flags(&fw_compiler.cpp_flags(), user_overlay, "dummy.cpp"); let fw_signature = framework_signature(include_dirs, &fw_c_flags, &fw_cpp_flags); - let cache = FrameworkLibraryCache::new( + let framework_cache = FrameworkLibraryCache::new( ¶ms.project_dir, params.profile, &fw_signature, &builtin_libs_dir, ); - // Use gcc-ar for LTO archives so the linker-plugin index is written. - let ar_path = crate::pipeline::pick_archiver( - &toolchain.get_ar_path(), - &toolchain.get_gcc_ar_path(), - &fw_c_flags, - &fw_cpp_flags, - ) - .to_path_buf(); - let ctx = FwLibsContext { - build_dir: fw_libs_build_dir, - include_dirs: include_dirs.to_vec(), - gcc_path: toolchain.get_gcc_path(), - gxx_path: toolchain.get_gxx_path(), - ar_path, - c_flags: fw_c_flags, - cpp_flags: fw_cpp_flags, - signature: fw_signature, - verbose: params.verbose, - compiler_cache: compiler_cache.map(Path::to_path_buf), - compile_cwd: fw_compile_cwd, - // FastLED/fbuild#986: route framework-lib TU compiles through the - // in-process embedded zccache service (the same path sketch/core use) - // so they are cached and hit cross-project via #985's - // project-independent keys. - backend: Arc::new(crate::compile_backend::EmbeddedLibBackend), - cache, - }; - let mut plan = FwLibsPlan::new( - ctx, - selected_libraries, - already_compiled, - params.clean_all, - params.clean_only, - perf, - build_log, - )?; - plan.prepare_elapsed = started.elapsed(); - Ok(plan) -} - -impl FwLibsPlan { - fn empty(prepare_elapsed: Duration) -> Self { - Self { - ctx: None, - entries: Vec::new(), - prepare_elapsed, + if params.clean_all { + match framework_cache.remove() { + Ok(()) => tracing::info!("removed ESP32 framework library cache"), + Err(error) => { + tracing::warn!("failed to remove ESP32 framework library cache: {}", error) + } } } + if params.clean_only { + return Ok(()); + } + let hydrate_outcome = framework_cache.hydrate(&fw_libs_build_dir); + build_log.push(hydrate_summary(&hydrate_outcome)); + match &hydrate_outcome { + Ok(copied) if *copied > 0 => tracing::info!( + "hydrated {} cached ESP32 framework library archives", + copied + ), + Ok(_) => {} + Err(error) => tracing::warn!("failed to hydrate ESP32 framework library cache: {}", error), + } - /// Evict (`clean_all`) / hydrate the cache, then classify each selected - /// library in order: skipped, already archived, or to compile. - pub(super) fn new( - ctx: FwLibsContext, - selected_libraries: &[fbuild_packages::library::FrameworkLibrary], - already_compiled: &HashSet, - clean_all: bool, - clean_only: bool, - perf: &mut PerfTimer, - build_log: &mut fbuild_core::BuildLog, - ) -> Result { - if clean_all { - match ctx.cache.remove() { - Ok(()) => tracing::info!("removed ESP32 framework library cache"), - Err(error) => { - tracing::warn!("failed to remove ESP32 framework library cache: {}", error) - } + let mut fw_lib_stored = 0; + let mut fw_lib_count = 0; + let mut fw_lib_seen = 0; + for library in selected_libraries { + let lib_name = library.name.to_lowercase(); + if already_compiled.contains(&lib_name) { + continue; + } + + fw_lib_seen += 1; + + // Check if archive already exists + let archive_path = fw_libs_build_dir.join(format!("lib{}.a", lib_name)); + if archive_path.exists() { + if perf.is_active() { + perf.checkpoint(format!( + "fw-lib-cache-hit name={} index={}", + lib_name, fw_lib_seen + )); } + library_archives.push(archive_path); + fw_lib_count += 1; + continue; } - if clean_only { - return Ok(Self::empty(Duration::ZERO)); + + let sources = &library.source_files; + if sources.is_empty() { + continue; } - let hydrate_outcome = ctx.cache.hydrate(&ctx.build_dir); - build_log.push(hydrate_summary(&hydrate_outcome)); - match &hydrate_outcome { - Ok(copied) if *copied > 0 => tracing::info!( - "hydrated {} cached ESP32 framework library archives", - copied - ), - Ok(_) => {} - Err(error) => { - tracing::warn!("failed to hydrate ESP32 framework library cache: {}", error) + let failure_marker = framework_failure_marker(&fw_libs_build_dir, &lib_name); + if should_skip_failed_framework_lib(&failure_marker, &fw_signature, sources)? { + if perf.is_active() { + perf.checkpoint(format!( + "fw-lib-skip-failed name={} index={} sources={}", + lib_name, + fw_lib_seen, + sources.len() + )); } + tracing::debug!( + "skipping previously failed framework library '{}'", + lib_name + ); + continue; } - - let mut entries = Vec::new(); - let mut seen = 0; - for library in selected_libraries { - let lib_name = library.name.to_lowercase(); - if already_compiled.contains(&lib_name) { - continue; + if framework_cache.has_failed(&lib_name) { + if perf.is_active() { + perf.checkpoint(format!( + "fw-lib-cache-skip-failed name={} index={}", + lib_name, fw_lib_seen + )); } - seen += 1; + continue; + } - // Check if archive already exists - let archive = ctx.build_dir.join(format!("lib{}.a", lib_name)); - if archive.exists() { - if perf.is_active() { - perf.checkpoint(format!("fw-lib-cache-hit name={} index={}", lib_name, seen)); + let fw_jobs = crate::parallel::effective_jobs(params.jobs); + if perf.is_active() { + perf.checkpoint(format!( + "fw-lib-compile-start name={} index={} sources={} jobs={}", + lib_name, + fw_lib_seen, + sources.len(), + fw_jobs + )); + } + // Use gcc-ar for LTO archives so the linker-plugin index is written. + let fw_ar_path = toolchain.get_ar_path(); + let fw_gcc_ar_path = toolchain.get_gcc_ar_path(); + let fw_lib_ar_path = crate::pipeline::pick_archiver( + &fw_ar_path, + &fw_gcc_ar_path, + &fw_c_flags, + &fw_cpp_flags, + ); + match fbuild_packages::library::library_compiler::compile_library_with_jobs( + &lib_name, + sources, + include_dirs, + &toolchain.get_gcc_path(), + &toolchain.get_gxx_path(), + fw_lib_ar_path, + &fw_c_flags, + &fw_cpp_flags, + &fw_libs_build_dir, + params.verbose, + fw_jobs, + compiler_cache, + fw_compile_cwd.clone(), + Some(lib_backend.clone()), + ) + .await + { + Ok(Some(archive)) => { + let _ = std::fs::remove_file(&failure_marker); + match framework_cache.store_archive(&archive) { + Ok(()) => fw_lib_stored += 1, + Err(error) => { + tracing::warn!("failed to cache framework library {}: {}", lib_name, error) + } } - entries.push(FwLibEntry::Cached { archive }); - continue; - } - - let sources = &library.source_files; - if sources.is_empty() { - continue; - } - let marker = framework_failure_marker(&ctx.build_dir, &lib_name); - if should_skip_failed_framework_lib(&marker, &ctx.signature, sources)? { + library_archives.push(archive); + fw_lib_count += 1; if perf.is_active() { perf.checkpoint(format!( - "fw-lib-skip-failed name={} index={} sources={}", - lib_name, - seen, - sources.len() + "fw-lib-compile-finish name={} index={} count={}", + lib_name, fw_lib_seen, fw_lib_count )); } - tracing::debug!( - "skipping previously failed framework library '{}'", - lib_name - ); - continue; } - if ctx.cache.has_failed(&lib_name) { + Ok(None) => { if perf.is_active() { perf.checkpoint(format!( - "fw-lib-cache-skip-failed name={} index={}", - lib_name, seen + "fw-lib-header-only name={} index={}", + lib_name, fw_lib_seen )); } - continue; } - if perf.is_active() { - perf.checkpoint(format!( - "fw-lib-compile-queued name={} index={} sources={}", - lib_name, - seen, - sources.len() - )); - } - entries.push(FwLibEntry::Compile { - name: lib_name, - index: seen, - sources: sources.clone(), - marker, - }); - } - Ok(Self { - ctx: Some(ctx), - entries, - prepare_elapsed: Duration::ZERO, - }) - } - - /// Compile every pending library concurrently, each translation unit - /// drawing a permit from `gate`. A failing library does not stop the - /// others: framework-library failures are tolerated (see [`Self::finish`]). - pub(super) async fn compile(&self, gate: &JobGate) -> FwLibsCompiled { - let started = Instant::now(); - let Some(ctx) = &self.ctx else { - return FwLibsCompiled { - outcomes: Vec::new(), - elapsed: Duration::ZERO, - }; - }; - let compiles = self.entries.iter().filter_map(|entry| match entry { - FwLibEntry::Cached { .. } => None, - FwLibEntry::Compile { name, sources, .. } => Some(async move { - fbuild_packages::library::library_compiler::compile_library_gated( - name, - sources, - &ctx.include_dirs, - &ctx.gcc_path, - &ctx.gxx_path, - &ctx.ar_path, - &ctx.c_flags, - &ctx.cpp_flags, - &ctx.build_dir, - ctx.verbose, - gate, - ctx.compiler_cache.as_deref(), - ctx.compile_cwd.clone(), - Some(ctx.backend.clone()), - None, - ) - .await - }), - }); - // join_all runs every compile to completion even if one fails. - let outcomes = futures::future::join_all(compiles).await; - FwLibsCompiled { - outcomes, - elapsed: started.elapsed(), - } - } - - /// Apply compile outcomes in selected-library order and return the - /// archives to link, cache hits and fresh compiles interleaved exactly as - /// selected. Failures are recorded (failure marker + cache) and skipped: - /// some framework libraries do not build for every chip, and the linker - /// reports any symbol that was actually needed. - pub(super) fn finish( - self, - compiled: FwLibsCompiled, - perf: &mut PerfTimer, - build_log: &mut fbuild_core::BuildLog, - ) -> Vec { - let finish_started = Instant::now(); - let mut archives = Vec::new(); - let Some(ctx) = self.ctx else { - perf.record("fw-libs", self.prepare_elapsed + compiled.elapsed); - perf.checkpoint("fw-libs-finish"); - return archives; - }; - let mut outcomes = compiled.outcomes.into_iter(); - let mut stored = 0; - for entry in self.entries { - let (name, index, marker) = match entry { - FwLibEntry::Cached { archive } => { - archives.push(archive); - continue; - } - FwLibEntry::Compile { - name, - index, - marker, - .. - } => (name, index, marker), - }; - let outcome = outcomes.next().unwrap_or_else(|| { - Err(fbuild_core::FbuildError::BuildFailed( - "framework library was not compiled".into(), - )) - }); - match outcome { - Ok(Some(archive)) => { - let _ = std::fs::remove_file(&marker); - match ctx.cache.store_archive(&archive) { - Ok(()) => stored += 1, - Err(error) => { - tracing::warn!("failed to cache framework library {}: {}", name, error) - } - } - archives.push(archive); - if perf.is_active() { - perf.checkpoint(format!( - "fw-lib-compile-finish name={} index={} count={}", - name, - index, - archives.len() - )); - } - } - Ok(None) => { - if perf.is_active() { - perf.checkpoint(format!( - "fw-lib-header-only name={} index={}", - name, index - )); - } + Err(e) => { + // Non-fatal: some framework libs may fail to compile + // (e.g., platform-specific ones). The linker will report + // if any actually-needed symbols are missing. + if perf.is_active() { + perf.checkpoint(format!( + "fw-lib-compile-error name={} index={}", + lib_name, fw_lib_seen + )); } - Err(e) => { - if perf.is_active() { - perf.checkpoint(format!( - "fw-lib-compile-error name={} index={}", - name, index - )); - } - tracing::debug!("framework library {} failed to compile: {}", name, e); - record_failed_framework_lib(&marker, &ctx.signature, &e.to_string()); - if let Err(error) = ctx.cache.record_failure(&name) { - tracing::warn!( - "failed to cache framework library failure {}: {}", - name, - error - ); - } + tracing::debug!("framework library {} failed to compile: {}", lib_name, e); + record_failed_framework_lib(&failure_marker, &fw_signature, &e.to_string()); + if let Err(error) = framework_cache.record_failure(&lib_name) { + tracing::warn!( + "failed to cache framework library failure {}: {}", + lib_name, + error + ); } } } + } - if !archives.is_empty() { - tracing::info!("compiled {} framework built-in libraries", archives.len()); - } - build_log.push(store_summary(stored)); - perf.record( - "fw-libs", - self.prepare_elapsed + compiled.elapsed + finish_started.elapsed(), - ); - perf.checkpoint("fw-libs-finish"); - archives + if fw_lib_count > 0 { + tracing::info!("compiled {} framework built-in libraries", fw_lib_count); } + build_log.push(store_summary(fw_lib_stored)); + perf.record("fw-libs", fw_libs_started.elapsed()); + perf.checkpoint("fw-libs-finish"); + Ok(()) } - -#[cfg(test)] -#[path = "framework_libs_tests.rs"] -pub(super) mod tests; diff --git a/crates/fbuild-build-esp/src/esp32/orchestrator/framework_libs_tests.rs b/crates/fbuild-build-esp/src/esp32/orchestrator/framework_libs_tests.rs deleted file mode 100644 index 7f10c143..00000000 --- a/crates/fbuild-build-esp/src/esp32/orchestrator/framework_libs_tests.rs +++ /dev/null @@ -1,292 +0,0 @@ -//! Tests for the framework-library prepare / compile / finish split -//! (FastLED/fbuild#1559). - -use std::path::Path; -use std::sync::atomic::{AtomicUsize, Ordering}; - -use fbuild_core::path::NormalizedPath; -use fbuild_packages::library::FrameworkLibrary; -use fbuild_packages::library::library_compiler::{LibCompileOutcome, job_gate}; - -use super::*; - -/// Install an executable script via staging + rename (avoids ETXTBSY when -/// tests run in parallel). -pub(crate) fn install_script(path: &Path, script: &str) { - let staging = path.with_extension("staging"); - std::fs::write(&staging, script).unwrap(); - fbuild_core::platform::fs::set_executable(&staging).unwrap(); - std::fs::rename(&staging, path).unwrap(); -} - -/// A fake `ar` that creates its 2nd argument (the archive). -pub(crate) fn install_fake_ar(path: &Path) { - install_script( - path, - "#!/bin/sh\n# argv: rcs \ntouch \"$2\"\n", - ); -} - -/// A [`LibCompileBackend`] that tracks in-flight compiles on a counter it may -/// share with other fakes, sleeps, writes the `-o` object, and fails sources -/// whose file name starts with `bad`. -pub(crate) struct FakeBackend { - pub in_flight: Arc, - pub max_in_flight: Arc, - pub calls: AtomicUsize, -} - -impl FakeBackend { - pub(crate) fn new(in_flight: Arc, max_in_flight: Arc) -> Self { - Self { - in_flight, - max_in_flight, - calls: AtomicUsize::new(0), - } - } -} - -#[async_trait::async_trait] -impl LibCompileBackend for FakeBackend { - async fn compile( - &self, - _compiler: &Path, - args: Vec, - cwd: NormalizedPath, - _env: Vec<(String, String)>, - ) -> Result { - self.calls.fetch_add(1, Ordering::SeqCst); - let after = |flag: &str| { - args.iter() - .position(|a| a == flag) - .and_then(|i| args.get(i + 1)) - .cloned() - .unwrap_or_default() - }; - let (source, object) = (after("-c"), after("-o")); - let now = self.in_flight.fetch_add(1, Ordering::SeqCst) + 1; - self.max_in_flight.fetch_max(now, Ordering::SeqCst); - tokio::time::sleep(Duration::from_millis(40)).await; - self.in_flight.fetch_sub(1, Ordering::SeqCst); - - let name = Path::new(&source) - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(); - if name.starts_with("bad") { - return Ok(LibCompileOutcome { - exit_code: 1, - stdout: Vec::new(), - stderr: b"boom".to_vec(), - }); - } - let object = cwd.as_path().join(object); - std::fs::create_dir_all(object.parent().unwrap()).unwrap(); - std::fs::write(&object, "obj").unwrap(); - Ok(LibCompileOutcome { - exit_code: 0, - stdout: Vec::new(), - stderr: Vec::new(), - }) - } -} - -pub(crate) struct Fixture { - pub tmp: tempfile::TempDir, - pub backend: Arc, -} - -impl Fixture { - pub(crate) fn new() -> Self { - Self::with_backend(Arc::new(FakeBackend::new( - Arc::new(AtomicUsize::new(0)), - Arc::new(AtomicUsize::new(0)), - ))) - } - - pub(crate) fn with_backend(backend: Arc) -> Self { - let tmp = tempfile::TempDir::new().unwrap(); - install_fake_ar(&tmp.path().join("ar")); - std::fs::create_dir_all(tmp.path().join("framework")).unwrap(); - std::fs::create_dir_all(tmp.path().join("src")).unwrap(); - Self { tmp, backend } - } - - pub(crate) fn build_dir(&self) -> NormalizedPath { - NormalizedPath::new(self.tmp.path().join("build").join("fw_libs")) - } - - fn ctx(&self) -> FwLibsContext { - let root = self.tmp.path(); - FwLibsContext { - build_dir: self.build_dir().into_path_buf(), - include_dirs: Vec::new(), - gcc_path: root.join("gcc"), - gxx_path: root.join("g++"), - ar_path: root.join("ar"), - c_flags: Vec::new(), - cpp_flags: Vec::new(), - signature: "sig".to_string(), - verbose: false, - compiler_cache: None, - compile_cwd: None, - backend: self.backend.clone(), - cache: FrameworkLibraryCache::with_cache_root( - &root.join("project"), - &root.join("cache"), - fbuild_core::BuildProfile::Release, - "sig", - &root.join("framework"), - ), - } - } - - /// A framework library `name` with one source per entry of `sources`. - pub(crate) fn library(&self, name: &str, sources: &[&str]) -> FrameworkLibrary { - let dir = self.tmp.path().join("src").join(name); - std::fs::create_dir_all(&dir).unwrap(); - let source_files = sources - .iter() - .map(|source| { - let path = dir.join(source); - std::fs::write(&path, "int x;").unwrap(); - path - }) - .collect(); - FrameworkLibrary { - name: name.to_string(), - dir: dir.clone(), - include_dirs: vec![dir], - source_files, - } - } - - pub(crate) fn plan( - &self, - libraries: &[FrameworkLibrary], - already_compiled: &HashSet, - log: &mut fbuild_core::BuildLog, - ) -> FwLibsPlan { - let mut perf = PerfTimer::new("fw-libs-test"); - FwLibsPlan::new( - self.ctx(), - libraries, - already_compiled, - false, - false, - &mut perf, - log, - ) - .unwrap() - } - - async fn build( - &self, - libraries: &[FrameworkLibrary], - already_compiled: &HashSet, - jobs: usize, - ) -> (Vec, Vec) { - let mut log = fbuild_core::BuildLog::new(); - let plan = self.plan(libraries, already_compiled, &mut log); - let compiled = plan.compile(&job_gate(jobs)).await; - let mut perf = PerfTimer::new("fw-libs-test"); - let archives = plan.finish(compiled, &mut perf, &mut log); - let names = archives - .iter() - .map(|archive| archive.file_name().unwrap().to_string_lossy().into_owned()) - .collect(); - (names, log.into_lines()) - } -} - -/// Cache hits and fresh compiles come back interleaved in selection order, -/// exactly as the serial loop pushed them; a failing library is recorded and -/// left out without failing the build; a header-only library and one a -/// `lib_deps` archive already provides are skipped. -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn finish_keeps_selection_order_and_tolerates_failures() { - if fbuild_core::platform::host::is_windows() { - return; - } - let fx = Fixture::new(); - let libraries = vec![ - fx.library("Alpha", &["a1.cpp", "a2.cpp"]), - fx.library("Beta", &["b.cpp"]), - fx.library("Delta", &["d.cpp", "bad.cpp"]), - fx.library("Empty", &[]), - fx.library("FastLED", &["f.cpp"]), - fx.library("Gamma", &["g.c"]), - ]; - // Beta's archive is already in the build dir (a hydrated cache hit). - std::fs::create_dir_all(fx.build_dir()).unwrap(); - std::fs::write(fx.build_dir().join("libbeta.a"), "cached").unwrap(); - let already: HashSet = ["fastled".to_string()].into(); - - let (archives, log) = fx.build(&libraries, &already, 2).await; - assert_eq!(archives, ["libalpha.a", "libbeta.a", "libgamma.a"]); - assert!( - log.iter() - .any(|l| l.ends_with("framework-libs cache: stored 2")), - "{log:?}" - ); - let marker = framework_failure_marker(fx.build_dir().as_path(), "delta"); - assert!(marker.is_file(), "failure marker recorded"); - assert!(fx.ctx().cache.has_failed("delta")); - let calls = fx.backend.calls.load(Ordering::SeqCst); - assert_eq!(calls, 2 + 2 + 1, "Alpha, Delta and Gamma compiled"); - - // Next build: everything is a cache hit or a remembered failure, so - // nothing compiles and the order is unchanged. - let (again, _) = fx.build(&libraries, &already, 2).await; - assert_eq!(again, archives); - assert_eq!(fx.backend.calls.load(Ordering::SeqCst), calls); -} - -/// Pending libraries compile concurrently: with one TU each, two in flight is -/// only possible across libraries, and the gate is never exceeded. -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn pending_libraries_share_one_gate() { - if fbuild_core::platform::host::is_windows() { - return; - } - let fx = Fixture::new(); - let libraries: Vec<_> = ["A", "B", "C", "D", "E"] - .iter() - .map(|name| fx.library(name, &["one.cpp"])) - .collect(); - - let (archives, _) = fx.build(&libraries, &HashSet::new(), 3).await; - assert_eq!(archives, ["liba.a", "libb.a", "libc.a", "libd.a", "libe.a"]); - let max = fx.backend.max_in_flight.load(Ordering::SeqCst); - assert!((2..=3).contains(&max), "max in-flight {max} with 3 permits"); -} - -/// `clean_only` plans nothing (no hydrate); `clean_all` evicts the cache. -#[tokio::test] -async fn clean_only_plans_nothing_and_clean_all_evicts() { - let fx = Fixture::new(); - let archive = fx.tmp.path().join("libalpha.a"); - std::fs::write(&archive, "a").unwrap(); - fx.ctx().cache.store_archive(&archive).unwrap(); - let libraries = vec![fx.library("Alpha", &["a.cpp"])]; - - let mut log = fbuild_core::BuildLog::new(); - let mut perf = PerfTimer::new("fw-libs-test"); - let plan = FwLibsPlan::new( - fx.ctx(), - &libraries, - &HashSet::new(), - true, - true, - &mut perf, - &mut log, - ) - .unwrap(); - assert!(plan.ctx.is_none() && plan.entries.is_empty()); - assert!(log.is_empty(), "clean_only must not hydrate"); - assert_eq!(fx.ctx().cache.hydrate(&fx.build_dir()).unwrap(), 0); - let compiled = plan.compile(&job_gate(1)).await; - assert!(plan.finish(compiled, &mut perf, &mut log).is_empty()); - assert_eq!(fx.backend.calls.load(Ordering::SeqCst), 0); -} diff --git a/crates/fbuild-build-esp/src/esp32/orchestrator/helpers.rs b/crates/fbuild-build-esp/src/esp32/orchestrator/helpers.rs index d03666a6..25f30fbd 100644 --- a/crates/fbuild-build-esp/src/esp32/orchestrator/helpers.rs +++ b/crates/fbuild-build-esp/src/esp32/orchestrator/helpers.rs @@ -219,90 +219,6 @@ pub(super) fn compile_db_is_current(build_dir: &Path, project_dir: &Path) -> boo crate::compile_database::CompileDatabase::expected_output_path(build_dir, project_dir).exists() } -/// The SDK `-I` block collapsed into a header farm for compiler argv -/// (FastLED/fbuild#1537). Library selection keeps the original list. -pub(super) struct SdkIncludeFarm { - range: std::ops::Range, - block: Vec, - farm: crate::include_farm::IncludeFarm, -} - -impl SdkIncludeFarm { - /// Farm `dirs[range]`, the SDK block; `None` (plain `-I`) on any failure. - pub(super) async fn build(dirs: &[PathBuf], range: std::ops::Range) -> Option { - if fbuild_core::platform::host::is_windows() || range.is_empty() { - return None; - } - let before: Vec<_> = dirs[..range.start] - .iter() - .map(fbuild_core::path::NormalizedPath::new) - .collect(); - let block: Vec<_> = dirs[range.clone()] - .iter() - .map(fbuild_core::path::NormalizedPath::new) - .collect(); - let farms_root = fbuild_paths::get_cache_root().join("include-farms"); - let planned = tokio::task::spawn_blocking(move || { - crate::include_farm::ensure_farm(&farms_root, &before, &block) - }) - .await; - match planned { - Ok(Ok(farm)) => { - tracing::info!( - "SDK include farm: {} -I dirs -> {} at {}", - range.len(), - farm.kept.len() + 1, - farm.dir.display() - ); - Some(Self { - block: dirs[range.clone()].to_vec(), - range, - farm, - }) - } - Ok(Err(error)) => { - tracing::warn!("SDK include farm unavailable, keeping plain -I: {error}"); - None - } - Err(error) => { - tracing::warn!("SDK include farm task failed, keeping plain -I: {error}"); - None - } - } - } - - /// `-fmacro-prefix-map` for headers reached through the farm. - pub(super) fn macro_prefix_map(&self) -> String { - self.farm - .macro_prefix_map("framework-arduinoespressif32/sdk") - } -} - -/// The `-I` list a compiler should receive: `dirs` with the SDK block swapped -/// for the farm. Lists are only appended to after the block, so it sits at the -/// recorded range; if it does not, the list is returned unchanged. -pub(super) fn compile_include_dirs( - farm: Option<&SdkIncludeFarm>, - dirs: &[PathBuf], -) -> Vec { - let Some(farm) = farm else { - return dirs.to_vec(); - }; - if dirs.get(farm.range.clone()) != Some(farm.block.as_slice()) { - tracing::warn!("SDK include block moved; compiling with plain -I"); - return dirs.to_vec(); - } - let mut out = dirs[..farm.range.start].to_vec(); - out.extend( - farm.farm - .replacement() - .into_iter() - .map(fbuild_core::path::NormalizedPath::into_path_buf), - ); - out.extend_from_slice(&dirs[farm.range.end..]); - out -} - /// `-fmacro-prefix-map` that shortens the framework's install root in `__FILE__`. /// /// The core's log macros embed `__FILE__` in flash, so fbuild's long cache path @@ -317,72 +233,6 @@ pub(super) fn framework_macro_prefix_map(core_dir: &Path) -> Option { )) } -/// Download (or resolve) `lib_deps` and plan their compilation, without -/// compiling: library selection needs only their sources, and the compile -/// joins the build's shared job pool later (FastLED/fbuild#1559). -/// -/// `include_dirs` is every bundled include root, since external libraries are -/// planned before their framework dependencies are selected. User build_flags -/// apply to library compilation, matching PlatformIO (e.g. `-std=gnu++2a` -/// replaces the MCU config's `-std=gnu++2b`). -#[allow(clippy::too_many_arguments)] -pub(super) async fn resolve_lib_deps( - params: &crate::BuildParams, - lib_deps: &[String], - lib_ignore: &[String], - toolchain: &fbuild_packages::toolchain::Esp32Toolchain, - mcu_config: &super::super::mcu_config::Esp32McuConfig, - board: &fbuild_config::BoardConfig, - build_unflags: &[String], - eh_frame_policy: crate::eh_frame_policy::EhFramePolicy, - include_dirs: &[PathBuf], - user_overlay: &crate::flag_overlay::LanguageExtraFlags, - build_dir: &Path, - compiler_cache: Option<&Path>, -) -> Result { - use crate::compiler::Compiler as _; - use crate::flag_overlay::apply_overlay_flags; - use fbuild_packages::Toolchain; - - let mut defines = board.get_defines(); - defines.extend(mcu_config.defines_map()); - let temp_compiler = super::super::esp32_compiler::Esp32Compiler::with_temp_dir( - toolchain.get_gcc_path(), - toolchain.get_gxx_path(), - mcu_config.clone(), - &board.f_cpu, - defines, - include_dirs.to_vec(), - params.profile, - params.verbose, - build_dir.join("tmp"), - ) - .with_build_unflags(build_unflags.to_vec()) - .with_eh_frame_policy(eh_frame_policy); - let c_flags = apply_overlay_flags(&temp_compiler.c_flags(), user_overlay, "dummy.c"); - let cpp_flags = apply_overlay_flags(&temp_compiler.cpp_flags(), user_overlay, "dummy.cpp"); - - // Use gcc-ar for LTO archives so the linker-plugin index is written. - let ar_path = toolchain.get_ar_path(); - let gcc_ar_path = toolchain.get_gcc_ar_path(); - let archiver = crate::pipeline::pick_archiver(&ar_path, &gcc_ar_path, &c_flags, &cpp_flags); - fbuild_packages::library::library_manager::resolve_libraries( - lib_deps, - lib_ignore, - &toolchain.get_gcc_path(), - &toolchain.get_gxx_path(), - archiver, - &c_flags, - &cpp_flags, - include_dirs, - ¶ms.project_dir, - &build_dir.join("libs"), - params.verbose, - compiler_cache, - ) - .await -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/fbuild-build-esp/src/esp32/orchestrator/job_pool.rs b/crates/fbuild-build-esp/src/esp32/orchestrator/job_pool.rs deleted file mode 100644 index 24fdbfcb..00000000 --- a/crates/fbuild-build-esp/src/esp32/orchestrator/job_pool.rs +++ /dev/null @@ -1,151 +0,0 @@ -//! The ESP32 build's single compile job pool (FastLED/fbuild#1559). -//! -//! Everything is resolved first: `lib_deps` (sources only), library -//! selection, framework-library cache hits, the project-as-library and local -//! `lib/` libraries. Then every compile runs in one `tokio::join!`, each -//! translation unit drawing a permit from one [`JobGate`]: `lib_deps`, -//! framework libraries, project-as-library, core + sketch and local -//! libraries, with the boot artifacts (not gated) alongside. Only the link -//! waits for all of it. -//! -//! [`link_order`] assembles the library archives in exactly the order the -//! former serial phases produced them, so the link line is unchanged. - -use std::collections::HashSet; -use std::future::Future; -use std::path::Path; -use std::sync::Mutex; -use std::time::{Duration, Instant}; - -use fbuild_core::{BuildLog, Result}; -use fbuild_packages::library::library_compiler::JobGate; - -use super::compile_phases::{CompileTarget, compile_core_and_sketch}; -use super::framework_libs::{FwLibsCompiled, FwLibsPlan}; -use crate::compiler::Compiler; -use crate::parallel::ParallelCompileResult; -use crate::perf_log::PerfTimer; - -/// Framework-library names that a `lib_deps` or project-as-library archive -/// already provides, so the framework copy is skipped. -/// -/// Mirrors the former check against the file stems of the archives built -/// before the framework libraries (`lib{name}.a`, name case preserved): the -/// `lib_deps` archives and the project-as-library. Local `lib/` libraries -/// were built afterwards and never took part. -pub(super) fn already_compiled( - lib_deps: Vec, - project_library: Option, -) -> HashSet { - lib_deps.into_iter().chain(project_library).collect() -} - -/// Library archives in link order: `lib_deps`, the project-as-library, -/// framework libraries (selection order), then local `lib/` libraries. -pub(super) fn link_order( - lib_deps: Vec, - project_library: Option, - framework_libs: Vec, - local_libs: Vec, -) -> Vec { - lib_deps - .into_iter() - .chain(project_library) - .chain(framework_libs) - .chain(local_libs) - .collect() -} - -/// The library compiles that join the pool, as futures already bound to the -/// pool's gate. Framework libraries and core + sketch are run by [`run`]. -pub(super) struct LibraryJobs { - /// `lib_deps` archives in library order. - pub lib_deps: LD, - /// The project-as-library archive, if any. - pub project_library: PL, - /// Local `lib/` archives in library order. - pub local_libs: LL, - /// Boot artifacts; not a compile, so not gated. Yields its duration. - pub boot_artifacts: B, -} - -/// Everything the pool compiled; see [`link_order`] for the archives. -pub(super) struct PoolOutput { - pub lib_deps: Vec, - pub project_library: Option, - pub framework_libs: FwLibsCompiled, - pub core: ParallelCompileResult, - pub sketch: ParallelCompileResult, - pub local_libs: Vec, -} - -/// Run every compile of the build concurrently on `gate`. -/// -/// Uses `tokio::join!`, never `try_join!`: `compile_sources_parallel_shared` -/// extends its borrows to `'static` and is sound only when every branch is -/// awaited to completion. Errors are propagated afterwards, in the order the -/// former serial phases would have hit them. Framework-library failures are -/// not errors; [`FwLibsPlan::finish`] records them. -#[allow(clippy::too_many_arguments)] -pub(super) async fn run( - perf: &mut PerfTimer, - gate: &JobGate, - compiler: &(dyn Compiler + Send + Sync), - core: CompileTarget<'_, S>, - sketch: CompileTarget<'_, T>, - build_log: &Mutex, - framework_libs: &FwLibsPlan, - jobs: LibraryJobs, -) -> Result> -where - S: AsRef + Send + Sync, - T: AsRef + Send + Sync, - LD: Future>>, - PL: Future>>, - LL: Future>>, - B: Future>, -{ - let lib_deps = timed(jobs.lib_deps); - let project_library = timed(jobs.project_library); - let local_libs = timed(jobs.local_libs); - let fw = framework_libs.compile(gate); - // Sketch first inside, so its single TU starts with the first wave. - let core_sketch = compile_core_and_sketch(compiler, perf, gate, core, sketch, build_log); - let (compiled, fw, (lib_deps, lib_deps_elapsed), project_library, local_libs, boot) = tokio::join!( - core_sketch, - fw, - lib_deps, - project_library, - local_libs, - jobs.boot_artifacts - ); - perf.record("lib-deps", lib_deps_elapsed); - perf.record("project-lib", project_library.1); - perf.record("compile-local-libs", local_libs.1); - - let lib_deps = lib_deps?; - let project_library = project_library.0?; - let (core, sketch) = compiled?; - perf.record("boot-artifacts", boot?); - let local_libs = local_libs.0?; - Ok(PoolOutput { - lib_deps, - project_library, - framework_libs: fw, - core, - sketch, - local_libs, - }) -} - -/// `fut`'s output with its own wall time, so overlapping branches each report -/// how long they took rather than the whole pool's span. -async fn timed(fut: F) -> (F::Output, Duration) { - let started = Instant::now(); - let output = fut.await; - (output, started.elapsed()) -} - -#[cfg(test)] -#[path = "job_pool_tests.rs"] -mod tests; diff --git a/crates/fbuild-build-esp/src/esp32/orchestrator/job_pool_tests.rs b/crates/fbuild-build-esp/src/esp32/orchestrator/job_pool_tests.rs deleted file mode 100644 index 258ce763..00000000 --- a/crates/fbuild-build-esp/src/esp32/orchestrator/job_pool_tests.rs +++ /dev/null @@ -1,231 +0,0 @@ -//! Tests for the ESP32 build's single job pool (FastLED/fbuild#1559). - -use std::sync::Arc; -use std::sync::atomic::{AtomicUsize, Ordering}; - -use fbuild_packages::library::library_compiler::{ - LibCompileBackend, compile_library_gated, job_gate, -}; - -use super::super::framework_libs::tests::{FakeBackend, Fixture}; -use super::*; -use crate::flag_overlay::LanguageExtraFlags; - -#[test] -fn link_order_is_lib_deps_project_framework_then_local() { - let order = link_order( - vec!["libfastled.a", "libzlib.a"], - Some("libproject.a"), - vec!["libfs.a", "libwifi.a"], - vec!["libmine.a"], - ); - assert_eq!( - order, - [ - "libfastled.a", - "libzlib.a", - "libproject.a", - "libfs.a", - "libwifi.a", - "libmine.a" - ] - ); - assert_eq!( - link_order(Vec::new(), None, vec!["libfs.a"], Vec::new()), - ["libfs.a"] - ); -} - -#[test] -fn already_compiled_holds_lib_deps_and_project_library_names() { - let names = already_compiled( - vec!["FastLED".to_string(), "zlib".to_string()], - Some("myproject".to_string()), - ); - let expected: HashSet = ["FastLED", "zlib", "myproject"].map(String::from).into(); - assert_eq!(names, expected); - assert!(already_compiled(Vec::new(), None).is_empty()); -} - -/// A [`Compiler`] for core/sketch TUs counting in-flight compiles on the same -/// counters as the library [`FakeBackend`], so one maximum covers the pool. -struct FakeCompiler { - gcc: fbuild_core::path::NormalizedPath, - backend: Arc, -} - -#[async_trait::async_trait] -impl Compiler for FakeCompiler { - async fn compile_one( - &self, - _compiler_path: &Path, - _source: &Path, - output: &Path, - _flags: &[String], - _extra_flags: &[String], - ) -> Result { - let now = self.backend.in_flight.fetch_add(1, Ordering::SeqCst) + 1; - self.backend.max_in_flight.fetch_max(now, Ordering::SeqCst); - self.backend.calls.fetch_add(1, Ordering::SeqCst); - tokio::time::sleep(Duration::from_millis(60)).await; - self.backend.in_flight.fetch_sub(1, Ordering::SeqCst); - std::fs::write(output, "obj").unwrap(); - Ok(crate::compiler::CompileResult { - success: true, - object_file: output.to_path_buf(), - stdout: String::new(), - stderr: String::new(), - exit_code: 0, - }) - } - - fn gcc_path(&self) -> &Path { - self.gcc.as_path() - } - - fn gxx_path(&self) -> &Path { - self.gcc.as_path() - } - - fn c_flags(&self) -> Vec { - Vec::new() - } - - fn cpp_flags(&self) -> Vec { - Vec::new() - } - - fn rebuild_signature(&self, source: &Path, _extra: &[String], _out: &Path) -> String { - source.to_string_lossy().into_owned() - } -} - -/// Runs a pool of core (2 TUs), sketch (1), one framework library (1), one -/// `lib_deps` library (1) and a project library (none) on `jobs` permits. -/// Returns the pool's max in-flight compiles and the archives in link order. -async fn run_pool(jobs: usize) -> (usize, Vec) { - let backend = Arc::new(FakeBackend::new( - Arc::new(AtomicUsize::new(0)), - Arc::new(AtomicUsize::new(0)), - )); - let fx = Fixture::with_backend(backend.clone()); - let root = fx.tmp.path(); - let compiler = FakeCompiler { - gcc: fbuild_core::path::NormalizedPath::new(root.join("gcc")), - backend: backend.clone(), - }; - let write = |name: &str| { - let path = root.join("src").join(name); - std::fs::write(&path, "int x;").unwrap(); - path - }; - let core_sources = vec![write("core1.cpp"), write("core2.cpp")]; - let sketch_sources = vec![write("main.cpp")]; - let dep_sources = vec![write("dep.cpp")]; - let (core_dir, sketch_dir) = (root.join("core"), root.join("sketch")); - std::fs::create_dir_all(&core_dir).unwrap(); - std::fs::create_dir_all(&sketch_dir).unwrap(); - let overlay = LanguageExtraFlags::default(); - - let mut log = BuildLog::new(); - let libraries = vec![fx.library("WiFi", &["wifi.cpp"])]; - let fw_libs = fx.plan(&libraries, &HashSet::new(), &mut log); - let build_log = Mutex::new(log); - let gate = job_gate(jobs); - let lib_backend: Arc = backend.clone(); - let (ar, deps_dir) = (root.join("ar"), root.join("libs").join("dep")); - let jobs = LibraryJobs { - lib_deps: async { - let archive = compile_library_gated( - "dep", - &dep_sources, - &[], - &root.join("gcc"), - &root.join("g++"), - &ar, - &[], - &[], - &deps_dir, - false, - &gate, - None, - None, - Some(lib_backend), - None, - ) - .await?; - Ok(archive.into_iter().collect()) - }, - project_library: async { Ok(None) }, - local_libs: async { Ok(Vec::new()) }, - boot_artifacts: async { Ok(Duration::ZERO) }, - }; - let mut perf = PerfTimer::new("job-pool-test"); - let pool = run( - &mut perf, - &gate, - &compiler, - CompileTarget { - sources: &core_sources, - build_dir: &core_dir, - overlay: &overlay, - }, - CompileTarget { - sources: &sketch_sources, - build_dir: &sketch_dir, - overlay: &overlay, - }, - &build_log, - &fw_libs, - jobs, - ) - .await - .unwrap(); - assert_eq!(pool.core.objects.len(), 2); - assert_eq!(pool.sketch.objects.len(), 1); - assert_eq!(backend.calls.load(Ordering::SeqCst), 5); - - let mut log = build_log.into_inner().unwrap_or_else(|e| e.into_inner()); - let framework = fw_libs.finish(pool.framework_libs, &mut perf, &mut log); - let archives = link_order( - pool.lib_deps, - pool.project_library, - framework, - pool.local_libs, - ) - .iter() - .map(|archive| archive.file_name().unwrap().to_string_lossy().into_owned()) - .collect(); - (backend.max_in_flight.load(Ordering::SeqCst), archives) -} - -/// Every compile of the build draws from the caller's one gate: core + -/// sketch, framework libraries and `lib_deps` together never exceed it. -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn pool_never_exceeds_the_shared_gate() { - if fbuild_core::platform::host::is_windows() { - return; - } - let (max, archives) = run_pool(2).await; - assert!( - max <= 2, - "max in-flight {max} exceeded the gate's 2 permits" - ); - assert_eq!(archives, ["libdep.a", "libwifi.a"]); -} - -/// With enough permits, TUs from different phases are in flight at once: the -/// largest phase (core) has two TUs, so three or more means phases overlap -/// instead of running one after another. -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn pool_overlaps_every_phase() { - if fbuild_core::platform::host::is_windows() { - return; - } - let (max, archives) = run_pool(8).await; - assert!( - (3..=5).contains(&max), - "max in-flight {max}: core, sketch, framework and lib_deps TUs overlap" - ); - assert_eq!(archives, ["libdep.a", "libwifi.a"]); -} diff --git a/crates/fbuild-build-esp/src/esp32/orchestrator/local_libs.rs b/crates/fbuild-build-esp/src/esp32/orchestrator/local_libs.rs index 05689d5e..9026046b 100644 --- a/crates/fbuild-build-esp/src/esp32/orchestrator/local_libs.rs +++ b/crates/fbuild-build-esp/src/esp32/orchestrator/local_libs.rs @@ -3,20 +3,15 @@ use std::path::{Path, PathBuf}; use fbuild_core::Result; -use fbuild_packages::library::library_compiler::JobGate; use super::super::esp32_compiler::Esp32Compiler; use crate::compiler::Compiler as _; use crate::flag_overlay::{LanguageExtraFlags, apply_overlay_flags}; -/// Compile each selected `lib/` library as an archive. `libraries` comes from +/// Compile each selected `lib/` library as an archive, appended to +/// `library_archives`. `libraries` comes from /// [`crate::framework_libs::select_local_libraries`], so a library nothing /// includes is never compiled (FastLED/fbuild#1410). -/// -/// All libraries compile concurrently, each translation unit drawing a permit -/// from the build's shared `gate` (FastLED/fbuild#1559). Archives come back in -/// library order (the link order); the first failure in that order is the -/// error, reported once every library has finished. #[allow(clippy::too_many_arguments)] pub(super) async fn compile_local_libraries( libraries: &[fbuild_packages::library::FrameworkLibrary], @@ -25,62 +20,59 @@ pub(super) async fn compile_local_libraries( toolchain: &fbuild_packages::toolchain::Esp32Toolchain, include_dirs: &[PathBuf], src_overlay: &LanguageExtraFlags, - gate: &JobGate, + jobs: usize, verbose: bool, compiler_cache: Option<&Path>, -) -> Result> { + library_archives: &mut Vec, +) -> Result<()> { use fbuild_packages::Toolchain; - let c_flags = apply_overlay_flags(&compiler.c_flags(), src_overlay, "dummy.c"); - let cpp_flags = apply_overlay_flags(&compiler.cpp_flags(), src_overlay, "dummy.cpp"); - // Use gcc-ar for LTO archives so the linker-plugin index is written. - let ar_path = toolchain.get_ar_path(); - let gcc_ar_path = toolchain.get_gcc_ar_path(); - let archiver = crate::pipeline::pick_archiver(&ar_path, &gcc_ar_path, &c_flags, &cpp_flags); - let gcc_path = toolchain.get_gcc_path(); - let gxx_path = toolchain.get_gxx_path(); + for library in libraries { + let lib_name = library.name.clone(); + let lib_sources = &library.source_files; + if lib_sources.is_empty() { + continue; + } - let compiles = libraries - .iter() - .filter(|library| !library.source_files.is_empty()) - .map(|library| { - let lib_build_dir = build_dir.join("lib").join(&library.name); - tracing::info!( - "compiling local library '{}': {} source files", - library.name, - library.source_files.len() - ); - let (c_flags, cpp_flags) = (&c_flags, &cpp_flags); - let (gcc_path, gxx_path) = (&gcc_path, &gxx_path); - async move { - let result = fbuild_packages::library::library_compiler::compile_library_gated( - &library.name, - &library.source_files, - include_dirs, - gcc_path, - gxx_path, - archiver, - c_flags, - cpp_flags, - &lib_build_dir, - verbose, - gate, - compiler_cache, - None, - None, - None, - ) - .await; - (library.name.as_str(), result) - } - }); - // join_all runs every compile to completion even if one fails. - let outcomes = futures::future::join_all(compiles).await; + let lib_build_dir = build_dir.join("lib").join(&lib_name); + tracing::info!( + "compiling local library '{}': {} source files", + lib_name, + lib_sources.len() + ); - let mut archives = Vec::new(); - for (lib_name, outcome) in outcomes { - match outcome { - Ok(Some(archive)) => archives.push(archive), + // Use gcc-ar for LTO archives so the linker-plugin index is written. + let local_ar_path = toolchain.get_ar_path(); + let local_gcc_ar_path = toolchain.get_gcc_ar_path(); + let local_c_flags = apply_overlay_flags(&compiler.c_flags(), src_overlay, "dummy.c"); + let local_cpp_flags = apply_overlay_flags(&compiler.cpp_flags(), src_overlay, "dummy.cpp"); + let local_lib_ar_path = crate::pipeline::pick_archiver( + &local_ar_path, + &local_gcc_ar_path, + &local_c_flags, + &local_cpp_flags, + ); + match fbuild_packages::library::library_compiler::compile_library_with_jobs( + &lib_name, + lib_sources, + include_dirs, + &toolchain.get_gcc_path(), + &toolchain.get_gxx_path(), + local_lib_ar_path, + &local_c_flags, + &local_cpp_flags, + &lib_build_dir, + verbose, + jobs, + compiler_cache, + None, + None, + ) + .await + { + Ok(Some(archive)) => { + library_archives.push(archive); + } Ok(None) => {} // header-only Err(e) => { return Err(fbuild_core::FbuildError::BuildFailed(format!( @@ -90,127 +82,5 @@ pub(super) async fn compile_local_libraries( } } } - Ok(archives) -} - -/// The project's own `src/` compiled as a library archive when the project -/// root carries `library.json`/`library.properties` (e.g. FastLED building an -/// example); shared with the sequential pipeline via -/// [`crate::pipeline::compile_project_as_library`]. -pub(super) struct ProjectLibrary { - c_flags: Vec, - cpp_flags: Vec, - ar_path: PathBuf, - /// `lib/*` names, so a collision with the project-as-library is detected. - existing_lib_names: std::collections::HashSet, - jobs: usize, -} - -impl ProjectLibrary { - /// Flags as the sketch sees them: SDK defines + user flags, so the - /// archive matches what sketch sources compile against. - #[allow(clippy::too_many_arguments)] - pub(super) fn prepare( - params: &crate::BuildParams, - toolchain: &fbuild_packages::toolchain::Esp32Toolchain, - mcu_config: &super::super::mcu_config::Esp32McuConfig, - board: &fbuild_config::BoardConfig, - build_unflags: &[String], - eh_frame_policy: crate::eh_frame_policy::EhFramePolicy, - include_dirs: &[PathBuf], - src_overlay: &LanguageExtraFlags, - build_dir: &Path, - ) -> Self { - use fbuild_packages::Toolchain; - - let mut defines = board.get_defines(); - defines.extend(mcu_config.defines_map()); - let compiler = Esp32Compiler::with_temp_dir( - toolchain.get_gcc_path(), - toolchain.get_gxx_path(), - mcu_config.clone(), - &board.f_cpu, - defines, - include_dirs.to_vec(), - params.profile, - params.verbose, - build_dir.join("tmp"), - ) - .with_build_unflags(build_unflags.to_vec()) - .with_eh_frame_policy(eh_frame_policy); - let c_flags = apply_overlay_flags(&compiler.c_flags(), src_overlay, "dummy.c"); - let cpp_flags = apply_overlay_flags(&compiler.cpp_flags(), src_overlay, "dummy.cpp"); - - let mut existing_lib_names = std::collections::HashSet::new(); - if let Ok(entries) = std::fs::read_dir(params.project_dir.join("lib")) { - for entry in entries.flatten() { - let path = entry.path(); - if path.is_dir() { - if let Some(name) = path.file_name().and_then(|n| n.to_str()) { - existing_lib_names.insert(name.to_lowercase()); - } - } - } - } - - // Use gcc-ar for LTO archives so the linker-plugin index is written. - let ar_path = crate::pipeline::pick_archiver( - &toolchain.get_ar_path(), - &toolchain.get_gcc_ar_path(), - &c_flags, - &cpp_flags, - ) - .to_path_buf(); - Self { - c_flags, - cpp_flags, - ar_path, - existing_lib_names, - jobs: crate::parallel::effective_jobs(params.jobs), - } - } - - /// The archive name (`lib{name}.a`) this produces, if any, before compiling. - pub(super) fn name(&self, project_dir: &Path, src_dir: &Path) -> Option { - crate::pipeline::project_library_name(project_dir, src_dir, &self.existing_lib_names) - } - - /// Compile on the build's shared `gate`. - #[allow(clippy::too_many_arguments)] - pub(super) async fn compile( - &self, - project_dir: &Path, - src_dir: &Path, - build_dir: &Path, - toolchain: &fbuild_packages::toolchain::Esp32Toolchain, - include_dirs: &[PathBuf], - verbose: bool, - compiler_cache: Option<&Path>, - gate: &JobGate, - ) -> Result> { - use fbuild_packages::Toolchain; - - let gcc_path = toolchain.get_gcc_path(); - let gxx_path = toolchain.get_gxx_path(); - let env = crate::pipeline::LibraryBuildEnv { - gcc_path: &gcc_path, - gxx_path: &gxx_path, - ar_path: &self.ar_path, - c_flags: &self.c_flags, - cpp_flags: &self.cpp_flags, - include_dirs, - verbose, - jobs: self.jobs, - compiler_cache, - }; - crate::pipeline::compile_project_as_library( - project_dir, - src_dir, - build_dir, - &env, - &self.existing_lib_names, - gate, - ) - .await - } + Ok(()) } diff --git a/crates/fbuild-build-esp/src/esp32/orchestrator/mod.rs b/crates/fbuild-build-esp/src/esp32/orchestrator/mod.rs index f271a0f3..32ca57f8 100644 --- a/crates/fbuild-build-esp/src/esp32/orchestrator/mod.rs +++ b/crates/fbuild-build-esp/src/esp32/orchestrator/mod.rs @@ -25,14 +25,12 @@ mod boot_artifacts; mod build; mod cdc; -mod compile_phases; mod embed; mod embed_stage; mod fingerprint; mod framework_library_cache; mod framework_libs; mod helpers; -mod job_pool; mod local_libs; mod packages; diff --git a/crates/fbuild-build-esp/src/esp8266/orchestrator.rs b/crates/fbuild-build-esp/src/esp8266/orchestrator.rs index b1de6e0a..f240369c 100644 --- a/crates/fbuild-build-esp/src/esp8266/orchestrator.rs +++ b/crates/fbuild-build-esp/src/esp8266/orchestrator.rs @@ -430,7 +430,7 @@ impl BuildOrchestrator for Esp8266Orchestrator { .config .get_lib_ignore(¶ms.env_name) .unwrap_or_default(); - let lib_deps_plan = if !lib_deps.is_empty() { + let lib_archives = if !lib_deps.is_empty() { let temp_compiler = Esp8266Compiler::new( toolchain.get_gcc_path(), toolchain.get_gxx_path(), @@ -454,11 +454,12 @@ impl BuildOrchestrator for Esp8266Orchestrator { &crate::compiler::Compiler::cpp_flags(&temp_compiler), &mut include_dirs, params.verbose, + crate::parallel::effective_jobs(params.jobs), None, ) .await? } else { - pipeline::LibDeps::default() + Vec::new() }; let compiler = Esp8266Compiler::new( @@ -532,8 +533,7 @@ impl BuildOrchestrator for Esp8266Orchestrator { ctx, params, &sources, - &[], - lib_deps_plan, + &lib_archives, Some(&lib_env), TargetArchitecture::Xtensa, "ESP8266", diff --git a/crates/fbuild-build-mcu/Cargo.toml b/crates/fbuild-build-mcu/Cargo.toml index 31d51013..7a706edf 100644 --- a/crates/fbuild-build-mcu/Cargo.toml +++ b/crates/fbuild-build-mcu/Cargo.toml @@ -34,7 +34,7 @@ tar = { workspace = true } zip = { workspace = true } tree-sitter = { workspace = true } tree-sitter-cpp = { workspace = true } -zccache = { git = "https://github.com/zackees/zccache", rev = "b385a7bfb308c2d1456613622ad361eb66d31cdd" } +zccache = { git = "https://github.com/zackees/zccache", rev = "3f9d4e66fe3675cd48060bca03b62b1f5a8112eb" } [dev-dependencies] filetime = { workspace = true } diff --git a/crates/fbuild-build-mcu/src/avr/orchestrator.rs b/crates/fbuild-build-mcu/src/avr/orchestrator.rs index 23d4914e..5666218f 100644 --- a/crates/fbuild-build-mcu/src/avr/orchestrator.rs +++ b/crates/fbuild-build-mcu/src/avr/orchestrator.rs @@ -259,7 +259,7 @@ impl BuildOrchestrator for AvrOrchestrator { .config .get_lib_ignore(¶ms.env_name) .unwrap_or_default(); - let lib_deps_plan = if !lib_deps.is_empty() { + let lib_archives = if !lib_deps.is_empty() { // Build a temp compiler solely to get the c/cxx flags for library // compilation. The temp compiler is discarded — the *real* compiler // is created afterwards with the full include-dir set. @@ -287,11 +287,12 @@ impl BuildOrchestrator for AvrOrchestrator { &temp_compiler.cpp_flags(), &mut include_dirs, params.verbose, + crate::parallel::effective_jobs(params.jobs), compiler_cache.as_deref(), ) .await? } else { - pipeline::LibDeps::default() + Vec::new() }; // 6b. Create compiler with the full include-dir set (core, variant, @@ -356,8 +357,7 @@ impl BuildOrchestrator for AvrOrchestrator { ctx, params, &sources, - &[], - lib_deps_plan, + &lib_archives, Some(&lib_env), TargetArchitecture::Avr, "AVR", diff --git a/crates/fbuild-build-mcu/src/ch32v/orchestrator.rs b/crates/fbuild-build-mcu/src/ch32v/orchestrator.rs index b0c75bf0..ead08e77 100644 --- a/crates/fbuild-build-mcu/src/ch32v/orchestrator.rs +++ b/crates/fbuild-build-mcu/src/ch32v/orchestrator.rs @@ -352,7 +352,7 @@ impl BuildOrchestrator for Ch32vOrchestrator { .config .get_lib_ignore(¶ms.env_name) .unwrap_or_default(); - let lib_deps_plan = if !lib_deps.is_empty() { + let lib_archives = if !lib_deps.is_empty() { let temp_compiler = Ch32vCompiler::new( toolchain.get_gcc_path(), toolchain.get_gxx_path(), @@ -378,11 +378,12 @@ impl BuildOrchestrator for Ch32vOrchestrator { &crate::compiler::Compiler::cpp_flags(&temp_compiler), &mut include_dirs, params.verbose, + crate::parallel::effective_jobs(params.jobs), None, ) .await? } else { - pipeline::LibDeps::default() + Vec::new() }; let compiler = Ch32vCompiler::new( @@ -458,8 +459,7 @@ impl BuildOrchestrator for Ch32vOrchestrator { ctx, params, &sources, - &[], - lib_deps_plan, + &lib_archives, Some(&lib_env), TargetArchitecture::Riscv32, "CH32V", diff --git a/crates/fbuild-build/Cargo.toml b/crates/fbuild-build/Cargo.toml index 4ddd5d1c..67bf17d0 100644 --- a/crates/fbuild-build/Cargo.toml +++ b/crates/fbuild-build/Cargo.toml @@ -48,17 +48,17 @@ tree-sitter-cpp = { workspace = true } # child processes) was deleted in the same PR; there is no # `--no-default-features` escape hatch. # -# Pinned at zccache main `b385a7b` (1.14.14 plus its post-release perf -# work: #1768's heartbeat-free compile requests, #1770's mtime-floor skip on -# C/C++ hits, #1761/#1763's dropped durability syncs, #1775 unchanged-state -# shutdown skip), which takes +# Pinned at zccache main `3f9d4e6` (1.14.14 + zackees/zccache#1670's +# single-pass include lexer, #1708's single-flight toolchain probes +# (FastLED/fbuild#1464/#1466), and #1710's non-blocking embedded bring-up +# plus periodic metadata snapshot (FastLED/fbuild#1480)), which takes # running-process =4.10.14 from crates.io. Keep both pins aligned when # repinning so fbuild links exactly one copy of the exported broker symbols. # # `running-process` in the workspace root must be pinned to the SAME source # and version zccache uses, or its unmangled `rp_*_public` C symbols get # linked twice (guarded by crates/fbuild-core/tests/dep_identity.rs). -zccache = { git = "https://github.com/zackees/zccache", rev = "b385a7bfb308c2d1456613622ad361eb66d31cdd" } +zccache = { git = "https://github.com/zackees/zccache", rev = "3f9d4e66fe3675cd48060bca03b62b1f5a8112eb" } [dev-dependencies] filetime = { workspace = true } diff --git a/crates/fbuild-core/src/path.rs b/crates/fbuild-core/src/path.rs index 02141dcc..b1711c26 100644 --- a/crates/fbuild-core/src/path.rs +++ b/crates/fbuild-core/src/path.rs @@ -407,52 +407,12 @@ pub fn normalize_flags_for_compile_cwd(flags: &[String], cwd: &Path) -> Vec &'static std::sync::RwLock> { - static MEMO: std::sync::OnceLock< - std::sync::RwLock>, - > = std::sync::OnceLock::new(); - MEMO.get_or_init(|| std::sync::RwLock::new(std::collections::HashMap::new())) -} - /// Canonicalize an existing path (stripping the Windows `\\?\` prefix), /// falling back to canonicalizing the parent + rejoining the file name when /// the full path does not yet exist. Returns `None` if neither resolves. fn canonicalize_lexical(path: &Path) -> Option { - if let Some(hit) = canonicalize_memo() - .read() - .unwrap_or_else(|e| e.into_inner()) - .get(path) - { - return Some(hit.clone()); - } if let Ok(canonical) = path.canonicalize() { - let resolved = strip_unc_prefix(&canonical); - let mut memo = canonicalize_memo() - .write() - .unwrap_or_else(|e| e.into_inner()); - if memo.len() >= CANONICALIZE_MEMO_CAPACITY { - memo.clear(); - } - memo.insert(path.to_path_buf(), resolved.clone()); - return Some(resolved); + return Some(strip_unc_prefix(&canonical)); } let parent = path.parent()?.canonicalize().ok()?; let joined = match path.file_name() { @@ -874,33 +834,6 @@ mod tests { assert_eq!(arg, "src/sketch/main.cpp"); } - #[test] - fn canonicalize_lexical_memoizes_resolved_paths() { - let tmp = tempfile::TempDir::new().unwrap(); - let dir = tmp.path().join("include"); - std::fs::create_dir_all(&dir).unwrap(); - let resolved = dir.canonicalize().unwrap(); - assert_eq!(canonicalize_lexical(&dir), Some(resolved.clone())); - // A second call is served from the memo rather than a fresh realpath. - assert_eq!(canonicalize_lexical(&dir), Some(resolved)); - } - - #[test] - fn canonicalize_lexical_does_not_memoize_the_missing_parent_fallback() { - let tmp = tempfile::TempDir::new().unwrap(); - let missing = tmp.path().join("later"); - // First call takes the parent-plus-filename fallback. - let before = canonicalize_lexical(&missing).unwrap(); - assert!(before.ends_with("later")); - std::fs::create_dir_all(&missing).unwrap(); - // Once the directory exists the real path must win, so the fallback - // result cannot have been memoized. - assert_eq!( - canonicalize_lexical(&missing), - Some(missing.canonicalize().unwrap()) - ); - } - #[test] fn path_arg_for_compile_cwd_returns_dot_for_workspace_root() { let tmp = tempfile::TempDir::new().unwrap(); diff --git a/crates/fbuild-core/src/platform/fs.rs b/crates/fbuild-core/src/platform/fs.rs index 87d22e61..e28e46eb 100644 --- a/crates/fbuild-core/src/platform/fs.rs +++ b/crates/fbuild-core/src/platform/fs.rs @@ -84,11 +84,6 @@ pub fn symlink_dir(original: &Path, link: &Path) -> std::io::Result<()> { super::selected::fs::symlink_dir(original, link) } -/// Create a file symlink using the host-native operation. -pub fn symlink_file(original: &Path, link: &Path) -> std::io::Result<()> { - super::selected::fs::symlink_file(original, link) -} - /// Return whether a path is a symbolic link or Windows reparse point. pub fn is_link_or_reparse(path: &Path) -> std::io::Result { super::selected::fs::is_link_or_reparse(path) diff --git a/crates/fbuild-core/src/platform/linux/fs.rs b/crates/fbuild-core/src/platform/linux/fs.rs index 912184b1..6e8eb2ba 100644 --- a/crates/fbuild-core/src/platform/linux/fs.rs +++ b/crates/fbuild-core/src/platform/linux/fs.rs @@ -57,10 +57,6 @@ pub(crate) fn symlink_dir(original: &Path, link: &Path) -> std::io::Result<()> { std::os::unix::fs::symlink(original, link) } -pub(crate) fn symlink_file(original: &Path, link: &Path) -> std::io::Result<()> { - std::os::unix::fs::symlink(original, link) -} - pub(crate) fn is_link_or_reparse(path: &Path) -> std::io::Result { Ok(std::fs::symlink_metadata(path)?.file_type().is_symlink()) } diff --git a/crates/fbuild-core/src/platform/macos/fs.rs b/crates/fbuild-core/src/platform/macos/fs.rs index 0f870ed4..8d1d8876 100644 --- a/crates/fbuild-core/src/platform/macos/fs.rs +++ b/crates/fbuild-core/src/platform/macos/fs.rs @@ -57,10 +57,6 @@ pub(crate) fn symlink_dir(original: &Path, link: &Path) -> std::io::Result<()> { std::os::unix::fs::symlink(original, link) } -pub(crate) fn symlink_file(original: &Path, link: &Path) -> std::io::Result<()> { - std::os::unix::fs::symlink(original, link) -} - pub(crate) fn is_link_or_reparse(path: &Path) -> std::io::Result { Ok(std::fs::symlink_metadata(path)?.file_type().is_symlink()) } diff --git a/crates/fbuild-core/src/platform/windows/fs.rs b/crates/fbuild-core/src/platform/windows/fs.rs index 3fe10626..274dedb8 100644 --- a/crates/fbuild-core/src/platform/windows/fs.rs +++ b/crates/fbuild-core/src/platform/windows/fs.rs @@ -60,10 +60,6 @@ pub(crate) fn symlink_dir(original: &Path, link: &Path) -> std::io::Result<()> { std::os::windows::fs::symlink_dir(original, link) } -pub(crate) fn symlink_file(original: &Path, link: &Path) -> std::io::Result<()> { - std::os::windows::fs::symlink_file(original, link) -} - pub(crate) fn is_link_or_reparse(path: &Path) -> std::io::Result { const FILE_ATTRIBUTE_REPARSE_POINT: u32 = 0x0000_0400; Ok(std::fs::symlink_metadata(path)?.file_attributes() & FILE_ATTRIBUTE_REPARSE_POINT != 0) diff --git a/crates/fbuild-daemon/src/main.rs b/crates/fbuild-daemon/src/main.rs index 0e7be223..fa240481 100644 --- a/crates/fbuild-daemon/src/main.rs +++ b/crates/fbuild-daemon/src/main.rs @@ -74,27 +74,6 @@ async fn main() { } } - { - use fbuild_build::zccache_embedded::{ - FBUILD_MAX_PARALLEL_COMPILES_ENV, ZCCACHE_MAX_PARALLEL_COMPILES_ENV, - compile_cap_env_value, - }; - let cores = std::thread::available_parallelism().map_or(1, |n| n.get()); - if let Some(cap) = compile_cap_env_value( - std::env::var(ZCCACHE_MAX_PARALLEL_COMPILES_ENV) - .ok() - .as_deref(), - std::env::var(FBUILD_MAX_PARALLEL_COMPILES_ENV) - .ok() - .as_deref(), - cores, - ) { - // SAFETY: daemon startup, before the embedded zccache service - // (the only reader) starts. - unsafe { std::env::set_var(ZCCACHE_MAX_PARALLEL_COMPILES_ENV, cap) }; - } - } - // Install the process-wide containment group as early as possible so // every subprocess the daemon spawns (compilers, linkers, esptool, // avrdude, qemu, simavr, node, npm, …) is born inside a Windows Job diff --git a/crates/fbuild-header-scan/src/lib.rs b/crates/fbuild-header-scan/src/lib.rs index b06f5f05..a9ace6d0 100644 --- a/crates/fbuild-header-scan/src/lib.rs +++ b/crates/fbuild-header-scan/src/lib.rs @@ -14,8 +14,8 @@ pub use scanner::{ scan, scan_active, scan_active_with_known, }; pub use walker::{ - WalkResult, WalkState, collect_defined_macro_names, collect_defined_macro_names_with, walk, - walk_active, walk_with_state, walk_with_state_active, walk_with_state_active_known, + WalkResult, WalkState, collect_defined_macro_names, walk, walk_active, walk_with_state, + walk_with_state_active, walk_with_state_active_known, }; /// Bumped whenever the scanner output shape changes. Mixed into cache keys so a diff --git a/crates/fbuild-header-scan/src/walker.rs b/crates/fbuild-header-scan/src/walker.rs index c150c9f8..efc907de 100644 --- a/crates/fbuild-header-scan/src/walker.rs +++ b/crates/fbuild-header-scan/src/walker.rs @@ -59,116 +59,6 @@ pub struct WalkState { /// lifetime of this state. Each unique file is counted exactly once /// because subsequent walks hit `scan_cache` instead. files_read: usize, - resolver: IncludeResolver, -} - -/// Memoized include resolution for one search-path list. -/// -/// A name's search-path resolution does not depend on the including file, so -/// it is computed once instead of `is_file`-probing every search path for every -/// reference. That probing was two thirds of an incremental FastLED build's -/// daemon CPU with ~400 ESP32 search paths (FastLED/fbuild#1539). -#[derive(Debug, Default)] -struct IncludeResolver { - search_paths: Vec, - by_name: HashMap>, - canonical: HashMap, - /// First path component -> indices of the search paths whose root holds - /// it, in search order. Built on the first miss; `Some(None)` means a root - /// could not be listed and every lookup probes each path instead. - roots: Option>>>, -} - -/// Keys fold case so a case-insensitive filesystem's `Foo.h` still finds -/// `foo.h`. That only widens the candidate set; each candidate is confirmed -/// with `is_file` in search order, so the first match is the probe loop's. -fn root_key(name: &str) -> String { - name.to_lowercase() -} - -fn list_roots(search_paths: &[PathBuf]) -> Option>> { - let mut roots: HashMap> = HashMap::new(); - for (index, dir) in search_paths.iter().enumerate() { - let entries = match std::fs::read_dir(dir) { - Ok(entries) => entries, - // A missing search path holds nothing, exactly as probing it would. - Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, - Err(_) if !dir.is_dir() => continue, - Err(_) => return None, - }; - for entry in entries { - let name = entry.ok()?.file_name().into_string().ok()?; - let owners = roots.entry(root_key(&name)).or_default(); - if owners.last() != Some(&index) { - owners.push(index); - } - } - } - Some(roots) -} - -impl IncludeResolver { - fn bind(&mut self, search_paths: &[PathBuf]) { - if self.search_paths != search_paths { - self.search_paths = search_paths.to_vec(); - self.by_name.clear(); - self.roots = None; - } - } - - fn resolve(&mut self, inc: &IncludeRef, from: &Path) -> Option { - if inc.kind == IncludeKind::Quoted { - if let Some(parent) = from.parent() { - let candidate = parent.join(&inc.path); - if candidate.is_file() { - return Some(candidate); - } - } - } - if let Some(found) = self.by_name.get(&inc.path) { - return found.clone(); - } - let found = self.search(&inc.path); - self.by_name.insert(inc.path.clone(), found.clone()); - found - } - - /// The first search path holding `name`. Only paths whose root lists the - /// name's first component can hold it, so those are the only ones probed. - fn search(&mut self, name: &str) -> Option { - let plain = !name.contains('\\') - && !Path::new(name).is_absolute() - && name - .split('/') - .all(|part| !part.is_empty() && part != "." && part != ".."); - let roots = self - .roots - .get_or_insert_with(|| list_roots(&self.search_paths)); - match (plain, roots) { - (true, Some(roots)) => { - let first = name.split('/').next().unwrap_or(name); - let owners = roots.get(&root_key(first))?; - owners - .iter() - .map(|&index| self.search_paths[index].join(name)) - .find(|candidate| candidate.is_file()) - } - _ => self - .search_paths - .iter() - .map(|sp| sp.join(name)) - .find(|candidate| candidate.is_file()), - } - } - - fn canon(&mut self, path: &Path) -> PathBuf { - if let Some(found) = self.canonical.get(path) { - return found.clone(); - } - let resolved = canon(path); - self.canonical.insert(path.to_path_buf(), resolved.clone()); - resolved - } } impl WalkState { @@ -268,38 +158,15 @@ pub fn walk_with_state_active_known( /// *could* define — a conditional must not filter the answer. Uses its own /// [`WalkState`] so the active passes keep their own scan cache semantics. pub fn collect_defined_macro_names(seeds: &[PathBuf], search_paths: &[PathBuf]) -> HashSet { - collect_defined_macro_names_with(seeds, search_paths, &mut WalkState::new()) -} - -/// [`collect_defined_macro_names`] reusing `state`'s include resolution. -/// -/// Only the resolver is shared: the textual walk keeps its own scan cache and -/// visited set, so the active passes that follow on `state` are unaffected. -pub fn collect_defined_macro_names_with( - seeds: &[PathBuf], - search_paths: &[PathBuf], - state: &mut WalkState, -) -> HashSet { - let mut textual = WalkState { - resolver: std::mem::take(&mut state.resolver), - ..WalkState::default() - }; - let result = walk_with_state(seeds, search_paths, &mut textual); - state.resolver = textual.resolver; - // Parallel, like the walk's own reads: thousands of files on FastLED. - result - .reached - .par_iter() - .filter_map(|path| std::fs::read_to_string(path).ok()) - .map(|src| { - defined_macro_names(&src) - .into_iter() - .collect::>() - }) - .reduce(HashSet::new, |mut all, names| { - all.extend(names); - all - }) + let mut state = WalkState::new(); + let result = walk_with_state(seeds, search_paths, &mut state); + let mut names = HashSet::new(); + for path in &result.reached { + if let Ok(src) = std::fs::read_to_string(path) { + names.extend(defined_macro_names(&src)); + } + } + names } fn walk_with_state_scanner( @@ -319,10 +186,9 @@ where let mut reached: BTreeSet = BTreeSet::new(); let mut unresolved: BTreeSet = BTreeSet::new(); let mut frontier: VecDeque = VecDeque::new(); - state.resolver.bind(search_paths); for seed in seeds { - let canon = state.resolver.canon(seed); + let canon = canon(seed); if state.visited.insert(canon.clone()) { frontier.push_back(canon.clone()); reached.insert(canon); @@ -362,9 +228,9 @@ where continue; }; for inc in &includes { - match state.resolver.resolve(inc, file) { + match resolve_include(inc, file, search_paths) { Some(resolved) => { - let canon = state.resolver.canon(&resolved); + let canon = canon(&resolved); if state.visited.insert(canon.clone()) { reached.insert(canon.clone()); frontier.push_back(canon); @@ -384,6 +250,24 @@ where } } +fn resolve_include(inc: &IncludeRef, from: &Path, search_paths: &[PathBuf]) -> Option { + if inc.kind == IncludeKind::Quoted { + if let Some(parent) = from.parent() { + let candidate = parent.join(&inc.path); + if candidate.is_file() { + return Some(candidate); + } + } + } + for sp in search_paths { + let candidate = sp.join(&inc.path); + if candidate.is_file() { + return Some(candidate); + } + } + None +} + fn canon(p: &Path) -> PathBuf { // FastLED/fbuild#844 sync-context allowlist: this helper runs inside // a rayon-parallel BFS (`walk_with_state`). Using @@ -544,108 +428,4 @@ mod tests { let r2 = walk(seeds, &[]); assert_eq!(r1, r2); } - - #[test] - fn w30_cached_search_result_never_overrides_a_sibling_quoted_include() { - let tmp = tempdir(); - let inc = tmp.path().join("inc"); - write(&inc.join("x.h"), ""); - // a/main.cpp has its own sibling x.h; b/main.cpp does not. - let a_main = tmp.path().join("a/main.cpp"); - let b_main = tmp.path().join("b/main.cpp"); - write(&tmp.path().join("a/x.h"), ""); - write(&a_main, "#include \"x.h\"\n"); - write(&b_main, "#include \"x.h\"\n#include \n"); - - let res = walk(&[b_main, a_main], std::slice::from_ref(&inc)); - - let canon = |p: PathBuf| std::fs::canonicalize(p).unwrap(); - assert!(res.reached.contains(&canon(tmp.path().join("a/x.h")))); - assert!(res.reached.contains(&canon(inc.join("x.h")))); - } - - #[test] - fn w31_reused_state_with_new_search_paths_resolves_afresh() { - let tmp = tempdir(); - let first = tmp.path().join("first"); - let second = tmp.path().join("second"); - write(&second.join("only_second.h"), ""); - let main = tmp.path().join("src/main.cpp"); - write(&main, "#include \n"); - let seeds = std::slice::from_ref(&main); - let mut state = WalkState::new(); - - let res = walk_with_state(seeds, std::slice::from_ref(&first), &mut state); - assert_eq!(res.unresolved, ["only_second.h"]); - - let mut fresh = WalkState { - resolver: std::mem::take(&mut state.resolver), - ..WalkState::default() - }; - let res = walk_with_state(seeds, &[first, second.clone()], &mut fresh); - assert!(res.unresolved.is_empty()); - assert!( - res.reached - .contains(&std::fs::canonicalize(second.join("only_second.h")).unwrap()) - ); - } - - #[test] - fn w32_first_component_in_several_roots_keeps_search_order() { - let tmp = tempdir(); - let (one, two, three) = ( - tmp.path().join("one"), - tmp.path().join("two"), - tmp.path().join("three"), - ); - std::fs::create_dir_all(one.join("a")).unwrap(); - write(&two.join("a/x.h"), ""); - write(&three.join("a/x.h"), ""); - let main = tmp.path().join("src/main.cpp"); - write(&main, "#include \n"); - - let res = walk(std::slice::from_ref(&main), &[one, two.clone(), three]); - - assert!( - res.reached - .contains(&std::fs::canonicalize(two.join("a/x.h")).unwrap()) - ); - assert!(res.unresolved.is_empty()); - } - - #[test] - fn w33_dot_dot_names_still_resolve() { - let tmp = tempdir(); - let inc = tmp.path().join("inc"); - std::fs::create_dir_all(inc.join("sub")).unwrap(); - write(&inc.join("up.h"), ""); - let main = tmp.path().join("src/main.cpp"); - write(&main, "#include \n"); - - let res = walk(std::slice::from_ref(&main), std::slice::from_ref(&inc)); - - assert!( - res.reached - .contains(&std::fs::canonicalize(inc.join("up.h")).unwrap()) - ); - } - - #[test] - fn w34_missing_search_path_is_skipped() { - let tmp = tempdir(); - let inc = tmp.path().join("inc"); - write(&inc.join("x.h"), ""); - let main = tmp.path().join("src/main.cpp"); - write(&main, "#include \n"); - - let res = walk( - std::slice::from_ref(&main), - &[tmp.path().join("absent"), inc.clone()], - ); - - assert!( - res.reached - .contains(&std::fs::canonicalize(inc.join("x.h")).unwrap()) - ); - } } diff --git a/crates/fbuild-library-select/src/lib.rs b/crates/fbuild-library-select/src/lib.rs index 269ab891..449eeed9 100644 --- a/crates/fbuild-library-select/src/lib.rs +++ b/crates/fbuild-library-select/src/lib.rs @@ -21,7 +21,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::path::{Path, PathBuf}; use fbuild_header_scan::{ - WalkState, active_defines_with_known, collect_defined_macro_names_with, walk_with_state, + WalkState, active_defines_with_known, collect_defined_macro_names, walk_with_state, walk_with_state_active_known, }; use fbuild_packages::library::FrameworkLibrary; @@ -160,17 +160,14 @@ pub fn resolve_with_stats_active_declared( } } } - let mut state = WalkState::new(); - let defined_somewhere = collect_defined_macro_names_with(seeds, &full_search_paths, &mut state); + let defined_somewhere = collect_defined_macro_names(seeds, &full_search_paths); let effective_defines = seed_defines(seeds, defines, &defined_somewhere); - resolve_with_state( + resolve_with_stats_impl_declared( seeds, project_search_paths, libraries, Some(&effective_defines), declared, - state, - Some(defined_somewhere), ) } @@ -260,32 +257,11 @@ fn resolve_with_stats_impl_declared( libraries: &[FrameworkLibrary], defines: Option<&HashMap>, declared: &[String], -) -> (Selection, ResolveStats) { - resolve_with_state( - seeds, - project_search_paths, - libraries, - defines, - declared, - WalkState::new(), - None, - ) -} - -/// The resolver proper. `defined_somewhere`, when the caller already walked -/// the corpus for it with `state`, is reused instead of walked again. -fn resolve_with_state( - seeds: &[PathBuf], - project_search_paths: &[PathBuf], - libraries: &[FrameworkLibrary], - defines: Option<&HashMap>, - declared: &[String], - mut state: WalkState, - defined_somewhere: Option>, ) -> (Selection, ResolveStats) { let mut selected: BTreeSet = BTreeSet::new(); let mut all_included: BTreeSet = BTreeSet::new(); let mut all_unresolved: BTreeSet = BTreeSet::new(); + let mut state = WalkState::new(); let mut pass_count: usize = 0; let canon_lib_dirs: Vec> = libraries @@ -350,10 +326,10 @@ fn resolve_with_state( // // Only computed when branch evaluation is on; the textual mode already // scans every arm. - let defined_somewhere = match (defines, defined_somewhere) { - (Some(_), Some(known)) => known, - (Some(_), None) => collect_defined_macro_names_with(seeds, &full_search_paths, &mut state), - (None, _) => Default::default(), + let defined_somewhere = if defines.is_some() { + collect_defined_macro_names(seeds, &full_search_paths) + } else { + Default::default() }; // Pass 1: BFS from project seeds. diff --git a/crates/fbuild-library/Cargo.toml b/crates/fbuild-library/Cargo.toml index 29914cbc..12086636 100644 --- a/crates/fbuild-library/Cargo.toml +++ b/crates/fbuild-library/Cargo.toml @@ -14,7 +14,6 @@ fbuild-core = { path = "../fbuild-core" } fbuild-paths = { path = "../fbuild-paths" } fbuild-config = { path = "../fbuild-config" } async-trait = { workspace = true } -futures = { workspace = true } tokio = { workspace = true } reqwest = { workspace = true } serde = { workspace = true } @@ -23,7 +22,6 @@ thiserror = { workspace = true } tracing = { workspace = true } tempfile = { workspace = true } sha2 = { workspace = true } -blake3 = { workspace = true } flate2 = { workspace = true } tar = { workspace = true } zip = { workspace = true } diff --git a/crates/fbuild-library/src/library/README.md b/crates/fbuild-library/src/library/README.md index 135e4000..cbc24f5e 100644 --- a/crates/fbuild-library/src/library/README.md +++ b/crates/fbuild-library/src/library/README.md @@ -9,7 +9,6 @@ Arduino library and framework dependency management: spec parsing, download, com - **`library_downloader.rs`** -- Downloads libraries from GitHub URLs or the PlatformIO registry - **`library_info.rs`** -- Scans installed libraries for include directories and source files - **`library_compiler.rs`** -- Compiles library C/C++ sources and archives into static `.a` files -- **`tu_history.rs`** -- Per-TU compile duration history under the cache root, used to dispatch the longest translation units first (FastLED/fbuild#1564) - **`library_manager.rs`** -- Top-level orchestrator: spec parsing, download, discovery, compile, archive - **`registry.rs`** -- PlatformIO registry API client with semver version constraint resolution - **`arduino_core.rs`** -- Arduino AVR Core framework package (ArduinoCore-avr from GitHub) diff --git a/crates/fbuild-library/src/library/esp32_framework/parsing.rs b/crates/fbuild-library/src/library/esp32_framework/parsing.rs index 5caea8a2..81c02515 100644 --- a/crates/fbuild-library/src/library/esp32_framework/parsing.rs +++ b/crates/fbuild-library/src/library/esp32_framework/parsing.rs @@ -107,77 +107,3 @@ pub(crate) fn extract_framework_version(url: &str) -> String { // Fallback: hash crate::cache::hash_url(url) } - -/// SDK `-D` flags from the `CPPDEFINES` block of an arduino-esp32 2.x -/// `tools/platformio-build-.py`, in the raw form `flags/defines` uses. -/// -/// Those SDK layouts ship no `flags/defines`, so this block is the only place -/// PlatformIO's SDK defines (`ESP_PLATFORM`, `IDF_VER`, `MBEDTLS_CONFIG_FILE`, -/// ...) are written down (FastLED/fbuild#1549). Entries computed from the -/// SCons environment are skipped, as are the Arduino/board-level names -/// (`ARDUINO*`, `ESP32`, `F_CPU`) that fbuild derives from the board itself. -pub(crate) fn parse_pio_cppdefines(script: &str) -> Vec { - let mut defines = Vec::new(); - let mut in_block = false; - for line in script.lines() { - let entry = line.trim(); - if !in_block { - in_block = entry.starts_with("CPPDEFINES=[") || entry.starts_with("CPPDEFINES = ["); - continue; - } - if entry.starts_with(']') { - break; - } - let entry = entry.trim_end_matches(',').trim(); - if entry.contains('%') || entry.contains('$') || entry.contains("env.") { - continue; - } - let (name, value) = match entry.strip_prefix('(').and_then(|e| e.strip_suffix(')')) { - Some(tuple) => { - let Some((name, value)) = tuple.split_once(',') else { - continue; - }; - let Some(value) = python_literal(value.trim()) else { - continue; - }; - (name.trim(), Some(value)) - } - None => (entry, None), - }; - let Some(name) = python_literal(name).filter(|n| is_sdk_define_name(n)) else { - continue; - }; - defines.push(match value { - Some(value) => format!("-D{name}={value}"), - None => format!("-D{name}"), - }); - } - defines -} - -fn is_sdk_define_name(name: &str) -> bool { - !name.is_empty() - && name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') - && !name.starts_with("ARDUINO") - && !matches!(name, "ESP32" | "F_CPU") -} - -/// A Python string literal's contents (`\\` -> `\`, `\'` -> `'`, `\"` -> `"`), -/// or an integer literal as written. -fn python_literal(token: &str) -> Option { - if !token.is_empty() && token.chars().all(|c| c.is_ascii_digit()) { - return Some(token.to_string()); - } - let quote = token.chars().next().filter(|c| matches!(c, '"' | '\''))?; - let body = token.strip_prefix(quote)?.strip_suffix(quote)?; - let mut out = String::with_capacity(body.len()); - let mut chars = body.chars(); - while let Some(c) = chars.next() { - if c == '\\' { - out.push(chars.next()?); - } else { - out.push(c); - } - } - Some(out) -} diff --git a/crates/fbuild-library/src/library/esp32_framework/sdk_paths.rs b/crates/fbuild-library/src/library/esp32_framework/sdk_paths.rs index 08537f5d..94560046 100644 --- a/crates/fbuild-library/src/library/esp32_framework/sdk_paths.rs +++ b/crates/fbuild-library/src/library/esp32_framework/sdk_paths.rs @@ -2,96 +2,11 @@ //! shipped with the ESP32 Arduino framework. use std::cmp::Ordering; -use std::collections::HashSet; use std::path::{Path, PathBuf}; use super::Esp32Framework; use super::fs_utils::{collect_archive_files, scan_include_dirs_recursive}; -use super::parsing::{parse_include_flags, parse_pio_cppdefines, split_defines}; - -/// A parsed builder script yielding fewer entries than this is treated as a -/// failed parse. An upstream format change must degrade to the tree scan, -/// never to a silently truncated include path. -const MIN_PIO_CPPPATH_ENTRIES: usize = 20; - -/// Resolve a `join(FRAMEWORK_DIR, "tools", "sdk", ...)` call from the -/// framework's PlatformIO builder script into a path relative to the -/// framework root. Returns `None` when the call is anything other than plain -/// string literals (e.g. one computed from `env.BoardConfig()`). -fn pio_join_literals(line: &str) -> Option { - let mut path = PathBuf::new(); - let mut rest = line; - while let Some(start) = rest.find('"') { - let after = &rest[start + 1..]; - let end = after.find('"')?; - path.push(&after[..end]); - rest = &after[end + 1..]; - } - if path.as_os_str().is_empty() { - return None; - } - Some(path) -} - -/// Read the include list out of the framework's own PlatformIO builder script -/// (`tools/platformio-build-.py`). -/// -/// SDK layouts that predate `flags/includes` (arduino-esp32 2.x) have no -/// machine-readable include list, but they do ship the SCons builder that -/// PlatformIO uses for exactly this framework version, and its `CPPPATH` -/// block is that list, in PlatformIO's order. -/// -/// Reconstructing the list by scanning the tree instead is wrong in both -/// directions: the depth cap misses leaves PlatformIO passes (e.g. -/// `bt/common/api/include/api`, `lwip/port/esp32/include/arch`) while still -/// emitting hundreds of dirs PlatformIO never passes. Every extra `-I` costs -/// a failed path lookup on every unresolved `#include`, which measured ~41% -/// per translation unit on ESP32-S3 (FastLED/fbuild#1537). -/// -/// Returns `None` when the script is absent or does not parse, so the caller -/// can fall back to the tree scan. -fn parse_pio_cpppath(root: &Path, mcu: &str) -> Option> { - let script = root - .join("tools") - .join(format!("platformio-build-{mcu}.py")); - let content = std::fs::read_to_string(&script).ok()?; - - let mut dirs = Vec::new(); - let mut in_block = false; - for line in content.lines() { - let trimmed = line.trim(); - if !in_block { - if trimmed.starts_with("CPPPATH=[") || trimmed.starts_with("CPPPATH = [") { - in_block = true; - } - continue; - } - if trimmed.starts_with(']') { - break; - } - if !trimmed.contains("join(FRAMEWORK_DIR") || trimmed.contains("env.") { - continue; - } - if let Some(rel) = pio_join_literals(trimmed) { - let resolved = root.join(rel); - if resolved.exists() { - dirs.push(resolved); - } - } - } - - // Dedupe before applying the threshold: the count that matters is distinct - // paths, so a script that repeats one entry enough times cannot pass the - // guard and then dedupe down to a truncated include path. - let mut seen = HashSet::new(); - dirs.retain(|dir| seen.insert(dir.clone())); - - if dirs.len() < MIN_PIO_CPPPATH_ENTRIES { - return None; - } - - Some(dirs) -} +use super::parsing::{parse_include_flags, split_defines}; /// Get the SDK directory for a given MCU. /// @@ -182,20 +97,6 @@ impl Esp32Framework { } } - // Old-layout SDK (arduino-esp32 2.x) with no `flags/includes`: use the - // include list from the framework's own PlatformIO builder script. - if let Some(mut dirs) = parse_pio_cpppath(&root, mcu) { - // The flash/PSRAM variant entry is computed from board config in - // the script, so the caller supplies it (as it does above). - if let Some(variant_dir) = sdk_memory_variant_dir(&sdk_dir, memory_type) { - let v_include = variant_dir.join("include"); - if v_include.exists() && !dirs.contains(&v_include) { - dirs.push(v_include); - } - } - return dirs; - } - // Fallback: recursively scan include/ subdirectories. // The 2.x framework (PlatformIO-compat) has deeply nested includes // under tools/sdk/{mcu}/include/ (e.g., freertos/include/freertos, @@ -326,10 +227,8 @@ impl Esp32Framework { /// Get the SDK compiler defines from `flags/defines`. /// /// Returns `-D` flags that must be passed to the compiler for SDK headers - /// to work correctly (e.g., `MBEDTLS_CONFIG_FILE`, `IDF_VER`). SDK layouts - /// without `flags/defines` (arduino-esp32 2.x) fall back to the - /// `CPPDEFINES` block of the framework's PlatformIO builder script; empty - /// when neither exists. + /// to work correctly (e.g., `MBEDTLS_CONFIG_FILE`, `IDF_VER`). + /// Returns empty if the flags file doesn't exist. /// /// Uses `split_defines` instead of `shell_split` because define values /// like `-DMBEDTLS_CONFIG_FILE=\"mbedtls/esp_config.h\"` contain escaped @@ -339,13 +238,7 @@ impl Esp32Framework { if let Ok(content) = std::fs::read_to_string(&defines_file) { return split_defines(&content); } - let script = self - .resolved_dir() - .join("tools") - .join(format!("platformio-build-{mcu}.py")); - std::fs::read_to_string(script) - .map(|content| parse_pio_cppdefines(&content)) - .unwrap_or_default() + Vec::new() } /// Get the ordered SDK linker flags from `flags/ld_flags`. diff --git a/crates/fbuild-library/src/library/esp32_framework/tests.rs b/crates/fbuild-library/src/library/esp32_framework/tests.rs index 4bccd68a..b0124b4e 100644 --- a/crates/fbuild-library/src/library/esp32_framework/tests.rs +++ b/crates/fbuild-library/src/library/esp32_framework/tests.rs @@ -2,7 +2,7 @@ use std::path::Path; use super::Esp32Framework; use super::fs_utils::{collect_archive_files, find_framework_root}; -use super::parsing::{parse_include_flags, parse_pio_cppdefines, split_defines}; +use super::parsing::{parse_include_flags, split_defines}; use crate::{CacheSubdir, Package, PackageBase}; #[test] @@ -442,217 +442,3 @@ fn test_split_defines_empty() { fn test_split_defines_single() { assert_eq!(split_defines("-DFOO=1"), vec!["-DFOO=1"]); } - -#[test] -fn old_sdk_uses_pio_builder_include_list_over_tree_scan() { - let tmp = tempfile::TempDir::new().unwrap(); - let root = tmp.path(); - // A tree scan would find the decoy (a header dir under include/) and miss - // the deeply nested leaf that only the builder script names. - let decoy = root.join("tools/sdk/esp32s3/include/decoy/include"); - std::fs::create_dir_all(&decoy).unwrap(); - std::fs::write(decoy.join("decoy.h"), "\n").unwrap(); - - let leaf = root.join("tools/sdk/esp32s3/include/bt/common/api/include/api"); - std::fs::create_dir_all(&leaf).unwrap(); - std::fs::write(leaf.join("esp_bt.h"), "\n").unwrap(); - let deep = root.join("tools/sdk/esp32s3/include/lwip/port/esp32/include"); - std::fs::create_dir_all(&deep).unwrap(); - std::fs::write(deep.join("lwipopts.h"), "\n").unwrap(); - - std::fs::create_dir_all(root.join("tools")).unwrap(); - let mut script = String::from("env.Append(\n CPPPATH=[\n"); - for i in 0..25 { - let rel = format!("tools/sdk/esp32s3/include/comp{i}/include"); - std::fs::create_dir_all(root.join(&rel)).unwrap(); - std::fs::write(root.join(&rel).join("h.h"), "\n").unwrap(); - script.push_str(&format!(" join(FRAMEWORK_DIR, \"tools\", \"sdk\", \"esp32s3\", \"include\", \"comp{i}\", \"include\"),\n")); - } - script.push_str(" join(FRAMEWORK_DIR, \"tools\", \"sdk\", \"esp32s3\", \"include\", \"bt\", \"common\", \"api\", \"include\", \"api\"),\n"); - script.push_str(" join(FRAMEWORK_DIR, \"tools\", \"sdk\", \"esp32s3\", \"include\", \"lwip\", \"port\", \"esp32\", \"include\"),\n"); - script.push_str(" join(FRAMEWORK_DIR, \"tools\", \"sdk\", \"esp32s3\", env.BoardConfig().get(\"build.flash_mode\"), \"include\"),\n"); - script.push_str(" join(FRAMEWORK_DIR, \"cores\", env.BoardConfig().get(\"build.core\"))\n ],\n)\n"); - std::fs::write(root.join("tools/platformio-build-esp32s3.py"), script).unwrap(); - - let fw = Esp32Framework { - base: PackageBase::new( - "test", - "1.0", - "http://example.com", - "http://example.com", - None, - CacheSubdir::Platforms, - tmp.path(), - ), - install_dir: Some(tmp.path().to_path_buf()), - }; - - let dirs = fw.get_sdk_include_dirs("esp32s3", None); - assert!( - dirs.contains(&deep), - "deep leaf from builder script missing" - ); - assert!( - dirs.contains(&leaf), - "nested leaf from builder script missing" - ); - assert!( - !dirs.contains(&decoy), - "tree scan decoy leaked into the builder-script list" - ); - // Order follows the script, not a sort: the script names `bt` before `lwip`. - assert!(dirs.iter().position(|d| d == &leaf) < dirs.iter().position(|d| d == &deep)); -} - -#[test] -fn old_sdk_falls_back_to_tree_scan_when_builder_script_unparseable() { - let tmp = tempfile::TempDir::new().unwrap(); - let root = tmp.path(); - let include = root.join("tools/sdk/esp32s3/include/efuse/include"); - std::fs::create_dir_all(&include).unwrap(); - std::fs::write(include.join("esp_efuse.h"), "\n").unwrap(); - std::fs::create_dir_all(root.join("tools")).unwrap(); - // Truncated/substituted CPPPATH that yields too few entries to trust. - std::fs::write( - root.join("tools/platformio-build-esp32s3.py"), - "env.Append(\n CPPPATH=[\n join(FRAMEWORK_DIR, \"cores\")\n ],\n)\n", - ) - .unwrap(); - - let fw = Esp32Framework { - base: PackageBase::new( - "test", - "1.0", - "http://example.com", - "http://example.com", - None, - CacheSubdir::Platforms, - tmp.path(), - ), - install_dir: Some(tmp.path().to_path_buf()), - }; - - let dirs = fw.get_sdk_include_dirs("esp32s3", None); - assert!(dirs.contains(&include), "tree-scan fallback did not run"); -} - -#[test] -fn old_sdk_rejects_builder_script_padded_with_duplicate_entries() { - let tmp = tempfile::TempDir::new().unwrap(); - let root = tmp.path(); - // A dir only the tree scan would find, used to prove the fallback ran. - let scanned = root.join("tools/sdk/esp32s3/include/efuse/include"); - std::fs::create_dir_all(&scanned).unwrap(); - std::fs::write(scanned.join("esp_efuse.h"), "\n").unwrap(); - - let dup = root.join("tools/sdk/esp32s3/include/only/include"); - std::fs::create_dir_all(&dup).unwrap(); - std::fs::write(dup.join("only.h"), "\n").unwrap(); - - std::fs::create_dir_all(root.join("tools")).unwrap(); - // Same entry repeated past the minimum: distinct-path count is 1, so this - // must not be trusted even though the line count clears the threshold. - let mut script = String::from("env.Append(\n CPPPATH=[\n"); - for _ in 0..30 { - script.push_str( - " join(FRAMEWORK_DIR, \"tools\", \"sdk\", \"esp32s3\", \"include\", \"only\", \"include\"),\n", - ); - } - script.push_str(" ],\n)\n"); - std::fs::write(root.join("tools/platformio-build-esp32s3.py"), script).unwrap(); - - let fw = Esp32Framework { - base: PackageBase::new( - "test", - "1.0", - "http://example.com", - "http://example.com", - None, - CacheSubdir::Platforms, - tmp.path(), - ), - install_dir: Some(tmp.path().to_path_buf()), - }; - - let dirs = fw.get_sdk_include_dirs("esp32s3", None); - assert!( - dirs.contains(&scanned), - "duplicate-padded script should have been rejected in favour of the tree scan" - ); -} - -/// The `CPPDEFINES` block of arduino-esp32 2.x `platformio-build-esp32s3.py`. -const PIO_BUILD_SCRIPT: &str = r#" -env.Append( - CPPDEFINES=[ - "HAVE_CONFIG_H", - ("MBEDTLS_CONFIG_FILE", '\\"mbedtls/esp_config.h\\"'), - "UNITY_INCLUDE_CONFIG_H", - "WITH_POSIX", - "_GNU_SOURCE", - ("IDF_VER", '\\"v4.4.7-dirty\\"'), - "ESP_PLATFORM", - "_POSIX_READER_WRITER_LOCKS", - "ARDUINO_ARCH_ESP32", - "ESP32", - ("F_CPU", "$BOARD_F_CPU"), - ("ARDUINO", 10812), - ("ARDUINO_VARIANT", '\\"%s\\"' % env.BoardConfig().get("build.variant").replace('"', "")), - "ARDUINO_PARTITION_%s" % basename(env.BoardConfig().get( - "build.partitions", "default.csv")).replace(".csv", "").replace("-", "_") - ] -) -"#; - -#[test] -fn pio_cppdefines_yield_the_sdk_defines_in_flags_defines_form() { - assert_eq!( - parse_pio_cppdefines(PIO_BUILD_SCRIPT), - [ - "-DHAVE_CONFIG_H", - r#"-DMBEDTLS_CONFIG_FILE=\"mbedtls/esp_config.h\""#, - "-DUNITY_INCLUDE_CONFIG_H", - "-DWITH_POSIX", - "-D_GNU_SOURCE", - r#"-DIDF_VER=\"v4.4.7-dirty\""#, - "-DESP_PLATFORM", - "-D_POSIX_READER_WRITER_LOCKS", - ] - ); -} - -#[test] -fn sdk_defines_fall_back_to_the_pio_build_script_without_flags_defines() { - let tmp = tempfile::TempDir::new().unwrap(); - let root = tmp.path(); - std::fs::create_dir_all(root.join("tools/sdk/esp32s3")).unwrap(); - std::fs::write( - root.join("tools/platformio-build-esp32s3.py"), - PIO_BUILD_SCRIPT, - ) - .unwrap(); - let mut fw = Esp32Framework::new(root, "esp32s3"); - fw.install_dir = Some(root.to_path_buf()); - - let defines = fw.get_sdk_defines("esp32s3"); - - assert!(defines.contains(&"-DESP_PLATFORM".to_string())); - assert_eq!(defines.len(), 8); -} - -#[test] -fn sdk_defines_prefer_flags_defines_over_the_pio_build_script() { - let tmp = tempfile::TempDir::new().unwrap(); - let root = tmp.path(); - std::fs::create_dir_all(root.join("tools/sdk/esp32s3/flags")).unwrap(); - std::fs::write(root.join("tools/sdk/esp32s3/flags/defines"), "-DFROM_FLAGS").unwrap(); - std::fs::write( - root.join("tools/platformio-build-esp32s3.py"), - PIO_BUILD_SCRIPT, - ) - .unwrap(); - let mut fw = Esp32Framework::new(root, "esp32s3"); - fw.install_dir = Some(root.to_path_buf()); - - assert_eq!(fw.get_sdk_defines("esp32s3"), ["-DFROM_FLAGS"]); -} diff --git a/crates/fbuild-library/src/library/esptool.rs b/crates/fbuild-library/src/library/esptool.rs index 9e836aef..7f0c1521 100644 --- a/crates/fbuild-library/src/library/esptool.rs +++ b/crates/fbuild-library/src/library/esptool.rs @@ -224,7 +224,7 @@ impl Esptool { // runnable. let _ = fbuild_core::platform::fs::ensure_executable(bin.as_path()); - if let Err(error) = verify_esptool_once(bin.as_path(), &install_path, None).await { + if let Err(error) = verify_esptool_binary(bin.as_path()).await { if let Err(remove_error) = remove_cached_install(&install_path) { tracing::warn!( path = %install_path.display(), @@ -274,15 +274,23 @@ impl Esptool { ) .await?; } - verify_esptool_once( - &binary, - &install_path, - self.expected_runtime_version.as_deref(), + verify_esptool_binary(&binary).await?; + let binary_arg = binary.to_string_lossy(); + let output = run_command( + &[binary_arg.as_ref(), "version"], + None, + None, + Some(std::time::Duration::from_secs(10)), ) - .await - .map_err(|error| { - FbuildError::PackageError(format!("pinned esptool archive {url}: {error}")) - })?; + .await?; + if let Some(version) = &self.expected_runtime_version { + let expected = format!("v{version}"); + if !output.stdout.contains(&expected) && !output.stderr.contains(&expected) { + return Err(FbuildError::PackageError(format!( + "pinned esptool archive {url} installed the wrong version; expected {version}" + ))); + } + } Ok(binary) } } @@ -389,59 +397,7 @@ fn remove_cached_install(install_path: &Path) -> Result<()> { /// `Path::is_file` is insufficient: a restored cache can retain a regular /// file whose interpreter or dynamic loader is unavailable, which surfaces as /// `ENOENT` only when the later `elf2image` command is spawned. -/// Left in an install whose esptool already passed [`verify_esptool_once`]. -const VERIFIED_STAMP: &str = "esptool-verified.stamp"; - -/// Run ` version` once per install instead of on every build: each -/// spawn starts a Python interpreter, which cost ~140 ms on every ESP32 build, -/// warm ones included (FastLED/fbuild#1540). The stamp records the binary's -/// size and mtime plus the expected version, so a replaced binary or a new pin -/// verifies again. Failures are never stamped. -async fn verify_esptool_once( - bin: &Path, - install: &Path, - expected_version: Option<&str>, -) -> Result<()> { - let stamp_path = install.join(VERIFIED_STAMP); - let stamp = verified_stamp(bin, expected_version); - if stamp.is_some() && std::fs::read_to_string(&stamp_path).ok() == stamp { - return Ok(()); - } - let output = verify_esptool_binary(bin).await?; - if let Some(version) = expected_version { - let expected = format!("v{version}"); - if !output.stdout.contains(&expected) && !output.stderr.contains(&expected) { - return Err(FbuildError::PackageError(format!( - "esptool at {} is not the pinned version; expected {version}", - bin.display() - ))); - } - } - if let Some(stamp) = stamp { - if let Err(error) = std::fs::write(&stamp_path, stamp) { - tracing::debug!(path = %stamp_path.display(), %error, "could not stamp verified esptool"); - } - } - Ok(()) -} - -fn verified_stamp(bin: &Path, expected_version: Option<&str>) -> Option { - let meta = std::fs::metadata(bin).ok()?; - let mtime = meta - .modified() - .ok()? - .duration_since(std::time::UNIX_EPOCH) - .ok()? - .as_nanos(); - Some(format!( - "{}\n{}\n{mtime}\n{}\n", - bin.display(), - meta.len(), - expected_version.unwrap_or("") - )) -} - -async fn verify_esptool_binary(bin: &Path) -> Result { +async fn verify_esptool_binary(bin: &Path) -> Result<()> { let bin_arg = bin.to_string_lossy(); let output = run_command( &[bin_arg.as_ref(), "version"], @@ -458,7 +414,7 @@ async fn verify_esptool_binary(bin: &Path) -> Result String { mod tests { use super::*; - /// Retry an exec that failed with ETXTBSY. A sibling test thread that - /// forks while a fake esptool is being written inherits the write fd until - /// its child execs, so even a staged-and-renamed script can be briefly busy. - macro_rules! busy_retry { - ($call:expr) => {{ - let mut attempts = 0; - loop { - let result = $call.await; - match &result { - Err(error) if attempts < 50 && error.to_string().contains("Text file busy") => { - attempts += 1; - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - } - _ => break result, - } - } - }}; - } - #[test] fn extract_version_from_pioarduino_metadata_url() { // The registry release tag (0.0.1) must NOT win over the real esptool @@ -752,91 +689,14 @@ mod tests { let tmp = tempfile::TempDir::new().unwrap(); let bin = tmp.path().join(esptool_bin_name()); - write_script( + std::fs::write( &bin, - "#!/bin/sh\n[ \"$#\" -eq 1 ] && [ \"$1\" = \"version\" ]\n", - ); - - busy_retry!(verify_esptool_binary(&bin)).unwrap(); - } - - /// A fake esptool that prints `esptool.py v4.5.1` and appends a line to - /// `calls` on every run, so tests can count spawns. - /// Install an executable script without ever exec'ing a file that is - /// open for writing: a sibling test thread forking mid-write inherits - /// the write fd, and exec then fails with ETXTBSY. - fn write_script(path: &Path, body: &str) { - let staging = path.with_extension("staging"); - std::fs::write(&staging, body).unwrap(); - fbuild_core::platform::fs::set_executable(&staging).unwrap(); - std::fs::rename(&staging, path).unwrap(); - } - - fn counting_esptool(dir: &Path) -> (NormalizedPath, NormalizedPath) { - let bin = NormalizedPath::new(dir.join(esptool_bin_name())); - let calls = NormalizedPath::new(dir.join("calls")); - write_script( - &bin, - &format!( - "#!/bin/sh\necho x >> '{}'\necho 'esptool.py v4.5.1'\n", - calls.display() - ), - ); - (bin, calls) - } - - fn spawn_count(calls: &Path) -> usize { - std::fs::read_to_string(calls) - .map(|s| s.lines().count()) - .unwrap_or(0) - } - - #[tokio::test] - async fn verify_once_spawns_esptool_only_for_the_first_build() { - if fbuild_core::platform::host::is_windows() { - return; - } - let tmp = tempfile::TempDir::new().unwrap(); - let (bin, calls) = counting_esptool(tmp.path()); - - busy_retry!(verify_esptool_once(&bin, tmp.path(), Some("4.5.1"))).unwrap(); - busy_retry!(verify_esptool_once(&bin, tmp.path(), Some("4.5.1"))).unwrap(); - busy_retry!(verify_esptool_once(&bin, tmp.path(), Some("4.5.1"))).unwrap(); - - assert_eq!(spawn_count(&calls), 1); - } - - #[tokio::test] - async fn verify_once_reverifies_a_replaced_binary() { - if fbuild_core::platform::host::is_windows() { - return; - } - let tmp = tempfile::TempDir::new().unwrap(); - let (bin, calls) = counting_esptool(tmp.path()); - busy_retry!(verify_esptool_once(&bin, tmp.path(), None)).unwrap(); - - let mut script = std::fs::read_to_string(&bin).unwrap(); - script.push_str("# replaced\n"); - write_script(&bin, &script); - busy_retry!(verify_esptool_once(&bin, tmp.path(), None)).unwrap(); - - assert_eq!(spawn_count(&calls), 2); - } - - #[tokio::test] - async fn verify_once_rejects_the_wrong_version_every_time() { - if fbuild_core::platform::host::is_windows() { - return; - } - let tmp = tempfile::TempDir::new().unwrap(); - let (bin, calls) = counting_esptool(tmp.path()); - - assert!(busy_retry!(verify_esptool_once(&bin, tmp.path(), Some("9.9.9"))).is_err()); - assert!(busy_retry!(verify_esptool_once(&bin, tmp.path(), Some("9.9.9"))).is_err()); - assert_eq!(spawn_count(&calls), 2, "a failed check must not be stamped"); + b"#!/bin/sh\n[ \"$#\" -eq 1 ] && [ \"$1\" = \"version\" ]\n", + ) + .unwrap(); + fbuild_core::platform::fs::set_executable(&bin).unwrap(); - busy_retry!(verify_esptool_once(&bin, tmp.path(), Some("4.5.1"))).unwrap(); - assert_eq!(spawn_count(&calls), 3, "a new expected version re-verifies"); + verify_esptool_binary(&bin).await.unwrap(); } /// `FBUILD_ESPTOOL_PATH` is process-global; serialize the tests that touch diff --git a/crates/fbuild-library/src/library/library_compiler.rs b/crates/fbuild-library/src/library/library_compiler.rs index fce44dd3..6f72397f 100644 --- a/crates/fbuild-library/src/library/library_compiler.rs +++ b/crates/fbuild-library/src/library/library_compiler.rs @@ -13,8 +13,6 @@ use fbuild_core::subprocess::run_command; use fbuild_core::{FbuildError, Result}; use sha2::{Digest, Sha256}; -use super::tu_history; - /// C++-only flags that must not be passed to gcc for .c files. const CXX_ONLY_PREFIXES: &[&str] = &["-std=gnu++", "-std=c++", "-fno-rtti", "-fuse-cxa-atexit"]; @@ -116,19 +114,7 @@ pub async fn compile_library( .await } -/// A build's shared compile job budget. Every compile phase that draws from -/// the same gate runs in one pool, so a library's tail no longer idles cores -/// that another library or the core could use (FastLED/fbuild#1559). -pub type JobGate = std::sync::Arc; - -/// A fresh gate admitting `jobs` concurrent compiles (at least one). -pub fn job_gate(jobs: usize) -> JobGate { - std::sync::Arc::new(tokio::sync::Semaphore::new(jobs.max(1))) -} - /// Compile all source files in a library with parallel jobs. -/// -/// Uses a gate of its own; [`compile_library_gated`] shares one. #[allow(clippy::too_many_arguments)] pub async fn compile_library_with_jobs( name: &str, @@ -154,55 +140,6 @@ pub async fn compile_library_with_jobs( // keeps the direct-subprocess path for every caller that hasn't opted in. backend: Option>, ) -> Result> { - compile_library_gated( - name, - source_files, - include_dirs, - gcc_path, - gxx_path, - ar_path, - c_flags, - cpp_flags, - output_dir, - verbose, - &job_gate(jobs), - compiler_cache, - compile_cwd, - backend, - None, - ) - .await -} - -/// [`compile_library_with_jobs`] drawing its permits from `gate`, so it can -/// share one job budget with the rest of the build (FastLED/fbuild#1559). -/// -/// Permits are taken in the spawn loop, in dispatch order: on a multi-thread -/// runtime spawned tasks first run in no particular order. Spawning stops at -/// the first failure; every spawned task is awaited before returning. -#[allow(clippy::too_many_arguments)] -pub async fn compile_library_gated( - name: &str, - source_files: &[PathBuf], - include_dirs: &[PathBuf], - gcc_path: &Path, - gxx_path: &Path, - ar_path: &Path, - c_flags: &[String], - cpp_flags: &[String], - output_dir: &Path, - verbose: bool, - gate: &JobGate, - compiler_cache: Option<&Path>, - compile_cwd: Option, - backend: Option>, - // `Some` overrides the TU-duration history root (tests only); `None` - // uses the real `fbuild_paths::get_cache_root()` (FastLED/fbuild#1564). - history_root: Option<&Path>, -) -> Result> { - let history_root_owned = history_root - .map(Path::to_path_buf) - .unwrap_or_else(fbuild_paths::get_cache_root); if source_files.is_empty() { tracing::debug!("library {} is header-only, skipping compile", name); return Ok(None); @@ -248,7 +185,7 @@ pub async fn compile_library_gated( .iter() .map(|source| object_path(source, &obj_dir)) .collect(); - let stale_sources: Vec<(PathBuf, String)> = source_files + let stale_sources: Vec = source_files .iter() .zip(all_objects.iter()) .filter_map(|(source, obj)| { @@ -261,7 +198,7 @@ pub async fn compile_library_gated( &include_flags, ); if object_needs_rebuild(source, obj, &signature).unwrap_or(true) { - Some((source.clone(), signature)) + Some(source.clone()) } else { None } @@ -281,17 +218,52 @@ pub async fn compile_library_gated( return Ok(Some(archive_path)); } - let mut stale_sources = stale_sources; - tu_history::dispatch_order_in( - &history_root_owned, - &mut stale_sources, - |(source, signature)| (NormalizedPath::new(source), signature.clone()), - ); + let jobs = jobs.max(1); + + if jobs <= 1 || stale_sources.len() <= 1 { + // Sequential path + for source in &stale_sources { + compile_one_source( + source, + &obj_dir, + gcc_path, + gxx_path, + &c_safe_flags, + &cpp_flags, + &include_flags, + name, + verbose, + compiler_cache, + compile_cwd.as_deref(), + backend.as_ref(), + ) + .await?; + } + + tracing::info!( + "archiving library {}: {} objects -> {}", + name, + all_objects.len(), + archive_path.display() + ); + archive_objects(ar_path, &all_objects, &archive_path).await?; + tracing::info!( + "compiled library {}: {} changed / {} total files -> {}", + name, + stale_sources.len(), + all_objects.len(), + archive_path.display() + ); + return Ok(Some(archive_path)); + } + + // Parallel path — use a tokio Semaphore to bound concurrency. let total = stale_sources.len(); + let thread_count = jobs.min(total); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(thread_count)); let mut tasks = tokio::task::JoinSet::new(); let compiled_count = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let failed = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); let obj_dir_owned = obj_dir.clone(); let gcc_path_owned = gcc_path.to_path_buf(); @@ -303,24 +275,9 @@ pub async fn compile_library_gated( let compiler_cache_owned = compiler_cache.map(|p| p.to_path_buf()); let compile_cwd_owned = compile_cwd.clone(); let backend_owned = backend.clone(); - let history_root_arc = std::sync::Arc::new(history_root_owned); - let mut first_error: Option = None; - for (source, signature) in stale_sources { - if failed.load(std::sync::atomic::Ordering::Relaxed) { - break; - } - let permit = match gate.clone().acquire_owned().await { - Ok(permit) => permit, - Err(e) => { - first_error = Some(format!("semaphore closed: {e}")); - break; - } - }; - // A compile may have failed while this loop waited for the permit. - if failed.load(std::sync::atomic::Ordering::Relaxed) { - break; - } + for source in stale_sources.clone() { + let sem = sem.clone(); let obj_dir_t = obj_dir_owned.clone(); let gcc_t = gcc_path_owned.clone(); let gxx_t = gxx_path_owned.clone(); @@ -332,12 +289,13 @@ pub async fn compile_library_gated( let cwd_t = compile_cwd_owned.clone(); let backend_t = backend_owned.clone(); let counter = compiled_count.clone(); - let failed_t = failed.clone(); - let history_root_t = history_root_arc.clone(); tasks.spawn(async move { - let _permit = permit; - let started = std::time::Instant::now(); - let result = compile_one_source( + let _permit = sem + .acquire() + .await + .map_err(|e| FbuildError::BuildFailed(format!("semaphore closed: {e}")))?; + let cache_ref = cache_t.as_deref(); + compile_one_source( &source, &obj_dir_t, &gcc_t, @@ -347,17 +305,11 @@ pub async fn compile_library_gated( &inc_t, &lib_name_t, verbose, - cache_t.as_deref(), + cache_ref, cwd_t.as_deref(), backend_t.as_ref(), ) - .await; - if result.is_err() { - failed_t.store(true, std::sync::atomic::Ordering::Relaxed); - } else { - tu_history::record_in(&history_root_t, &source, &signature, started.elapsed()); - } - result?; + .await?; let count = counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1; if count.is_multiple_of(20) || count == total { tracing::info!("[{}/{}] compiled [{}]", count, total, lib_name_t); @@ -366,6 +318,7 @@ pub async fn compile_library_gated( }); } + let mut first_error: Option = None; while let Some(joined) = tasks.join_next().await { match joined { Ok(Ok(())) => {} @@ -389,6 +342,7 @@ pub async fn compile_library_gated( let mut all_objects = all_objects; all_objects.sort(); // deterministic archive + // Archive tracing::info!( "archiving library {}: {} objects -> {}", name, @@ -398,10 +352,11 @@ pub async fn compile_library_gated( archive_objects(ar_path, &all_objects, &archive_path).await?; tracing::info!( - "compiled library {}: {} changed / {} total files -> {}", + "compiled library {}: {} changed / {} total files ({} threads) -> {}", name, total, all_objects.len(), + thread_count, archive_path.display() ); @@ -477,13 +432,15 @@ async fn compile_one_source( // the in-process zccache service — cached, and project-directory-independent // via #985 — mirroring fbuild-build's compile_source. Without one, keep the // historical direct-subprocess path byte-identical. - let (exit_code, stderr) = if let Some(backend) = backend { + let (success, stderr) = if let Some(backend) = backend { let sanitized = fbuild_core::compiler_flags::prepare_flags_for_exec(all_flags); - // Scratch dir for the compiler's TMP/TEMP: the library's build dir, - // never the compile cwd, which is the project root for workspace- - // relative compiles (FastLED/fbuild#1568). - let scratch = obj_dir.parent().unwrap_or(obj_dir); - let mut env = fbuild_core::subprocess::compile_env_for_build(scratch).unwrap_or_default(); + // Scratch dir for the compiler's TMP/TEMP (fbuild-owned, off the + // system temp — mirrors compile_env_for_build usage on the sketch path). + let scratch = compile_cwd + .map(Path::to_path_buf) + .or_else(|| obj.parent().map(Path::to_path_buf)) + .unwrap_or_else(|| PathBuf::from(".")); + let mut env = fbuild_core::subprocess::compile_env_for_build(&scratch).unwrap_or_default(); if let Some(root) = compile_cwd { if root.is_dir() { env.push(( @@ -500,7 +457,7 @@ async fn compile_one_source( // No @response-file: the embedded service manages long arg lists itself. let outcome = backend.compile(compiler, sanitized, cwd, env).await?; ( - outcome.exit_code, + outcome.exit_code == 0, String::from_utf8_lossy(&outcome.stderr).into_owned(), ) } else { @@ -543,12 +500,15 @@ async fn compile_one_source( Some(fbuild_core::time::REAL_BUILD_TIMEOUT), ) .await?; - (result.exit_code, result.stderr) + (result.success(), result.stderr) }; - if exit_code != 0 { - return Err(FbuildError::BuildFailed(compile_failure_message( - source, lib_name, exit_code, &stderr, + if !success { + return Err(FbuildError::BuildFailed(format!( + "failed to compile {} in library {}:\n{}", + source.display(), + lib_name, + stderr ))); } @@ -557,23 +517,6 @@ async fn compile_one_source( Ok(obj) } -/// The error for a failed library TU. The exit status is always included: a -/// compiler killed by a signal exits non-zero with no stderr, and without it -/// the failure is undiagnosable (FastLED/fbuild#1569). -fn compile_failure_message(source: &Path, lib_name: &str, exit_code: i32, stderr: &str) -> String { - let stderr = stderr.trim_end(); - let detail = if stderr.is_empty() { - "(the compiler wrote nothing to stderr)" - } else { - stderr - }; - format!( - "failed to compile {} in library {} (exit code {exit_code}):\n{detail}", - source.display(), - lib_name - ) -} - fn object_needs_rebuild(source: &Path, object: &Path, signature: &str) -> Result { if !object.exists() { return Ok(true); @@ -806,19 +749,174 @@ fn object_hash_key(source: &Path, obj_dir: &Path) -> String { fbuild_core::path::NormalizedPath::from(source).display_slash() } -/// Test helper kept here: the sibling test file may not name `PathBuf`. #[cfg(test)] -fn write_sources(dir: &Path, names: &[&str]) -> Vec { - names - .iter() - .map(|n| { - let p = dir.join(n); - std::fs::write(&p, "// src").unwrap(); - p - }) - .collect() -} +mod tests { + use super::*; + use std::time::Duration; -#[cfg(test)] -#[path = "library_compiler_tests.rs"] -mod tests; + fn test_signature() -> &'static str { + "test-signature" + } + + #[test] + fn test_is_cxx_only_flag() { + assert!(is_cxx_only_flag("-std=gnu++2b")); + assert!(is_cxx_only_flag("-std=c++17")); + assert!(is_cxx_only_flag("-fno-rtti")); + assert!(is_cxx_only_flag("-fuse-cxa-atexit")); + assert!(!is_cxx_only_flag("-std=gnu17")); + assert!(!is_cxx_only_flag("-Os")); + assert!(!is_cxx_only_flag("-DFOO")); + } + + #[test] + fn test_object_path_unique() { + let obj_dir = Path::new("/tmp/obj"); + let p1 = object_path(Path::new("/src/a/main.cpp"), obj_dir); + let p2 = object_path(Path::new("/src/b/main.cpp"), obj_dir); + assert_ne!( + p1, p2, + "different source paths should produce different object paths" + ); + } + + #[test] + fn test_object_path_extension() { + let obj_dir = Path::new("/tmp/obj"); + let p = object_path(Path::new("/src/main.cpp"), obj_dir); + assert_eq!(p.extension().unwrap(), "o"); + } + + #[tokio::test] + async fn test_build_include_flags_small() { + let tmp = tempfile::TempDir::new().unwrap(); + let dirs = vec![PathBuf::from("/a"), PathBuf::from("/b")]; + let flags = build_include_flags(&dirs, tmp.path()).await.unwrap(); + assert_eq!(flags.len(), 2); + assert!(flags[0].starts_with("-I")); + } + + #[test] + fn test_invocation_response_file_path_makes_relative_path_absolute() { + let relative = Path::new("build/tmp/test.rsp"); + let absolute = invocation_response_file_path(relative).unwrap(); + assert!(absolute.is_absolute()); + assert!(absolute.ends_with(relative)); + } + + #[test] + fn test_invocation_response_file_path_preserves_absolute_path() { + let absolute_input = std::env::current_dir().unwrap().join("build/tmp/test.rsp"); + let absolute = invocation_response_file_path(&absolute_input).unwrap(); + assert_eq!(absolute, absolute_input); + } + + #[test] + fn test_object_needs_rebuild_when_object_missing() { + let tmp = tempfile::TempDir::new().unwrap(); + let source = tmp.path().join("src.cpp"); + std::fs::write(&source, "int x;").unwrap(); + let object = tmp.path().join("src.o"); + + assert!(object_needs_rebuild(&source, &object, test_signature()).unwrap()); + } + + #[test] + fn test_object_needs_rebuild_when_source_newer() { + let tmp = tempfile::TempDir::new().unwrap(); + let source = tmp.path().join("src.cpp"); + let object = tmp.path().join("src.o"); + std::fs::write(&source, "int x;").unwrap(); + std::thread::sleep(Duration::from_millis(20)); + std::fs::write(&object, "obj").unwrap(); + std::fs::write(command_hash_path(&object), test_signature()).unwrap(); + std::thread::sleep(Duration::from_millis(20)); + std::fs::write(&source, "int y;").unwrap(); + + assert!(object_needs_rebuild(&source, &object, test_signature()).unwrap()); + } + + #[test] + fn test_object_needs_rebuild_when_object_current() { + let tmp = tempfile::TempDir::new().unwrap(); + let source = tmp.path().join("src.cpp"); + let object = tmp.path().join("src.o"); + std::fs::write(&source, "int x;").unwrap(); + std::thread::sleep(Duration::from_millis(20)); + std::fs::write(&object, "obj").unwrap(); + std::fs::write(command_hash_path(&object), test_signature()).unwrap(); + + assert!(!object_needs_rebuild(&source, &object, test_signature()).unwrap()); + } + + #[test] + fn test_object_needs_rebuild_when_header_dep_is_newer() { + let tmp = tempfile::TempDir::new().unwrap(); + let source = tmp.path().join("src.cpp"); + let header = tmp.path().join("config.h"); + let object = tmp.path().join("src.o"); + let depfile = tmp.path().join("src.d"); + + std::fs::write(&source, "#include \"config.h\"\n").unwrap(); + std::fs::write(&header, "#define X 1\n").unwrap(); + std::thread::sleep(Duration::from_millis(20)); + std::fs::write(&object, "obj").unwrap(); + std::fs::write( + &depfile, + format!( + "{}: {} {}\n", + object.display(), + source.display(), + header.display() + ), + ) + .unwrap(); + std::fs::write(command_hash_path(&object), test_signature()).unwrap(); + std::thread::sleep(Duration::from_millis(20)); + std::fs::write(&header, "#define X 2\n").unwrap(); + + assert!(object_needs_rebuild(&source, &object, test_signature()).unwrap()); + } + + #[test] + fn test_object_needs_rebuild_when_command_hash_changes() { + let tmp = tempfile::TempDir::new().unwrap(); + let source = tmp.path().join("src.cpp"); + let object = tmp.path().join("src.o"); + + std::fs::write(&source, "int x;").unwrap(); + std::thread::sleep(Duration::from_millis(20)); + std::fs::write(&object, "obj").unwrap(); + std::fs::write(command_hash_path(&object), "old-signature").unwrap(); + + assert!(object_needs_rebuild(&source, &object, test_signature()).unwrap()); + } + + #[test] + fn test_archive_is_up_to_date_when_archive_newer_than_all_objects() { + let tmp = tempfile::TempDir::new().unwrap(); + let object_a = tmp.path().join("a.o"); + let object_b = tmp.path().join("b.o"); + let archive = tmp.path().join("libx.a"); + std::fs::write(&object_a, "a").unwrap(); + std::fs::write(&object_b, "b").unwrap(); + std::thread::sleep(Duration::from_millis(20)); + std::fs::write(&archive, "archive").unwrap(); + + assert!(archive_is_up_to_date(&archive, &[object_a, object_b]).unwrap()); + } + + #[test] + fn test_archive_is_not_up_to_date_when_object_newer() { + let tmp = tempfile::TempDir::new().unwrap(); + let object = tmp.path().join("a.o"); + let archive = tmp.path().join("libx.a"); + std::fs::write(&object, "a").unwrap(); + std::thread::sleep(Duration::from_millis(20)); + std::fs::write(&archive, "archive").unwrap(); + std::thread::sleep(Duration::from_millis(20)); + std::fs::write(&object, "newer").unwrap(); + + assert!(!archive_is_up_to_date(&archive, &[object]).unwrap()); + } +} diff --git a/crates/fbuild-library/src/library/library_compiler_tests.rs b/crates/fbuild-library/src/library/library_compiler_tests.rs deleted file mode 100644 index 6b227e51..00000000 --- a/crates/fbuild-library/src/library/library_compiler_tests.rs +++ /dev/null @@ -1,735 +0,0 @@ -use super::*; -use std::time::Duration; - -fn test_signature() -> &'static str { - "test-signature" -} - -#[test] -fn test_is_cxx_only_flag() { - assert!(is_cxx_only_flag("-std=gnu++2b")); - assert!(is_cxx_only_flag("-std=c++17")); - assert!(is_cxx_only_flag("-fno-rtti")); - assert!(is_cxx_only_flag("-fuse-cxa-atexit")); - assert!(!is_cxx_only_flag("-std=gnu17")); - assert!(!is_cxx_only_flag("-Os")); - assert!(!is_cxx_only_flag("-DFOO")); -} - -#[test] -fn test_object_path_unique() { - let obj_dir = Path::new("/tmp/obj"); - let p1 = object_path(Path::new("/src/a/main.cpp"), obj_dir); - let p2 = object_path(Path::new("/src/b/main.cpp"), obj_dir); - assert_ne!( - p1, p2, - "different source paths should produce different object paths" - ); -} - -#[test] -fn test_object_path_extension() { - let obj_dir = Path::new("/tmp/obj"); - let p = object_path(Path::new("/src/main.cpp"), obj_dir); - assert_eq!(p.extension().unwrap(), "o"); -} - -#[tokio::test] -async fn test_build_include_flags_small() { - let tmp = tempfile::TempDir::new().unwrap(); - let dirs = vec![Path::new("/a").to_path_buf(), Path::new("/b").to_path_buf()]; - let flags = build_include_flags(&dirs, tmp.path()).await.unwrap(); - assert_eq!(flags.len(), 2); - assert!(flags[0].starts_with("-I")); -} - -#[test] -fn test_invocation_response_file_path_makes_relative_path_absolute() { - let relative = Path::new("build/tmp/test.rsp"); - let absolute = invocation_response_file_path(relative).unwrap(); - assert!(absolute.is_absolute()); - assert!(absolute.ends_with(relative)); -} - -#[test] -fn test_invocation_response_file_path_preserves_absolute_path() { - let absolute_input = std::env::current_dir().unwrap().join("build/tmp/test.rsp"); - let absolute = invocation_response_file_path(&absolute_input).unwrap(); - assert_eq!(absolute, absolute_input); -} - -#[test] -fn test_object_needs_rebuild_when_object_missing() { - let tmp = tempfile::TempDir::new().unwrap(); - let source = tmp.path().join("src.cpp"); - std::fs::write(&source, "int x;").unwrap(); - let object = tmp.path().join("src.o"); - - assert!(object_needs_rebuild(&source, &object, test_signature()).unwrap()); -} - -#[test] -fn test_object_needs_rebuild_when_source_newer() { - let tmp = tempfile::TempDir::new().unwrap(); - let source = tmp.path().join("src.cpp"); - let object = tmp.path().join("src.o"); - std::fs::write(&source, "int x;").unwrap(); - std::thread::sleep(Duration::from_millis(20)); - std::fs::write(&object, "obj").unwrap(); - std::fs::write(command_hash_path(&object), test_signature()).unwrap(); - std::thread::sleep(Duration::from_millis(20)); - std::fs::write(&source, "int y;").unwrap(); - - assert!(object_needs_rebuild(&source, &object, test_signature()).unwrap()); -} - -#[test] -fn test_object_needs_rebuild_when_object_current() { - let tmp = tempfile::TempDir::new().unwrap(); - let source = tmp.path().join("src.cpp"); - let object = tmp.path().join("src.o"); - std::fs::write(&source, "int x;").unwrap(); - std::thread::sleep(Duration::from_millis(20)); - std::fs::write(&object, "obj").unwrap(); - std::fs::write(command_hash_path(&object), test_signature()).unwrap(); - - assert!(!object_needs_rebuild(&source, &object, test_signature()).unwrap()); -} - -#[test] -fn test_object_needs_rebuild_when_header_dep_is_newer() { - let tmp = tempfile::TempDir::new().unwrap(); - let source = tmp.path().join("src.cpp"); - let header = tmp.path().join("config.h"); - let object = tmp.path().join("src.o"); - let depfile = tmp.path().join("src.d"); - - std::fs::write(&source, "#include \"config.h\"\n").unwrap(); - std::fs::write(&header, "#define X 1\n").unwrap(); - std::thread::sleep(Duration::from_millis(20)); - std::fs::write(&object, "obj").unwrap(); - std::fs::write( - &depfile, - format!( - "{}: {} {}\n", - object.display(), - source.display(), - header.display() - ), - ) - .unwrap(); - std::fs::write(command_hash_path(&object), test_signature()).unwrap(); - std::thread::sleep(Duration::from_millis(20)); - std::fs::write(&header, "#define X 2\n").unwrap(); - - assert!(object_needs_rebuild(&source, &object, test_signature()).unwrap()); -} - -#[test] -fn test_object_needs_rebuild_when_command_hash_changes() { - let tmp = tempfile::TempDir::new().unwrap(); - let source = tmp.path().join("src.cpp"); - let object = tmp.path().join("src.o"); - - std::fs::write(&source, "int x;").unwrap(); - std::thread::sleep(Duration::from_millis(20)); - std::fs::write(&object, "obj").unwrap(); - std::fs::write(command_hash_path(&object), "old-signature").unwrap(); - - assert!(object_needs_rebuild(&source, &object, test_signature()).unwrap()); -} - -#[test] -fn test_archive_is_up_to_date_when_archive_newer_than_all_objects() { - let tmp = tempfile::TempDir::new().unwrap(); - let object_a = tmp.path().join("a.o"); - let object_b = tmp.path().join("b.o"); - let archive = tmp.path().join("libx.a"); - std::fs::write(&object_a, "a").unwrap(); - std::fs::write(&object_b, "b").unwrap(); - std::thread::sleep(Duration::from_millis(20)); - std::fs::write(&archive, "archive").unwrap(); - - assert!(archive_is_up_to_date(&archive, &[object_a, object_b]).unwrap()); -} - -#[test] -fn test_archive_is_not_up_to_date_when_object_newer() { - let tmp = tempfile::TempDir::new().unwrap(); - let object = tmp.path().join("a.o"); - let archive = tmp.path().join("libx.a"); - std::fs::write(&object, "a").unwrap(); - std::thread::sleep(Duration::from_millis(20)); - std::fs::write(&archive, "archive").unwrap(); - std::thread::sleep(Duration::from_millis(20)); - std::fs::write(&object, "newer").unwrap(); - - assert!(!archive_is_up_to_date(&archive, &[object]).unwrap()); -} - -// ---- FastLED/fbuild#1559 / #1560 shared-gate scheduling tests ---- - -/// Install a fake `ar` shell script at `path` that creates its 2nd arg -/// (the archive) as an empty file. Installed via staging + rename to -/// avoid ETXTBSY under parallel tests. -fn install_fake_ar(path: &Path) { - let script = "#!/bin/sh\n# argv: rcs \ntouch \"$2\"\n"; - let staging = path.with_extension("staging"); - std::fs::write(&staging, script).unwrap(); - fbuild_core::platform::fs::set_executable(&staging).unwrap(); - std::fs::rename(&staging, path).unwrap(); -} - -/// A fake [`LibCompileBackend`] that tracks concurrency, start order, and -/// writes the object file, failing for sources whose file name starts -/// with "bad". -struct FakeBackend { - in_flight: std::sync::Arc, - max_in_flight: std::sync::Arc, - start_order: std::sync::Arc>>, - call_count: std::sync::Arc, -} - -impl FakeBackend { - fn new() -> Self { - Self { - in_flight: std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)), - max_in_flight: std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)), - start_order: std::sync::Arc::new(std::sync::Mutex::new(Vec::new())), - call_count: std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)), - } - } -} - -#[async_trait::async_trait] -impl LibCompileBackend for FakeBackend { - async fn compile( - &self, - _compiler: &Path, - args: Vec, - cwd: NormalizedPath, - _env: Vec<(String, String)>, - ) -> Result { - self.call_count - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - - // Locate the source arg (before "-c" wouldn't help; source is the - // arg right after "-c", obj is the arg right after "-o"). - let source_arg = args - .iter() - .position(|a| a == "-c") - .and_then(|i| args.get(i + 1)) - .cloned() - .unwrap_or_default(); - let obj_arg = args - .iter() - .position(|a| a == "-o") - .and_then(|i| args.get(i + 1)) - .cloned() - .unwrap_or_default(); - let source_name = Path::new(&source_arg) - .file_name() - .unwrap_or_default() - .to_string_lossy() - .to_string(); - - let current = self - .in_flight - .fetch_add(1, std::sync::atomic::Ordering::SeqCst) - + 1; - self.max_in_flight - .fetch_max(current, std::sync::atomic::Ordering::SeqCst); - self.start_order.lock().unwrap().push(source_name.clone()); - - tokio::time::sleep(Duration::from_millis(40)).await; - - self.in_flight - .fetch_sub(1, std::sync::atomic::Ordering::SeqCst); - - if source_name.starts_with("bad") { - return Ok(LibCompileOutcome { - exit_code: 1, - stdout: Vec::new(), - stderr: b"boom".to_vec(), - }); - } - - let obj_path = Path::new(&obj_arg); - let obj_path = if obj_path.is_absolute() { - obj_path.to_path_buf() - } else { - cwd.as_path().join(obj_path) - }; - if let Some(parent) = obj_path.parent() { - std::fs::create_dir_all(parent).unwrap(); - } - std::fs::write(&obj_path, "obj").unwrap(); - - Ok(LibCompileOutcome { - exit_code: 0, - stdout: Vec::new(), - stderr: Vec::new(), - }) - } -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn test_gate_shared_across_two_libraries_caps_concurrency() { - if fbuild_core::platform::host::is_windows() { - return; - } - let tmp = tempfile::TempDir::new().unwrap(); - let ar_path = tmp.path().join("ar"); - install_fake_ar(&ar_path); - - let gate = job_gate(2); - let backend = std::sync::Arc::new(FakeBackend::new()); - - let src_dir_a = tmp.path().join("a_src"); - let src_dir_b = tmp.path().join("b_src"); - std::fs::create_dir_all(&src_dir_a).unwrap(); - std::fs::create_dir_all(&src_dir_b).unwrap(); - let sources_a = write_sources(&src_dir_a, &["a1.cpp", "a2.cpp", "a3.cpp"]); - let sources_b = write_sources(&src_dir_b, &["b1.cpp", "b2.cpp", "b3.cpp"]); - - let out_a = tmp.path().join("out_a"); - let out_b = tmp.path().join("out_b"); - - let backend_a: std::sync::Arc = backend.clone(); - let backend_b: std::sync::Arc = backend.clone(); - - let fut_a = compile_library_gated( - "liba", - &sources_a, - &[], - Path::new("/fake/gcc"), - Path::new("/fake/g++"), - &ar_path, - &[], - &[], - &out_a, - false, - &gate, - None, - None, - Some(backend_a), - Some(tmp.path()), - ); - let fut_b = compile_library_gated( - "libb", - &sources_b, - &[], - Path::new("/fake/gcc"), - Path::new("/fake/g++"), - &ar_path, - &[], - &[], - &out_b, - false, - &gate, - None, - None, - Some(backend_b), - Some(tmp.path()), - ); - - let (res_a, res_b) = tokio::join!(fut_a, fut_b); - let archive_a = res_a.unwrap().unwrap(); - let archive_b = res_b.unwrap().unwrap(); - assert!(archive_a.exists()); - assert!(archive_b.exists()); - - let max = backend - .max_in_flight - .load(std::sync::atomic::Ordering::SeqCst); - assert!( - max <= 2, - "max in-flight compiles ({max}) exceeded the shared gate's 2 permits" - ); - assert!(max >= 1); -} - -#[tokio::test] -async fn test_dispatch_order_cpp_before_c_before_asm() { - if fbuild_core::platform::host::is_windows() { - return; - } - let tmp = tempfile::TempDir::new().unwrap(); - let ar_path = tmp.path().join("ar"); - install_fake_ar(&ar_path); - - let src_dir = tmp.path().join("src"); - std::fs::create_dir_all(&src_dir).unwrap(); - let sources = write_sources(&src_dir, &["a.c", "b.S", "c.cpp", "d.c", "e.cpp"]); - - let gate = job_gate(1); - let backend = std::sync::Arc::new(FakeBackend::new()); - let backend_dyn: std::sync::Arc = backend.clone(); - let out = tmp.path().join("out"); - - let archive = compile_library_gated( - "libx", - &sources, - &[], - Path::new("/fake/gcc"), - Path::new("/fake/g++"), - &ar_path, - &[], - &[], - &out, - false, - &gate, - None, - None, - Some(backend_dyn), - Some(tmp.path()), - ) - .await - .unwrap() - .unwrap(); - assert!(archive.exists()); - - let order = backend.start_order.lock().unwrap().clone(); - assert_eq!(order, vec!["c.cpp", "e.cpp", "a.c", "d.c", "b.S"]); -} - -#[tokio::test] -async fn test_failure_stops_spawning_and_releases_permit() { - if fbuild_core::platform::host::is_windows() { - return; - } - let tmp = tempfile::TempDir::new().unwrap(); - let ar_path = tmp.path().join("ar"); - install_fake_ar(&ar_path); - - let src_dir = tmp.path().join("src"); - std::fs::create_dir_all(&src_dir).unwrap(); - let sources = write_sources(&src_dir, &["bad.cpp", "ok1.cpp", "ok2.cpp"]); - - let gate = job_gate(1); - let backend = std::sync::Arc::new(FakeBackend::new()); - let backend_dyn: std::sync::Arc = backend.clone(); - let out = tmp.path().join("out"); - - let result = compile_library_gated( - "libx", - &sources, - &[], - Path::new("/fake/gcc"), - Path::new("/fake/g++"), - &ar_path, - &[], - &[], - &out, - false, - &gate, - None, - None, - Some(backend_dyn), - Some(tmp.path()), - ) - .await; - - assert!(result.is_err(), "expected an error from the failed compile"); - let order = backend.start_order.lock().unwrap().clone(); - assert_eq!( - order, - ["bad.cpp"], - "nothing may start once a compile has failed, got order {order:?}" - ); - assert_eq!( - gate.available_permits(), - 1, - "the permit held by the failed task must be released back to the gate" - ); -} - -#[tokio::test] -async fn test_up_to_date_skips_recompile() { - if fbuild_core::platform::host::is_windows() { - return; - } - let tmp = tempfile::TempDir::new().unwrap(); - let ar_path = tmp.path().join("ar"); - install_fake_ar(&ar_path); - - let src_dir = tmp.path().join("src"); - std::fs::create_dir_all(&src_dir).unwrap(); - let sources = write_sources(&src_dir, &["only.cpp"]); - - let gate = job_gate(1); - let backend = std::sync::Arc::new(FakeBackend::new()); - let out = tmp.path().join("out"); - - let backend_dyn1: std::sync::Arc = backend.clone(); - let archive1 = compile_library_gated( - "libx", - &sources, - &[], - Path::new("/fake/gcc"), - Path::new("/fake/g++"), - &ar_path, - &[], - &[], - &out, - false, - &gate, - None, - None, - Some(backend_dyn1), - Some(tmp.path()), - ) - .await - .unwrap() - .unwrap(); - assert!(archive1.exists()); - let first_count = backend.call_count.load(std::sync::atomic::Ordering::SeqCst); - assert_eq!(first_count, 1); - - let backend_dyn2: std::sync::Arc = backend.clone(); - let archive2 = compile_library_gated( - "libx", - &sources, - &[], - Path::new("/fake/gcc"), - Path::new("/fake/g++"), - &ar_path, - &[], - &[], - &out, - false, - &gate, - None, - None, - Some(backend_dyn2), - Some(tmp.path()), - ) - .await - .unwrap() - .unwrap(); - assert_eq!(archive2, archive1); - let second_count = backend.call_count.load(std::sync::atomic::Ordering::SeqCst); - assert_eq!( - second_count, first_count, - "second compile of an unchanged library should not call the backend again" - ); -} - -/// FastLED/fbuild#1564: with recorded history showing `z_slow.cpp` is by far -/// the longest TU, it must be dispatched first even though it sorts last in -/// both source order and the alphabetic/extension fallback order. The -/// history root is threaded explicitly (`compile_library_gated`'s last -/// argument) rather than via `FBUILD_CACHE_DIR`, so this test stays hermetic -/// under `cargo test`'s parallel test threads. -#[tokio::test] -async fn test_dispatch_order_uses_recorded_history() { - if fbuild_core::platform::host::is_windows() { - return; - } - let history_dir = tempfile::TempDir::new().unwrap(); - - let tmp = tempfile::TempDir::new().unwrap(); - let ar_path = tmp.path().join("ar"); - install_fake_ar(&ar_path); - - let src_dir = tmp.path().join("src"); - std::fs::create_dir_all(&src_dir).unwrap(); - let sources = write_sources(&src_dir, &["a_fast.cpp", "b_fast.cpp", "z_slow.cpp"]); - - let gcc = Path::new("/fake/gcc"); - let gxx = Path::new("/fake/g++"); - let signature = build_rebuild_signature(gxx, &[], &[]); - let z_slow = sources.iter().find(|s| s.ends_with("z_slow.cpp")).unwrap(); - tu_history::record_in( - history_dir.path(), - z_slow, - &signature, - Duration::from_secs(20), - ); - - let gate = job_gate(1); - let backend = std::sync::Arc::new(FakeBackend::new()); - let backend_dyn: std::sync::Arc = backend.clone(); - let out = tmp.path().join("out"); - - let archive = compile_library_gated( - "libx", - &sources, - &[], - gcc, - gxx, - &ar_path, - &[], - &[], - &out, - false, - &gate, - None, - None, - Some(backend_dyn), - Some(history_dir.path()), - ) - .await - .unwrap() - .unwrap(); - assert!(archive.exists()); - - let order = backend.start_order.lock().unwrap().clone(); - assert_eq!( - order[0], "z_slow.cpp", - "longest recorded TU must start first, got {order:?}" - ); - - // A history record now exists for every compiled TU. - for source in &sources { - assert!( - tu_history::lookup_in(history_dir.path(), source, &signature).is_some(), - "expected a recorded duration for {source:?}" - ); - } -} - -/// FastLED/fbuild#1564 acceptance criterion: TU-duration history changes -/// dispatch order only — it must never change what gets built. Compiling the -/// same sources with no history and with a recorded history produces -/// byte-identical `.cmdhash` files and the same object/archive contents. -#[tokio::test] -async fn history_has_no_effect_on_objects_or_signatures() { - if fbuild_core::platform::host::is_windows() { - return; - } - // Returns (archive exists, sorted `.cmdhash` contents, compile start order). - let run = |history_root: Option| async move { - let tmp = tempfile::TempDir::new().unwrap(); - let ar_path = tmp.path().join("ar"); - install_fake_ar(&ar_path); - let src_dir = tmp.path().join("src"); - std::fs::create_dir_all(&src_dir).unwrap(); - let sources = write_sources(&src_dir, &["a.cpp", "b.cpp", "c.c"]); - - if let Some(dir) = &history_root { - // `c.c` would otherwise dispatch last (C after C++); a large - // recorded duration must move it to the front. If history - // secretly changed what gets built, this would also show up in - // the returned cmdhashes/archive below. - let c_source = sources.iter().find(|s| s.ends_with("c.c")).unwrap(); - let signature = build_rebuild_signature(Path::new("/fake/gcc"), &[], &[]); - tu_history::record_in(dir.path(), c_source, &signature, Duration::from_secs(99)); - } - - let gate = job_gate(1); - let backend = std::sync::Arc::new(FakeBackend::new()); - let backend_dyn: std::sync::Arc = backend.clone(); - let out = tmp.path().join("out"); - let archive = compile_library_gated( - "libx", - &sources, - &[], - Path::new("/fake/gcc"), - Path::new("/fake/g++"), - &ar_path, - &[], - &[], - &out, - false, - &gate, - None, - None, - Some(backend_dyn), - history_root.as_ref().map(|d| d.path()), - ) - .await - .unwrap() - .unwrap(); - let obj_dir = out.join("obj"); - // Object file *names* are hashed from their absolute source path - // (which differs between the two runs' TempDirs); compare the - // `.cmdhash` **contents** only, since that's what "no effect on - // signatures" actually means. - let mut cmdhashes: Vec = std::fs::read_dir(&obj_dir) - .unwrap() - .filter_map(|e| e.ok()) - .filter(|e| e.path().extension().is_some_and(|ext| ext == "cmdhash")) - .map(|e| std::fs::read_to_string(e.path()).unwrap()) - .collect(); - cmdhashes.sort(); - let start_order = backend.start_order.lock().unwrap().clone(); - (archive.exists(), cmdhashes, start_order) - }; - - let (no_history_built, no_history_cmdhashes, no_history_order) = run(None).await; - let (with_history_built, with_history_cmdhashes, with_history_order) = - run(Some(tempfile::TempDir::new().unwrap())).await; - - // History changed *order* (proving the plumbing actually reordered work)... - assert_ne!( - no_history_order, with_history_order, - "seeding a recorded duration for c.c should have changed dispatch order" - ); - assert_eq!( - with_history_order[0], "c.c", - "c.c's huge recorded duration must dispatch it first, got {with_history_order:?}" - ); - // ...but never what got built. - assert_eq!(no_history_built, with_history_built); - assert_eq!(no_history_cmdhashes, with_history_cmdhashes); -} - -#[test] -fn failure_message_always_carries_the_exit_status() { - let silent = compile_failure_message(Path::new("/src/a.cpp"), "fastled", 137, ""); - assert!(silent.contains("exit code 137"), "{silent}"); - assert!(silent.contains("wrote nothing to stderr"), "{silent}"); - let noisy = - compile_failure_message(Path::new("/src/a.cpp"), "fastled", 1, "a.cpp:1: error: x\n"); - assert!( - noisy.contains("exit code 1") && noisy.ends_with("a.cpp:1: error: x"), - "{noisy}" - ); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn backend_compiles_get_their_temp_dir_under_the_library_build_dir() { - if fbuild_core::platform::host::is_windows() { - return; - } - let tmp = tempfile::TempDir::new().unwrap(); - let history = tempfile::TempDir::new().unwrap(); - let project = tmp.path().join("project"); - let src_dir = project.join("lib").join("src"); - std::fs::create_dir_all(&src_dir).unwrap(); - let sources = write_sources(&src_dir, &["a.cpp"]); - let out = project.join("build").join("fw_libs"); - let ar = tmp.path().join("ar"); - install_fake_ar(&ar); - let backend = std::sync::Arc::new(FakeBackend::new()); - let backend_dyn: std::sync::Arc = backend.clone(); - - let _ = compile_library_gated( - "lib", - &sources, - &[], - Path::new("gcc"), - Path::new("g++"), - &ar, - &[], - &[], - &out, - false, - &job_gate(1), - None, - Some(project.clone()), - Some(backend_dyn), - Some(history.path()), - ) - .await; - - assert!( - !project.join(".compile-tmp").exists(), - "no temp dir in the project root" - ); - assert!( - out.join(".compile-tmp").is_dir(), - "temp dir under the library build dir" - ); -} diff --git a/crates/fbuild-library/src/library/library_manager.rs b/crates/fbuild-library/src/library/library_manager.rs index 8c123b4e..9975a26b 100644 --- a/crates/fbuild-library/src/library/library_manager.rs +++ b/crates/fbuild-library/src/library/library_manager.rs @@ -87,103 +87,15 @@ pub fn parse_lib_specs(lib_specs: &[String], lib_ignore: &[String]) -> Vec, - /// Translation units the plan compiles (LDF seeds). - pub source_files: Vec, - /// The compile work, deferred until a job gate is available. - pub plan: LibraryCompilePlan, -} - -/// Deferred compilation of resolved libraries; see [`Self::compile`]. -pub struct LibraryCompilePlan { - libraries: Vec, - all_include_dirs: Vec, - gcc_path: PathBuf, - gxx_path: PathBuf, - ar_path: PathBuf, - c_flags: Vec, - cpp_flags: Vec, - verbose: bool, - compiler_cache: Option, -} - -struct PlannedLibrary { - name: String, - sources: Vec, - build_dir: PathBuf, -} - -impl LibraryCompilePlan { - /// Whether there is nothing to compile. - pub fn is_empty(&self) -> bool { - self.libraries.is_empty() - } - - /// Names of the libraries that [`Self::compile`] turns into an archive - /// (`lib{name}.a`), in library order. A library without sources yields no - /// archive and is left out, so this is known before anything compiles. - pub fn archive_names(&self) -> Vec { - self.libraries - .iter() - .filter(|lib| !lib.sources.is_empty()) - .map(|lib| lib.name.clone()) - .collect() - } - - /// Compile every library concurrently, each TU drawing a permit from - /// `gate`. Archives come back in library order (the link order). - /// - /// Every library goes through the compiler's own up-to-date check, so an - /// unchanged library costs only stats and an edited local or symlinked - /// one is rebuilt (FastLED/fbuild#1560). - pub async fn compile(self, gate: &library_compiler::JobGate) -> Result> { - let plan = &self; - // join_all runs every compile to completion even if one fails. - let archives = futures::future::join_all(plan.libraries.iter().map(|lib| async move { - library_compiler::compile_library_gated( - &lib.name, - &lib.sources, - &plan.all_include_dirs, - &plan.gcc_path, - &plan.gxx_path, - &plan.ar_path, - &plan.c_flags, - &plan.cpp_flags, - &lib.build_dir, - plan.verbose, - gate, - plan.compiler_cache.as_deref(), - None, - None, - None, - ) - .await - })) - .await; - let mut out = Vec::new(); - for archive in archives { - out.extend(archive?); - } - Ok(out) - } -} - -/// Download (or resolve) every library and plan its compilation. +/// Ensure all library dependencies are downloaded and compiled. /// /// Flow: /// 1. Download every library with [`download_libraries`] /// 2. Collect all include dirs (needed before compilation for cross-includes) -/// 3. Plan one compile per library that has sources +/// 3. Compile each library +/// 4. Return include dirs + archives #[allow(clippy::too_many_arguments)] -pub async fn resolve_libraries( +pub async fn ensure_libraries( lib_specs: &[String], lib_ignore: &[String], gcc_path: &Path, @@ -195,94 +107,76 @@ pub async fn resolve_libraries( project_dir: &Path, libs_dir: &Path, verbose: bool, + jobs: usize, compiler_cache: Option<&Path>, -) -> Result { +) -> Result { let installed = download_libraries(lib_specs, lib_ignore, project_dir, libs_dir).await?; + if installed.is_empty() { + return Ok(LibraryResult { + include_dirs: Vec::new(), + source_files: Vec::new(), + archives: Vec::new(), + }); + } + // 3. Collect all include dirs (needed for cross-library includes) let mut all_include_dirs: Vec = base_includes.to_vec(); for lib in &installed { all_include_dirs.extend(lib.get_include_dirs()); } - let libraries = installed - .iter() - .filter_map(|lib| { - if lib.is_header_only() { - tracing::info!("library {} is header-only", lib.name); - return None; - } - Some(PlannedLibrary { - name: lib.name.clone(), - sources: lib.get_source_files(), - build_dir: lib.build_dir.clone(), - }) - }) - .collect(); + // 4. Compile each library + let mut archives = Vec::new(); + for lib in &installed { + if lib.is_header_only() { + tracing::info!("library {} is header-only", lib.name); + continue; + } - Ok(ResolvedLibraries { - // Library includes only, not base includes. - include_dirs: installed - .iter() - .flat_map(|lib| lib.get_include_dirs()) - .collect(), - source_files: installed - .iter() - .flat_map(|lib| lib.get_source_files()) - .collect(), - plan: LibraryCompilePlan { - libraries, - all_include_dirs, - gcc_path: gcc_path.to_path_buf(), - gxx_path: gxx_path.to_path_buf(), - ar_path: ar_path.to_path_buf(), - c_flags: c_flags.to_vec(), - cpp_flags: cpp_flags.to_vec(), + // Check if archive already exists + let archive = lib.archive_path(); + if archive.exists() { + tracing::debug!("library {} already compiled", lib.name); + archives.push(archive); + continue; + } + + let sources = lib.get_source_files(); + if let Some(archive_path) = library_compiler::compile_library_with_jobs( + &lib.name, + &sources, + &all_include_dirs, + gcc_path, + gxx_path, + ar_path, + c_flags, + cpp_flags, + &lib.build_dir, verbose, - compiler_cache: compiler_cache.map(Path::to_path_buf), - }, - }) -} + jobs, + compiler_cache, + None, + None, + ) + .await? + { + archives.push(archive_path); + } + } + + // Return include dirs (library includes only, not base includes) + let lib_include_dirs: Vec = installed + .iter() + .flat_map(|lib| lib.get_include_dirs()) + .collect(); + let source_files: Vec = installed + .iter() + .flat_map(|lib| lib.get_source_files()) + .collect(); -/// Ensure all library dependencies are downloaded and compiled: [`resolve_libraries`] -/// then [`LibraryCompilePlan::compile`] on a gate of `jobs` permits. -#[allow(clippy::too_many_arguments)] -pub async fn ensure_libraries( - lib_specs: &[String], - lib_ignore: &[String], - gcc_path: &Path, - gxx_path: &Path, - ar_path: &Path, - c_flags: &[String], - cpp_flags: &[String], - base_includes: &[PathBuf], - project_dir: &Path, - libs_dir: &Path, - verbose: bool, - jobs: usize, - compiler_cache: Option<&Path>, -) -> Result { - let resolved = resolve_libraries( - lib_specs, - lib_ignore, - gcc_path, - gxx_path, - ar_path, - c_flags, - cpp_flags, - base_includes, - project_dir, - libs_dir, - verbose, - compiler_cache, - ) - .await?; - let archives = resolved - .plan - .compile(&library_compiler::job_gate(jobs)) - .await?; Ok(LibraryResult { - include_dirs: resolved.include_dirs, - source_files: resolved.source_files, + include_dirs: lib_include_dirs, + source_files, archives, }) } @@ -522,7 +416,6 @@ pub fn ensure_libraries_sync( #[cfg(test)] mod tests { use super::*; - use std::time::Duration; #[test] fn test_empty_specs() { @@ -601,188 +494,4 @@ mod tests { let expected = fbuild_core::path::strip_unc_prefix(&local_src.canonicalize().unwrap()); assert_eq!(result.include_dirs, vec![expected]); } - - // ---- FastLED/fbuild#1559 / #1560 plan-level scheduling tests ---- - - /// Install a fake compiler shell script at `path` that parses `-o `, - /// touches the object file, and appends the compiled source path to - /// `log`. Installed via staging + rename to avoid ETXTBSY under parallel - /// tests. - fn install_fake_compiler(path: &Path, log: &Path) { - let script = format!( - "#!/bin/sh\n\ - obj=\"\"\n\ - src=\"\"\n\ - prev=\"\"\n\ - for arg in \"$@\"; do\n\ - \x20 if [ \"$prev\" = \"-o\" ]; then obj=\"$arg\"; fi\n\ - \x20 if [ \"$prev\" = \"-c\" ]; then src=\"$arg\"; fi\n\ - \x20 prev=\"$arg\"\n\ - done\n\ - echo \"$src\" >> \"{log}\"\n\ - mkdir -p \"$(dirname \"$obj\")\"\n\ - touch \"$obj\"\n", - log = log.display() - ); - let staging = path.with_extension("staging"); - std::fs::write(&staging, script).unwrap(); - fbuild_core::platform::fs::set_executable(&staging).unwrap(); - std::fs::rename(&staging, path).unwrap(); - } - - /// Install a fake `ar` shell script that creates its 2nd arg (the - /// archive) as an empty file. - fn install_fake_ar(path: &Path) { - let script = "#!/bin/sh\ntouch \"$2\"\n"; - let staging = path.with_extension("staging"); - std::fs::write(&staging, script).unwrap(); - fbuild_core::platform::fs::set_executable(&staging).unwrap(); - std::fs::rename(&staging, path).unwrap(); - } - - fn make_local_library(project: &Path, rel: &str, name: &str) -> PathBuf { - let lib_dir = project.join(rel); - let src = lib_dir.join("src"); - std::fs::create_dir_all(&src).unwrap(); - std::fs::write(src.join(format!("{name}.cpp")), "int f() { return 1; }\n").unwrap(); - std::fs::write(src.join(format!("{name}.h")), "int f();\n").unwrap(); - lib_dir - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn test_plan_compiles_concurrently_and_recompiles_edited_local_library() { - if fbuild_core::platform::host::is_windows() { - return; - } - let tmp = tempfile::TempDir::new().unwrap(); - let project = tmp.path().join("project"); - make_local_library(&project, "liba", "liba"); - make_local_library(&project, "libb", "libb"); - let libs_dir = tmp.path().join("build").join("libs"); - std::fs::create_dir_all(&libs_dir).unwrap(); - - let tools_dir = tmp.path().join("tools"); - std::fs::create_dir_all(&tools_dir).unwrap(); - let log = tmp.path().join("compile.log"); - let gcc = tools_dir.join("gcc"); - let gxx = tools_dir.join("g++"); - let ar = tools_dir.join("ar"); - install_fake_compiler(&gcc, &log); - install_fake_compiler(&gxx, &log); - install_fake_ar(&ar); - - let specs = vec![ - "LibA=symlink://liba".to_string(), - "LibB=symlink://libb".to_string(), - ]; - - let resolved = resolve_libraries( - &specs, - &[], - &gcc, - &gxx, - &ar, - &[], - &[], - &[], - &project, - &libs_dir, - false, - None, - ) - .await - .unwrap(); - - let archives = resolved - .plan - .compile(&library_compiler::job_gate(4)) - .await - .unwrap(); - - let expected_a = libs_dir.join("liba").join("libliba.a"); - let expected_b = libs_dir.join("libb").join("liblibb.a"); - assert_eq!(archives, vec![expected_a.clone(), expected_b.clone()]); - assert!(expected_a.exists()); - assert!(expected_b.exists()); - - let log_after_first = std::fs::read_to_string(&log).unwrap(); - assert!(log_after_first.contains("liba.cpp")); - assert!(log_after_first.contains("libb.cpp")); - - // Modify LibA's source and push its mtime into the future so the - // up-to-date check (source mtime vs. object mtime) reliably sees it - // as stale on any filesystem timestamp granularity. - let liba_src = project.join("liba").join("src").join("liba.cpp"); - std::fs::write(&liba_src, "int f() { return 2; }\n").unwrap(); - let future = std::time::SystemTime::now() + Duration::from_secs(3600); - std::fs::File::options() - .write(true) - .open(&liba_src) - .unwrap() - .set_modified(future) - .unwrap(); - - let resolved2 = resolve_libraries( - &specs, - &[], - &gcc, - &gxx, - &ar, - &[], - &[], - &[], - &project, - &libs_dir, - false, - None, - ) - .await - .unwrap(); - let archives2 = resolved2 - .plan - .compile(&library_compiler::job_gate(4)) - .await - .unwrap(); - assert_eq!(archives2, vec![expected_a, expected_b]); - - let log_after_second = std::fs::read_to_string(&log).unwrap(); - let liba_compiles = log_after_second.matches("liba.cpp").count(); - let libb_compiles = log_after_second.matches("libb.cpp").count(); - assert_eq!( - liba_compiles, 2, - "edited LibA source must be recompiled on the second resolve+compile" - ); - assert_eq!( - libb_compiles, 1, - "untouched LibB must not be recompiled on the second resolve+compile" - ); - } - - /// FastLED/fbuild#1559: the ESP32 orchestrator skips a framework library - /// that a `lib_deps` archive already provides, before anything compiles. - /// Only libraries with sources yield an archive; names keep their case. - #[test] - fn test_archive_names_skip_libraries_without_sources() { - let planned = |name: &str, sources: Vec| PlannedLibrary { - name: name.to_string(), - sources, - build_dir: PathBuf::from("build").join(name), - }; - let plan = LibraryCompilePlan { - libraries: vec![ - planned("FastLED", vec![PathBuf::from("a.cpp")]), - planned("Empty", Vec::new()), - planned("zlib", vec![PathBuf::from("z.c")]), - ], - all_include_dirs: Vec::new(), - gcc_path: PathBuf::from("gcc"), - gxx_path: PathBuf::from("g++"), - ar_path: PathBuf::from("ar"), - c_flags: Vec::new(), - cpp_flags: Vec::new(), - verbose: false, - compiler_cache: None, - }; - assert_eq!(plan.archive_names(), vec!["FastLED", "zlib"]); - } } diff --git a/crates/fbuild-library/src/library/mod.rs b/crates/fbuild-library/src/library/mod.rs index beeac4f4..9d71b437 100644 --- a/crates/fbuild-library/src/library/mod.rs +++ b/crates/fbuild-library/src/library/mod.rs @@ -30,7 +30,6 @@ pub mod samd_core; pub mod silabs_core; pub mod stm32_core; pub mod teensy_core; -pub mod tu_history; pub use apollo3_core::Apollo3Cores; pub use arduino_core::ArduinoCore; diff --git a/crates/fbuild-library/src/library/tu_history.rs b/crates/fbuild-library/src/library/tu_history.rs deleted file mode 100644 index f57760bf..00000000 --- a/crates/fbuild-library/src/library/tu_history.rs +++ /dev/null @@ -1,358 +0,0 @@ -//! Per-translation-unit compile duration history (FastLED/fbuild#1564). -//! -//! Records how long each TU took to compile, keyed on the source path plus -//! its compile signature, so a later build of the same project can dispatch -//! the longest TUs first instead of guessing from source order alone. The -//! history lives under the fbuild cache root, so it survives `clean` of the -//! build dir, and it affects dispatch **order only** — it never touches -//! object files, rebuild signatures, or cache keys. A missing or corrupt -//! record is silently treated as "no history". - -use std::time::Duration; - -use fbuild_core::path::NormalizedPath; - -/// Subdirectory of the cache root holding one small file per TU record. -const HISTORY_DIR: &str = "tu-durations"; - -/// Default estimated cost (ms) for a C++ TU with no recorded history. -const DEFAULT_CPP_ESTIMATE_MS: u64 = 1000; -/// Default estimated cost (ms) for an assembly TU with no recorded history. -const DEFAULT_ASM_ESTIMATE_MS: u64 = 10; - -/// The dispatch rank fallback used when no history is available for a TU: -/// C++ starts first, then C, then assembly (FastLED/fbuild#1537/#1553). -fn dispatch_rank(source: &std::path::Path) -> u8 { - let ext = source - .extension() - .and_then(|e| e.to_str()) - .map(str::to_ascii_lowercase); - match ext.as_deref() { - Some("c") => 1, - Some("s" | "sx" | "asm") => 2, - _ => 0, - } -} - -/// The history key for a (source, signature) pair: a blake3 hash of the -/// canonical-ish source path string plus the compile signature, so an -/// identical source compiled with a different flag set (a different -/// signature) gets its own record. -fn history_key(source: &std::path::Path, signature: &str) -> String { - let mut hasher = blake3::Hasher::new(); - hasher.update(NormalizedPath::new(source).key().as_bytes()); - hasher.update(&[0]); - hasher.update(signature.as_bytes()); - hasher.finalize().to_hex().to_string() -} - -/// Record `duration` for `(source, signature)` under the global cache root. -/// Errors are logged at debug level and otherwise ignored: history is a -/// best-effort optimization, never a build requirement. -pub fn record(source: &std::path::Path, signature: &str, duration: Duration) { - let root = fbuild_paths::get_cache_root(); - record_in(&root, source, signature, duration); -} - -/// Look up a previously recorded duration for `(source, signature)` under -/// the global cache root. Returns `None` on any missing or corrupt record. -pub fn lookup(source: &std::path::Path, signature: &str) -> Option { - let root = fbuild_paths::get_cache_root(); - lookup_in(&root, source, signature) -} - -/// A process-wide counter guaranteeing a unique temp-file name even when two -/// compiles for the same `(source, signature)` race to record (two builds of -/// the same library at once): the pid alone is not enough to disambiguate -/// concurrent writers inside one process. -static TMP_COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); - -/// [`record`], with an explicit history root — used directly by tests and by -/// callers that need a hermetic, non-global history location. -/// -/// A record already present for this key is kept if `duration` is under 25% -/// of it: a cache hit (a few ms) must never overwrite a real compile's -/// measured cost (FastLED/fbuild#1564), since the *next* build would then -/// dispatch that TU last right when it needs to compile for real. -pub fn record_in( - root: &std::path::Path, - source: &std::path::Path, - signature: &str, - duration: Duration, -) { - if let Some(previous) = lookup_in(root, source, signature) { - if duration.as_millis() * 4 < previous.as_millis() { - return; - } - } - if let Err(e) = record_in_unconditional(root, source, signature, duration) { - tracing::debug!( - "tu_history: failed to record duration for {:?}: {e}", - source - ); - } -} - -fn record_in_unconditional( - root: &std::path::Path, - source: &std::path::Path, - signature: &str, - duration: Duration, -) -> std::io::Result<()> { - let dir = root.join(HISTORY_DIR); - std::fs::create_dir_all(&dir)?; - let key = history_key(source, signature); - let final_path = dir.join(&key); - let unique = TMP_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - let tmp_path = dir.join(format!("{key}.{}.{unique}.tmp", std::process::id())); - std::fs::write(&tmp_path, duration.as_millis().to_string())?; - std::fs::rename(&tmp_path, &final_path)?; - Ok(()) -} - -/// [`lookup`], with an explicit history root — used directly by tests and by -/// callers that need a hermetic, non-global history location. -pub fn lookup_in( - root: &std::path::Path, - source: &std::path::Path, - signature: &str, -) -> Option { - let key = history_key(source, signature); - let path = root.join(HISTORY_DIR).join(key); - let contents = std::fs::read_to_string(path).ok()?; - let ms: u64 = contents.trim().parse().ok()?; - Some(Duration::from_millis(ms)) -} - -/// Sort `items` longest-estimated-first for dispatch. -/// -/// `key(item)` returns `(source, signature)` for a work item. The estimated -/// cost of an item is: -/// - its recorded duration, if `lookup` finds one; -/// - otherwise, a rank-based default: the median (upper median, for an -/// even-sized sample) of *this batch's* recorded C++ durations (if any are -/// known), else `DEFAULT_CPP_ESTIMATE_MS` (1 s) for a C++ source, half that for -/// C, and `DEFAULT_ASM_ESTIMATE_MS` (10 ms) for assembly. -/// -/// Sorting is by estimated cost **descending**. Ties break known-before-estimated -/// (a recorded duration always outranks a guess that happens to match it), -/// then by original (stable-sort) source order — so items with no history at -/// all fall back to the existing C++-before-C-before-asm, source-order -/// behavior. -pub fn dispatch_order(items: &mut Vec, key: impl Fn(&T) -> (NormalizedPath, String)) { - dispatch_order_in(&fbuild_paths::get_cache_root(), items, key); -} - -/// [`dispatch_order`], with an explicit history root — used directly by -/// tests and by callers that need a hermetic, non-global history location. -pub fn dispatch_order_in( - root: &std::path::Path, - items: &mut Vec, - key: impl Fn(&T) -> (NormalizedPath, String), -) { - let known: Vec> = items - .iter() - .map(|item| { - let (source, signature) = key(item); - lookup_in(root, source.as_path(), &signature) - }) - .collect(); - - let mut known_cpp_ms: Vec = known - .iter() - .zip(items.iter()) - .filter_map(|(d, item)| { - let (source, _) = key(item); - if dispatch_rank(source.as_path()) == 0 { - d.map(|d| d.as_millis() as u64) - } else { - None - } - }) - .collect(); - let cpp_default_ms = if known_cpp_ms.is_empty() { - DEFAULT_CPP_ESTIMATE_MS - } else { - known_cpp_ms.sort_unstable(); - known_cpp_ms[known_cpp_ms.len() / 2] - }; - - let estimate_ms = |idx: usize, source: &std::path::Path| -> u64 { - if let Some(d) = known[idx] { - return d.as_millis() as u64; - } - match dispatch_rank(source) { - 1 => cpp_default_ms / 2, - 2 => DEFAULT_ASM_ESTIMATE_MS, - _ => cpp_default_ms, - } - }; - - // Sort descending by estimated cost. A tie between a *known* duration and - // an *estimated* one favors the known value (an unknown item's estimate - // is only ever a guess, even when it happens to equal a real recorded - // duration); ties between two items of the same kind keep source order. - let mut indexed: Vec<(usize, u64, bool)> = items - .iter() - .enumerate() - .map(|(idx, item)| { - let (source, _) = key(item); - ( - idx, - estimate_ms(idx, source.as_path()), - known[idx].is_some(), - ) - }) - .collect(); - indexed.sort_by(|a, b| b.1.cmp(&a.1).then(b.2.cmp(&a.2)).then(a.0.cmp(&b.0))); - - let order: Vec = indexed.into_iter().map(|(idx, _, _)| idx).collect(); - let mut reordered: Vec> = items.drain(..).map(Some).collect(); - for idx in order { - items.push(reordered[idx].take().expect("index used once")); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::path::Path; - - #[test] - fn record_and_lookup_round_trip() { - let dir = tempfile::tempdir().unwrap(); - let source = Path::new("/proj/foo.cpp"); - record_in(dir.path(), source, "sig1", Duration::from_millis(1234)); - let found = lookup_in(dir.path(), source, "sig1"); - assert_eq!(found, Some(Duration::from_millis(1234))); - } - - #[test] - fn lookup_missing_returns_none() { - let dir = tempfile::tempdir().unwrap(); - let source = Path::new("/proj/missing.cpp"); - assert_eq!(lookup_in(dir.path(), source, "sig1"), None); - } - - #[test] - fn lookup_corrupt_returns_none() { - let dir = tempfile::tempdir().unwrap(); - let source = Path::new("/proj/foo.cpp"); - record_in(dir.path(), source, "sig1", Duration::from_millis(1234)); - let key = history_key(source, "sig1"); - std::fs::write(dir.path().join(HISTORY_DIR).join(&key), "not-a-number").unwrap(); - assert_eq!(lookup_in(dir.path(), source, "sig1"), None); - } - - #[test] - fn different_signature_is_a_different_record() { - let dir = tempfile::tempdir().unwrap(); - let source = Path::new("/proj/foo.cpp"); - record_in(dir.path(), source, "sig1", Duration::from_millis(100)); - assert_eq!(lookup_in(dir.path(), source, "sig2"), None); - } - - #[test] - fn atomic_write_leaves_no_temp_files() { - let dir = tempfile::tempdir().unwrap(); - let source = Path::new("/proj/foo.cpp"); - record_in(dir.path(), source, "sig1", Duration::from_millis(1)); - let history_dir = dir.path().join(HISTORY_DIR); - let leftovers: Vec<_> = std::fs::read_dir(&history_dir) - .unwrap() - .filter_map(|e| e.ok()) - .filter(|e| e.file_name().to_string_lossy().contains(".tmp")) - .collect(); - assert!( - leftovers.is_empty(), - "temp files left behind: {leftovers:?}" - ); - } - - #[test] - fn dispatch_order_prefers_recorded_longest_first() { - let dir = tempfile::tempdir().unwrap(); - let a = Path::new("/proj/a.cpp").to_path_buf(); - let b = Path::new("/proj/b.cpp").to_path_buf(); - let c = Path::new("/proj/c.cpp").to_path_buf(); - record_in(dir.path(), &a, "sig", Duration::from_millis(100)); - record_in(dir.path(), &b, "sig", Duration::from_millis(5000)); - record_in(dir.path(), &c, "sig", Duration::from_millis(500)); - - let mut items = vec![a.clone(), b.clone(), c.clone()]; - dispatch_order_in(dir.path(), &mut items, |p| { - (NormalizedPath::new(p), "sig".to_string()) - }); - assert_eq!(items, vec![b, c, a]); - } - - #[test] - fn dispatch_order_without_history_keeps_rank_then_source_order() { - let dir = tempfile::tempdir().unwrap(); - let cpp1 = Path::new("/proj/cpp1.cpp").to_path_buf(); - let c1 = Path::new("/proj/c1.c").to_path_buf(); - let asm1 = Path::new("/proj/asm1.s").to_path_buf(); - let cpp2 = Path::new("/proj/cpp2.cpp").to_path_buf(); - - let mut items = vec![c1.clone(), asm1.clone(), cpp1.clone(), cpp2.clone()]; - dispatch_order_in(dir.path(), &mut items, |p| { - (NormalizedPath::new(p), "sig".to_string()) - }); - assert_eq!(items, vec![cpp1, cpp2, c1, asm1]); - } - - #[test] - fn dispatch_order_mixed_known_and_unknown() { - let dir = tempfile::tempdir().unwrap(); - let known_slow = Path::new("/proj/known_slow.cpp").to_path_buf(); - let unknown_cpp = Path::new("/proj/unknown.cpp").to_path_buf(); - let known_fast = Path::new("/proj/known_fast.cpp").to_path_buf(); - - // A known duration far above the default estimate sorts first. - record_in( - dir.path(), - &known_slow, - "sig", - Duration::from_millis(20_000), - ); - record_in(dir.path(), &known_fast, "sig", Duration::from_millis(1)); - - let mut items = vec![unknown_cpp.clone(), known_slow.clone(), known_fast.clone()]; - dispatch_order_in(dir.path(), &mut items, |p| { - (NormalizedPath::new(p), "sig".to_string()) - }); - assert_eq!(items[0], known_slow); - assert_eq!(items[2], known_fast); - } - - /// FastLED/fbuild#1564: a warm zccache hit compiles in a few ms; that must - /// never overwrite a real (cold) compile's measured cost, or the next - /// build would dispatch a genuinely slow TU last right when it needs to - /// compile for real. - #[test] - fn record_in_keeps_the_slow_sample_over_a_later_fast_hit() { - let dir = tempfile::tempdir().unwrap(); - let source = Path::new("/proj/fl.fx.cpp"); - record_in(dir.path(), source, "sig", Duration::from_millis(17_000)); - record_in(dir.path(), source, "sig", Duration::from_millis(20)); - assert_eq!( - lookup_in(dir.path(), source, "sig"), - Some(Duration::from_millis(17_000)), - "a cache-hit-fast sample must not overwrite the real recorded duration" - ); - } - - /// A duration in the same ballpark as (or slower than) the prior record - /// still replaces it — only an implausibly fast sample is suspect. - #[test] - fn record_in_replaces_a_comparable_or_slower_sample() { - let dir = tempfile::tempdir().unwrap(); - let source = Path::new("/proj/fl.fx.cpp"); - record_in(dir.path(), source, "sig", Duration::from_millis(1000)); - record_in(dir.path(), source, "sig", Duration::from_millis(1200)); - assert_eq!( - lookup_in(dir.path(), source, "sig"), - Some(Duration::from_millis(1200)) - ); - } -}