diff --git a/crates/mirth-lab/src/lib.rs b/crates/mirth-lab/src/lib.rs index ca3d0e3..f61b1ab 100644 --- a/crates/mirth-lab/src/lib.rs +++ b/crates/mirth-lab/src/lib.rs @@ -9,4 +9,5 @@ pub mod miri; pub mod mutations; pub mod normalize; pub mod rustc; +pub mod sitemap; pub mod uitest; diff --git a/crates/mirth-lab/src/main.rs b/crates/mirth-lab/src/main.rs index d15eceb..93167e9 100644 --- a/crates/mirth-lab/src/main.rs +++ b/crates/mirth-lab/src/main.rs @@ -10,13 +10,16 @@ mod tools { pub mod apply_suggestions; pub mod audit_options; pub mod callgraph; + pub mod codegen_coverage; pub mod coverage; pub mod coverage_compact; pub mod coverage_flags; pub mod coverage_generators; + pub mod coverage_static; pub mod crash_diff; pub mod debug_check; pub mod diag_check; + pub mod diag_coverage; pub mod fuzz; pub mod fuzz_replay; pub mod flag_fuzz; @@ -37,6 +40,7 @@ mod tools { pub mod replay; pub mod repro_diff; pub mod rewrite_diff; + pub mod rmeta_coverage; pub mod rustdoc_diff; pub mod scale_check; pub mod solver_diff; @@ -141,6 +145,14 @@ enum Check { DebugCheck(tools::debug_check::Args), /// Mutants of the UI tests that use unstable features (splices, moved items, extra gates, edits): ICEs and hangs. GateMutate(tools::gate_mutate::Args), + /// Coverage denominators from the compiler's source (arms, configuration branches, gates, delayed bugs, queries, tables). + CoverageStatic(tools::coverage_static::Args), + /// Error codes, lints, diagnostic structs and suggestion applicabilities a corpus makes rustc emit. + DiagCoverage(tools::diag_coverage::Args), + /// Metadata tables encoded and decoded per crate type, from the blessed metadata records. + RmetaCoverage(tools::rmeta_coverage::Args), + /// LLVM intrinsics, Rust intrinsic lowering and calling conventions the corpus reaches. + CodegenCoverage(tools::codegen_coverage::Args), } fn main() -> ExitCode { @@ -173,6 +185,10 @@ fn main() -> ExitCode { Check::FlagFuzz(a) => tools::flag_fuzz::run(a), Check::AuditOptions(a) => tools::audit_options::run(a), Check::Coverage(a) => tools::coverage::run(a), + Check::CoverageStatic(a) => tools::coverage_static::run(a), + Check::DiagCoverage(a) => tools::diag_coverage::run(a), + Check::RmetaCoverage(a) => tools::rmeta_coverage::run(a), + Check::CodegenCoverage(a) => tools::codegen_coverage::run(a), Check::CoverageCompact(a) => tools::coverage_compact::run(a), Check::CoverageGenerators(a) => tools::coverage_generators::run(a), Check::GrammarCoverage(a) => tools::grammar_coverage::run(a), diff --git a/crates/mirth-lab/src/sitemap.rs b/crates/mirth-lab/src/sitemap.rs new file mode 100644 index 0000000..b619c8c --- /dev/null +++ b/crates/mirth-lab/src/sitemap.rs @@ -0,0 +1,181 @@ +//! The coverage build's sites by source position, and what the measured suites reached: enough to +//! ask, for any span of the compiler's source, whether code in it ran, without a rebuild. +//! +//! A block site's span is a position inside the statement or terminator the block starts at, so a +//! span (a match arm's body, a call) "ran" when a site starting inside it was reached. Spans with no +//! site inside take the nearest site before them in the same function (`probe`). + +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; + +use crate::coverage::files_with; + +/// A position: (line, column), both 1-based as the site tables write them. +pub type Pos = (u32, u32); + +pub struct Site { + pub id: String, + pub path: String, + pub pos: Pos, + /// Only reached on a panic path (tag `panics`), or only in a logging macro (`log`). + pub panics: bool, + pub log: bool, + pub snippet: String, +} + +pub struct SiteMap { + pub sites: Vec, + /// File (`compiler/...`) -> indices into `sites`, sorted by position. + by_file: HashMap>, +} + +fn parse_pos(span: &str) -> Option<(String, Pos)> { + let mut it = span.rsplitn(3, ':'); + let col = it.next()?.parse().ok()?; + let line = it.next()?.parse().ok()?; + Some((it.next()?.to_owned(), (line, col))) +} + +impl SiteMap { + /// The `cover` and `block` sites of the tables in `dir`. + pub fn load(dir: &Path) -> SiteMap { + let mut sites = Vec::new(); + let mut by_file: HashMap> = HashMap::new(); + for table in files_with(dir, "sites") { + let text = String::from_utf8_lossy(&std::fs::read(&table).unwrap_or_default()).into_owned(); + for line in text.lines() { + let f: Vec<&str> = line.split('\t').collect(); + if f.len() < 7 || (f[1] != "cover" && f[1] != "block") { + continue; + } + let Some((file, pos)) = parse_pos(f[6]) else { continue }; + let tags: Vec<&str> = if f[1] == "block" { f[5].split(' ').skip(1).collect() } else { vec![] }; + by_file.entry(file).or_default().push(sites.len()); + sites.push(Site { + id: f[0].into(), + path: f[4].into(), + pos, + panics: tags.contains(&"panics"), + log: tags.contains(&"log"), + snippet: f.get(7).unwrap_or(&"").to_string(), + }); + } + } + for v in by_file.values_mut() { + v.sort_by_key(|&i| sites[i].pos); + } + SiteMap { sites, by_file } + } + + pub fn files(&self) -> impl Iterator { + self.by_file.keys() + } + + /// The sites starting in [start, end] of `file`; when there are none, the nearest site before + /// `start` that is not before `floor` (the enclosing function's start), if any. + pub fn probe(&self, file: &str, start: Pos, end: Pos, floor: Pos) -> Vec<&Site> { + let Some(v) = self.by_file.get(file) else { return Vec::new() }; + let lo = v.partition_point(|&i| self.sites[i].pos < start); + let hi = v.partition_point(|&i| self.sites[i].pos <= end); + if lo < hi { + return v[lo..hi].iter().map(|&i| &self.sites[i]).collect(); + } + match lo.checked_sub(1).map(|j| &self.sites[v[j]]) { + Some(s) if s.pos >= floor => vec![s], + _ => Vec::new(), + } + } + + /// For a file, the share of its sites whose snippet matches the source at their position: + /// below 0.9 the source has moved since the coverage build and spans cannot be trusted. + pub fn alignment(&self, file: &str, source: &str) -> f64 { + let Some(v) = self.by_file.get(file) else { return 1.0 }; + let lines: Vec<&str> = source.lines().collect(); + let (mut checked, mut ok) = (0u32, 0u32); + for &i in v.iter().step_by((v.len() / 40).max(1)) { + let s = &self.sites[i]; + let want: String = s.snippet.chars().filter(|c| !c.is_whitespace()).take(12).collect(); + if want.is_empty() { + continue; + } + let Some(line) = lines.get(s.pos.0 as usize - 1) else { continue }; + // Columns count characters; take the rest of the line and the next few lines. + let rest: String = line.chars().skip(s.pos.1 as usize - 1).collect::() + + &lines.iter().skip(s.pos.0 as usize).take(3).copied().collect::(); + let have: String = rest.chars().filter(|c| !c.is_whitespace()).take(want.chars().count()).collect(); + checked += 1; + ok += (have == want) as u32; + } + if checked == 0 { 1.0 } else { ok as f64 / checked as f64 } + } +} + +/// Every site the measured runs reached: the `union.txt` of each suite under the `suites` +/// directories, and the `V ` lines of the raw logs under `logs` (recursively). +pub fn reached(inputs: &Inputs) -> HashSet { + let mut out = HashSet::new(); + for suites in &inputs.suites { + let Ok(entries) = std::fs::read_dir(suites) else { continue }; + for e in entries.flatten() { + if let Ok(text) = std::fs::read_to_string(e.path().join("union.txt")) { + out.extend(text.split_whitespace().map(str::to_owned)); + } + } + } + for dir in &inputs.logs { + for e in walkdir::WalkDir::new(dir).into_iter().filter_map(Result::ok) { + if e.path().extension().is_some_and(|x| x == "log") { + let text = String::from_utf8_lossy(&std::fs::read(e.path()).unwrap_or_default()).into_owned(); + out.extend(crate::coverage::log_hits(&text).map(str::to_owned)); + } + } + } + out +} + +/// The functions `mirth-lab callgraph --json` found unreachable. +pub fn unreachable(gaps_json: &Path) -> HashSet { + let text = std::fs::read_to_string(gaps_json).unwrap_or_default(); + let v: serde_json::Value = serde_json::from_str(&text).unwrap_or_default(); + v["unreachable"].as_array().into_iter().flatten().filter_map(|x| x.as_str().map(str::to_owned)).collect() +} + +/// The Rust files under `root` (`/compiler`), with their key as the site tables write it. +pub fn compiler_files(rust: &Path) -> Vec<(String, PathBuf)> { + walkdir::WalkDir::new(rust.join("compiler")) + .into_iter() + .filter_map(Result::ok) + .filter(|e| e.path().extension().is_some_and(|x| x == "rs")) + .filter_map(|e| { + let rel = e.path().strip_prefix(rust).ok()?.to_string_lossy().into_owned(); + Some((rel, e.path().to_owned())) + }) + .collect() +} + +/// Defaults shared by the reports. +#[derive(clap::Args, Debug, Clone)] +pub struct Inputs { + /// The rust checkout whose `compiler/` the coverage build was built from (a snapshot works). + #[arg(long, default_value_os_t = home("mirth-work/rust"))] + pub rust: PathBuf, + /// The coverage build's site tables. + #[arg(long, default_value_os_t = home("mirth-work/build-blk/mirth-sites"))] + pub sites: PathBuf, + /// Directories of measured suites (`/union.txt`) of the block build: by default the + /// block corpus (`cov-blk/suites`) and the suites folded since (`cov-suites`, where the checks' + /// block runs go). A function-only run adds no block sites, so mixing it in only adds + /// function entries. + #[arg(long, default_values_os_t = [home("mirth-work/cov-blk/suites"), home("mirth-work/cov-suites")])] + pub suites: Vec, + /// Directories of raw logs of the block build (searched recursively). + #[arg(long, default_values_os_t = [home("mirth-work/cov-blk/sink"), home("mirth-work/cov-blk/flags")])] + pub logs: Vec, + /// `mirth-lab callgraph --json` output for the block build (its `unreachable` list). + #[arg(long, default_value_os_t = home("mirth-work/cov-blk/gaps.json"))] + pub gaps: PathBuf, +} + +pub fn home(rel: &str) -> PathBuf { + PathBuf::from(std::env::var_os("HOME").unwrap_or_default()).join(rel) +} diff --git a/crates/mirth-lab/src/tools/codegen_coverage.rs b/crates/mirth-lab/src/tools/codegen_coverage.rs new file mode 100644 index 0000000..05fb345 --- /dev/null +++ b/crates/mirth-lab/src/tools/codegen_coverage.rs @@ -0,0 +1,273 @@ +//! Codegen and target coverage (docs/coverage-plan.md, rank 10), from outputs and existing block +//! coverage, without a rebuild. +//! +//! - LLVM intrinsics: every `llvm.*` name rustc_codegen_llvm can emit (its string literals) against +//! those in the IR of the runnable UI tests (`--collect`: each test at -Copt-level=0 and 3, with +//! the calling conventions, linkage kinds and target-feature sets of the IR kept too). +//! - Rust intrinsics: the `sym::` arms of the codegen intrinsic matches +//! (rustc_codegen_llvm/src/intrinsic.rs, rustc_codegen_ssa/src/mir/intrinsic.rs), taken or not +//! by the measured suites (block sites inside the arm). +//! - Calling-convention lowering per architecture: block coverage of each +//! `rustc_target/src/callconv/.rs`, and the conventions abi-diff's IR shows per target. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt::Write as _; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; +use std::sync::LazyLock; + +use mirth_lab::driver::{self, Record, Sweep}; +use mirth_lab::rustc::{Compile, Status}; +use mirth_lab::sitemap::{self, home, Inputs, SiteMap}; +use mirth_lab::uitest::{self, Test}; +use quote::ToTokens; +use regex::Regex; +use serde::{Deserialize, Serialize}; +use syn::spanned::Spanned; +use syn::visit::{self, Visit}; + +#[derive(clap::Args, Debug)] +pub struct Args { + #[arg(long, default_value_os_t = home("mirth-work/campaign/rustc/bin/rustc"))] + rustc: PathBuf, + /// Compile the runnable UI tests to LLVM IR and keep what they use (/results.jsonl). + #[arg(long)] + collect: bool, + #[command(flatten)] + sweep: Sweep, + #[command(flatten)] + inputs: Inputs, + /// An abi-diff work directory (per-target r.ll), for conventions per target. + #[arg(long)] + abi: Option, + #[arg(long)] + out: Option, +} + +#[derive(Serialize, Deserialize, Default)] +struct Rec { + test: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + skip: Option, + #[serde(default)] + intrinsics: BTreeSet, + #[serde(default)] + cc: BTreeSet, + #[serde(default)] + linkage: BTreeSet, + #[serde(default)] + features: BTreeSet, +} + +impl Record for Rec { + fn findings(&self) -> Vec { + Vec::new() + } + fn test(&self) -> &str { + &self.test + } +} + +static LLVM_CALL: LazyLock = LazyLock::new(|| Regex::new(r"@(llvm\.[\w.]+)").unwrap()); +static CC: LazyLock = LazyLock::new(|| Regex::new(r"^(?:define|declare)\b[^@]*?\b(\w+cc|cc \d+)\b").unwrap()); +static LINKAGE: LazyLock = LazyLock::new(|| { + Regex::new(r"^define\s+(private|internal|available_externally|linkonce_odr|linkonce|weak_odr|weak|common|appending|extern_weak|external)?").unwrap() +}); +static FEATURES: LazyLock = LazyLock::new(|| Regex::new(r#""target-features"="([^"]*)""#).unwrap()); +/// An intrinsic's overloaded type suffix (`llvm.memcpy.p0.p0.i64` -> `llvm.memcpy`). +static OVERLOAD: LazyLock = LazyLock::new(|| Regex::new(r"(\.(p\d+|i\d+|f\d+|v\d+[if]\d+|nxv\d+[if]\d+|[a-z]\d+))+$").unwrap()); + +fn base(name: &str) -> String { + OVERLOAD.replace(name, "").into_owned() +} + +fn scan_ir(ir: &str, rec: &mut Rec) { + for line in ir.lines() { + if line.contains("@llvm.") && (line.contains("call ") || line.starts_with("declare")) { + for c in LLVM_CALL.captures_iter(line) { + rec.intrinsics.insert(base(&c[1])); + } + } + if line.starts_with("define") || line.starts_with("declare") { + if let Some(c) = CC.captures(line) { + rec.cc.insert(c[1].to_owned()); + } + } + if line.starts_with("define") { + rec.linkage.insert(LINKAGE.captures(line).and_then(|c| c.get(1)).map_or("external", |m| m.as_str()).to_owned()); + } + if let Some(c) = FEATURES.captures(line) { + rec.features.extend(c[1].split(',').filter(|f| !f.is_empty()).map(str::to_owned)); + } + } +} + +fn collect_one(args: &Args, test: &Test) -> Rec { + let mut rec = Rec { test: test.rel.clone(), ..Default::default() }; + for opt in ["0", "3"] { + let dir = driver::scratch_dir(&args.sweep); + let c = Compile::new(&args.rustc, &test.path, dir.path(), &test.flags, test.edition()) + .emit("llvm-ir") + .extra([format!("-Copt-level={opt}")]) + .timeout(120) + .run(); + if !matches!(c.status, Status::Ok) { + rec.skip = Some(format!("{:?} at -Copt-level={opt}", c.status).to_lowercase()); + continue; + } + scan_ir(&std::fs::read_to_string(dir.path().join("prog")).unwrap_or_default(), &mut rec); + } + rec +} + +/// `llvm.*` names in rustc_codegen_llvm's string literals (templates with `{}` left out). +fn llvm_names(rust: &Path) -> BTreeSet { + static LIT: LazyLock = LazyLock::new(|| Regex::new(r#""(llvm\.[a-z0-9_.]+)""#).unwrap()); + let mut out = BTreeSet::new(); + for (key, path) in sitemap::compiler_files(rust) { + if key.starts_with("compiler/rustc_codegen_llvm/") { + let text = std::fs::read_to_string(&path).unwrap_or_default(); + out.extend(LIT.captures_iter(&text).map(|c| base(c[1].trim_end_matches('.')))); + } + } + out +} + +/// The `sym::name` arms of the intrinsic matches: (file, name, body span). +struct Arms<'a> { + file: &'a str, + out: Vec<(String, String, (sitemap::Pos, sitemap::Pos))>, +} + +impl<'ast> Visit<'ast> for Arms<'_> { + fn visit_arm(&mut self, a: &'ast syn::Arm) { + static SYM: LazyLock = LazyLock::new(|| Regex::new(r"sym\s*::\s*(\w+)").unwrap()); + let pat = a.pat.to_token_stream().to_string(); + let span = a.body.span(); + let r = ((span.start().line as u32, span.start().column as u32 + 1), (span.end().line as u32, span.end().column as u32 + 1)); + for c in SYM.captures_iter(&pat) { + self.out.push((self.file.to_owned(), c[1].to_owned(), r)); + } + visit::visit_arm(self, a); + } +} + +pub fn run(args: Args) -> anyhow::Result { + if args.collect { + let tests = args.sweep.select(uitest::tests(&args.sweep.tests, uitest::RUNNABLE, |_| false)); + println!("{} tests", tests.len()); + return Ok(driver::drive(&tests, &args.sweep, |t| collect_one(&args, t))); + } + let inp = &args.inputs; + let mut out = String::new(); + let mut gaps = String::new(); + + // LLVM intrinsics in the IR. + let mut used: BTreeMap = BTreeMap::new(); + let (mut cc, mut linkage, mut features) = (BTreeMap::::new(), BTreeMap::::new(), BTreeMap::::new()); + let mut recs = 0; + for line in std::fs::read_to_string(args.sweep.work.join("results.jsonl")).unwrap_or_default().lines() { + let Ok(r) = serde_json::from_str::(line) else { continue }; + recs += 1; + for i in r.intrinsics { + *used.entry(i).or_default() += 1; + } + for (set, map) in [(r.cc, &mut cc), (r.linkage, &mut linkage), (r.features, &mut features)] { + for x in set { + *map.entry(x).or_default() += 1; + } + } + } + // Metadata names and special globals are not calls: kept apart from the intrinsics. + let (special, names): (BTreeSet, BTreeSet) = llvm_names(&inp.rust) + .into_iter() + .partition(|n| ["llvm.loop.", "llvm.ident", "llvm.metadata", "llvm.used", "llvm.compiler.used"].iter().any(|p| n.starts_with(p))); + let hit: Vec<&String> = names.iter().filter(|n| used.contains_key(*n)).collect(); + let _ = writeln!(out, "LLVM intrinsics: {} named in rustc_codegen_llvm (plus {} metadata names and special globals, not counted); {} appear in the IR of {recs} runnable tests at -O0/-O3 ({:.1}%); {} distinct intrinsics in that IR overall (LLVM adds its own)", names.len(), special.len(), hit.len(), 100.0 * hit.len() as f64 / names.len().max(1) as f64, used.len()); + let _ = writeln!(out, " calling conventions in the IR: {cc:?}"); + let _ = writeln!(out, " linkage kinds of definitions: {linkage:?}"); + let _ = writeln!(out, " target features enabled anywhere: {}", features.len()); + let _ = writeln!(gaps, "# LLVM intrinsics rustc can emit that no runnable test's IR contains: {}\n", names.len() - hit.len()); + for n in names.iter().filter(|n| !used.contains_key(*n)) { + let _ = writeln!(gaps, "- {n}"); + } + + // Rust intrinsic arms. + let map = SiteMap::load(&inp.sites); + let reached = sitemap::reached(inp); + let mut arms = Vec::new(); + for f in ["compiler/rustc_codegen_llvm/src/intrinsic.rs", "compiler/rustc_codegen_ssa/src/mir/intrinsic.rs"] { + let text = std::fs::read_to_string(inp.rust.join(f)).unwrap_or_default(); + if map.alignment(f, &text) < 0.9 { + let _ = writeln!(out, " {f} moved since the coverage build: skipped"); + continue; + } + if let Ok(file) = syn::parse_file(&text) { + let mut v = Arms { file: f, out: Vec::new() }; + v.visit_file(&file); + arms.extend(v.out); + } + } + let mut per: BTreeMap = BTreeMap::new(); + for (file, name, (a, b)) in &arms { + let inside = map.probe(file, *a, *b, (u32::MAX, 0)); + let e = per.entry(name.clone()).or_default(); + e.0 |= !inside.is_empty(); + e.1 |= inside.iter().any(|s| reached.contains(&s.id)); + } + let measurable = per.values().filter(|v| v.0).count(); + let taken = per.values().filter(|v| v.1).count(); + let _ = writeln!(out, "Rust intrinsics with their own codegen arm: {}; measurable {measurable}; lowered by a measured run {taken} ({:.1}%)", per.len(), 100.0 * taken as f64 / measurable.max(1) as f64); + let _ = writeln!(gaps, "\n# Rust intrinsics whose codegen arm never ran: {}\n", measurable - taken); + for (n, _) in per.iter().filter(|(_, v)| v.0 && !v.1) { + let _ = writeln!(gaps, "- {n}"); + } + + // Calling-convention lowering per architecture. + let _ = writeln!(out, "calling-convention lowering per architecture (block sites in rustc_target/src/callconv/.rs, reached by any suite):"); + let mut rows: Vec<(String, usize, usize)> = map + .files() + .filter(|f| f.starts_with("compiler/rustc_target/src/callconv/")) + .map(|f| { + let sites = map.probe(f, (0, 0), (u32::MAX, 0), (0, 0)); + (f.trim_start_matches("compiler/rustc_target/src/callconv/").to_owned(), sites.len(), sites.iter().filter(|s| reached.contains(&s.id)).count()) + }) + .collect(); + rows.sort_by(|a, b| (a.2 as f64 / a.1.max(1) as f64).partial_cmp(&(b.2 as f64 / b.1.max(1) as f64)).unwrap()); + let _ = writeln!(gaps, "\n# Calling-convention lowering by architecture\n\n| file | sites | reached | % |\n|---|---:|---:|---:|"); + let zero = rows.iter().filter(|r| r.2 == 0).count(); + for (f, n, h) in &rows { + let _ = writeln!(gaps, "| {f} | {n} | {h} | {:.1} |", 100.0 * *h as f64 / (*n).max(1) as f64); + } + let total: usize = rows.iter().map(|r| r.1).sum(); + let reached_n: usize = rows.iter().map(|r| r.2).sum(); + let _ = writeln!(out, " {} files, {total} sites, {reached_n} reached ({:.1}%); {zero} files never reached", rows.len(), 100.0 * reached_n as f64 / total.max(1) as f64); + + // Conventions per target, from abi-diff's IR. + if let Some(abi) = &args.abi { + let mut per_target: BTreeMap> = BTreeMap::new(); + if let Ok(entries) = std::fs::read_dir(abi) { + for e in entries.flatten() { + let ll = e.path().join("r.ll"); + if let Ok(ir) = std::fs::read_to_string(&ll) { + let mut rec = Rec::default(); + scan_ir(&ir, &mut rec); + per_target.insert(e.file_name().to_string_lossy().into_owned(), if rec.cc.is_empty() { ["ccc".to_owned()].into() } else { rec.cc }); + } + } + } + let mut by_cc: BTreeMap = BTreeMap::new(); + for s in per_target.values() { + for c in s { + *by_cc.entry(c.clone()).or_default() += 1; + } + } + let _ = writeln!(out, " extern \"C\" lowering in abi-diff's IR: {} targets; conventions used: {by_cc:?}", per_target.len()); + } + print!("{out}"); + if let Some(dir) = &args.out { + std::fs::create_dir_all(dir)?; + std::fs::write(dir.join("gaps-codegen.md"), gaps)?; + } + Ok(ExitCode::SUCCESS) +} diff --git a/crates/mirth-lab/src/tools/coverage_static.rs b/crates/mirth-lab/src/tools/coverage_static.rs new file mode 100644 index 0000000..c9a5160 --- /dev/null +++ b/crates/mirth-lab/src/tools/coverage_static.rs @@ -0,0 +1,546 @@ +//! The static denominators of docs/coverage-plan.md (milestone M1), counted from the compiler's +//! source with syn, and what today's block coverage already says about them (an estimate until +//! the arm-instrumented build of M2 measures arms exactly). +//! +//! - Branch arms: every arm of every decision in a function: `match` arms, `if` and its (possibly +//! implicit) `else`, `while`/`for` body and exit, `let ... else`, `?` (continue, return) and +//! `&&`/`||` (evaluate the right side, skip it). An arm with a body is measurable from block +//! sites when a site starts inside the body; implicit arms (no body: the `else` an `if` lacks, +//! `?`'s early return) are not, until M2. +//! - Configuration branches: decisions whose condition reads a feature gate, a `-Z`/`-C` option, +//! the edition or a target property. +//! - Feature-gate check sites: each call of a `Features` accessor (or `enabled(sym::…)`), per +//! unstable feature. +//! - Delayed-bug sites: each `span_delayed_bug`/`delayed_bug` call. +//! - Keyed engine sites: queries (and their flags) × engine blocks. +//! - Metadata tables: `define_tables!`. + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::fmt::Write as _; +use std::path::PathBuf; +use std::process::ExitCode; +use std::sync::LazyLock; + +use mirth_lab::sitemap::{self, Inputs, Pos, SiteMap}; +use quote::ToTokens; +use regex::Regex; +use syn::spanned::Spanned; +use syn::visit::{self, Visit}; + +#[derive(clap::Args, Debug)] +pub struct Args { + #[command(flatten)] + inputs: Inputs, + /// Write the gap lists (arms never taken, configuration decisions seen one way, delayed-bug + /// sites never reached) to this directory. + #[arg(long)] + out: Option, + #[arg(long)] + json: Option, +} + +#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)] +enum Kind { + Match, + If, + Loop, + LetElse, + Try, + ShortCircuit, +} + +struct Arm { + /// The arm's body; None for an implicit arm. + span: Option<(Pos, Pos)>, +} + +struct Decision { + kind: Kind, + file: String, + cond: (Pos, Pos), + floor: Pos, + arms: Vec, + config: Vec<(&'static str, String)>, +} + +struct Call { + file: String, + span: (Pos, Pos), + floor: Pos, + name: String, +} + +fn pos(s: proc_macro2::LineColumn) -> Pos { + (s.line as u32, s.column as u32 + 1) +} + +fn range(span: proc_macro2::Span) -> (Pos, Pos) { + (pos(span.start()), pos(span.end())) +} + +static FEATURE_LIST: LazyLock = LazyLock::new(|| Regex::new(r"\(\s*(unstable|incomplete|internal)\s*,\s*(\w+)\s*,").unwrap()); +static OPTION: LazyLock = LazyLock::new(|| Regex::new(r"\b(?:unstable_opts|cg)\s*\.\s*(?:read_)?(\w+)").unwrap()); +static EDITION: LazyLock = + LazyLock::new(|| Regex::new(r"\bedition\s*\(\s*\)|\b(?:at_least|is)_rust_20\d\d\b|Edition\s*::\s*Edition20\d\d").unwrap()); +static TARGET: LazyLock = LazyLock::new(|| Regex::new(r"\btarget\s*\.\s*(?:options\s*\.\s*)?(\w+)").unwrap()); +// `gate!(visitor, name, …)` (any first argument) and `gate_all!(name, …)`. +static GATE_MACRO: LazyLock = LazyLock::new(|| Regex::new(r"^[^,]*,\s*(\w+)\s*,").unwrap()); +static GATE_ALL: LazyLock = LazyLock::new(|| Regex::new(r"^\s*(\w+)\s*,").unwrap()); +static FEATURE_CALL: LazyLock = + LazyLock::new(|| Regex::new(r"features\s*(?:\(\s*\))?\s*\.\s*(\w+)\s*\(\s*\)|\benabled\s*\(\s*sym\s*::\s*(\w+)\s*\)").unwrap()); + +struct Scan<'a> { + file: String, + floor: Pos, + features: &'a HashSet, + decisions: Vec, + feature_calls: Vec, + delayed: Vec, +} + +impl Scan<'_> { + fn config(&self, cond: &str) -> Vec<(&'static str, String)> { + let mut out = Vec::new(); + for c in FEATURE_CALL.captures_iter(cond) { + let name = c.get(1).or(c.get(2)).unwrap().as_str(); + if self.features.contains(name) { + out.push(("feature", name.to_owned())); + } + } + out.extend(OPTION.captures_iter(cond).map(|c| ("option", c[1].to_owned()))); + if EDITION.is_match(cond) { + out.push(("edition", "edition".into())); + } + if cond.contains("sess") || cond.contains("tcx") { + out.extend(TARGET.captures_iter(cond).map(|c| ("target", c[1].to_owned()))); + } + out.sort(); + out.dedup(); + out + } + + fn decide(&mut self, kind: Kind, cond: proc_macro2::Span, cond_text: String, arms: Vec) { + let config = if matches!(kind, Kind::Try) { Vec::new() } else { self.config(&cond_text) }; + self.decisions.push(Decision { kind, file: self.file.clone(), cond: range(cond), floor: self.floor, arms, config }); + } + + fn with_floor(&mut self, at: proc_macro2::Span, f: impl FnOnce(&mut Self)) { + let saved = self.floor; + self.floor = pos(at.start()); + f(self); + self.floor = saved; + } +} + +fn body(span: proc_macro2::Span) -> Arm { + Arm { span: Some(range(span)) } +} + +const IMPLICIT: Arm = Arm { span: None }; + +impl<'ast> Visit<'ast> for Scan<'_> { + fn visit_item_fn(&mut self, i: &'ast syn::ItemFn) { + self.with_floor(i.span(), |s| visit::visit_item_fn(s, i)); + } + fn visit_impl_item_fn(&mut self, i: &'ast syn::ImplItemFn) { + self.with_floor(i.span(), |s| visit::visit_impl_item_fn(s, i)); + } + fn visit_trait_item_fn(&mut self, i: &'ast syn::TraitItemFn) { + self.with_floor(i.span(), |s| visit::visit_trait_item_fn(s, i)); + } + + fn visit_expr_match(&mut self, e: &'ast syn::ExprMatch) { + let arms = e.arms.iter().map(|a| body(a.body.span())).collect(); + self.decide(Kind::Match, e.expr.span(), e.expr.to_token_stream().to_string(), arms); + visit::visit_expr_match(self, e); + } + fn visit_expr_if(&mut self, e: &'ast syn::ExprIf) { + let mut arms = vec![body(e.then_branch.span())]; + arms.push(match &e.else_branch { + Some((_, els)) => body(els.span()), + None => IMPLICIT, + }); + self.decide(Kind::If, e.cond.span(), e.cond.to_token_stream().to_string(), arms); + visit::visit_expr_if(self, e); + } + fn visit_expr_while(&mut self, e: &'ast syn::ExprWhile) { + self.decide(Kind::Loop, e.cond.span(), e.cond.to_token_stream().to_string(), vec![body(e.body.span()), IMPLICIT]); + visit::visit_expr_while(self, e); + } + fn visit_expr_for_loop(&mut self, e: &'ast syn::ExprForLoop) { + self.decide(Kind::Loop, e.expr.span(), e.expr.to_token_stream().to_string(), vec![body(e.body.span()), IMPLICIT]); + visit::visit_expr_for_loop(self, e); + } + fn visit_local(&mut self, l: &'ast syn::Local) { + if let Some(init) = &l.init + && let Some((_, els)) = &init.diverge + { + self.decide(Kind::LetElse, init.expr.span(), init.expr.to_token_stream().to_string(), vec![IMPLICIT, body(els.span())]); + } + visit::visit_local(self, l); + } + fn visit_expr_try(&mut self, e: &'ast syn::ExprTry) { + self.decide(Kind::Try, e.span(), String::new(), vec![IMPLICIT, IMPLICIT]); + visit::visit_expr_try(self, e); + } + fn visit_expr_binary(&mut self, e: &'ast syn::ExprBinary) { + if matches!(e.op, syn::BinOp::And(_) | syn::BinOp::Or(_)) { + self.decide(Kind::ShortCircuit, e.left.span(), e.left.to_token_stream().to_string(), vec![body(e.right.span()), IMPLICIT]); + } + visit::visit_expr_binary(self, e); + } + fn visit_macro(&mut self, m: &'ast syn::Macro) { + // Gates checked through macros: `gate!(self, name, …)`, `gate_all!(name, …)`, and accessor + // calls written inside any macro's tokens. + let name = m.path.segments.last().map(|s| s.ident.to_string()).unwrap_or_default(); + let tokens = m.tokens.to_string(); + let mut found: Vec = Vec::new(); + if name.starts_with("gate") { + let re = if name.starts_with("gate_all") { &*GATE_ALL } else { &*GATE_MACRO }; + found.extend(re.captures(&tokens).map(|c| c[1].to_owned())); + } + for c in FEATURE_CALL.captures_iter(&tokens) { + found.push(c.get(1).or(c.get(2)).unwrap().as_str().to_owned()); + } + for f in found.into_iter().filter(|f| self.features.contains(f)) { + self.feature_calls.push(Call { file: self.file.clone(), span: range(m.span()), floor: self.floor, name: f }); + } + visit::visit_macro(self, m); + } + + fn visit_expr_method_call(&mut self, e: &'ast syn::ExprMethodCall) { + let name = e.method.to_string(); + if name == "span_delayed_bug" || name == "delayed_bug" { + self.delayed.push(Call { file: self.file.clone(), span: range(e.span()), floor: self.floor, name: name.clone() }); + } + let text = e.to_token_stream().to_string(); + // The accessor call itself: `<…features…>.name()` or `<…>.enabled(sym::name)`. + let feature = if e.args.is_empty() && self.features.contains(&name) && e.receiver.to_token_stream().to_string().contains("features") { + Some(name) + } else if name == "enabled" { + FEATURE_CALL.captures(&text).and_then(|c| c.get(2)).map(|m| m.as_str().to_owned()).filter(|f| self.features.contains(f)) + } else { + None + }; + if let Some(f) = feature { + self.feature_calls.push(Call { file: self.file.clone(), span: range(e.span()), floor: self.floor, name: f }); + } + visit::visit_expr_method_call(self, e); + } +} + +/// The queries of `rustc_middle/src/queries.rs` with their modifiers. +fn queries(source: &str) -> Vec<(String, Vec)> { + static QUERY: LazyLock = LazyLock::new(|| Regex::new(r"(?m)^\s*query\s+(\w+)\s*\(").unwrap()); + static MODIFIER: LazyLock = LazyLock::new(|| { + Regex::new(r"(?m)^\s*(eval_always|cache_on_disk_if|cache_on_disk|feedable|anon|no_hash|separate_provide_extern|depth_limit|cycle_\w+|return_result_from_ensure_ok|arena_cache)\b").unwrap() + }); + let starts: Vec<(usize, String)> = QUERY.captures_iter(source).map(|c| (c.get(0).unwrap().start(), c[1].to_owned())).collect(); + starts + .iter() + .enumerate() + .map(|(k, (at, name))| { + let end = starts.get(k + 1).map_or(source.len(), |n| n.0); + let mods = MODIFIER.captures_iter(&source[*at..end]).map(|c| c[1].to_owned()).collect(); + (name.clone(), mods) + }) + .collect() +} + +/// The tables of `define_tables!` in `rustc_metadata/src/rmeta/mod.rs`: (name, defaulted). +pub fn rmeta_tables(source: &str) -> Vec<(String, bool)> { + static TABLE: LazyLock = LazyLock::new(|| Regex::new(r"(?m)^\s*(\w+)\s*:\s*Table<").unwrap()); + let Some(start) = source.find("define_tables!") else { return Vec::new() }; + let body = &source[start..]; + let end = body.find("\n}\n").unwrap_or(body.len()); + let body = &body[..end]; + let optional = body.find("- optional:").unwrap_or(body.len()); + TABLE.captures_iter(body).map(|c| (c[1].to_owned(), c.get(0).unwrap().start() < optional)).collect() +} + +#[derive(Default, serde::Serialize)] +struct Totals { + decisions: usize, + arms: usize, + explicit: usize, + implicit: usize, + reachable_arms: usize, + measurable: usize, + panic_only: usize, + taken: usize, +} + +pub fn run(args: Args) -> anyhow::Result { + let inp = &args.inputs; + let map = SiteMap::load(&inp.sites); + let reached = sitemap::reached(inp); + let unreachable = sitemap::unreachable(&inp.gaps); + anyhow::ensure!(!map.sites.is_empty(), "no site tables in {}", inp.sites.display()); + anyhow::ensure!(!unreachable.is_empty(), "no unreachable list in {} (run mirth-lab callgraph --json)", inp.gaps.display()); + let features: HashSet = std::fs::read_to_string(inp.rust.join("compiler/rustc_feature/src/unstable.rs")) + .map(|s| FEATURE_LIST.captures_iter(&s).map(|c| c[2].to_owned()).collect()) + .unwrap_or_default(); + + let mut scan = Scan { file: String::new(), floor: (0, 0), features: &features, decisions: vec![], feature_calls: vec![], delayed: vec![] }; + let (mut parsed, mut skipped_moved, mut skipped_parse) = (0, Vec::new(), 0); + static SYM: LazyLock = LazyLock::new(|| Regex::new(r"\b(?:sym|Features)\s*::\s*(\w+)|\bunstable!\s*\(\s*(\w+)").unwrap()); + let mut referenced: HashSet = HashSet::new(); + for (key, path) in sitemap::compiler_files(&inp.rust) { + let Ok(text) = std::fs::read_to_string(&path) else { continue }; + if !key.ends_with("rustc_feature/src/unstable.rs") && !key.contains("rustc_span/src/symbol") { + referenced.extend(SYM.captures_iter(&text).filter_map(|c| c.get(1).or(c.get(2)).map(|m| m.as_str().to_owned())).filter(|n| features.contains(n))); + } + if map.alignment(&key, &text) < 0.9 { + skipped_moved.push(key); + continue; + } + let Ok(file) = syn::parse_file(&text) else { + skipped_parse += 1; + continue; + }; + scan.file = key; + scan.floor = (0, 0); + scan.visit_file(&file); + parsed += 1; + } + + let live = |s: &sitemap::Site| !unreachable.contains(&s.path); + let mut out = String::new(); + let _ = writeln!(out, "{parsed} source files scanned; {} skipped (moved since the coverage build: {:?}); {skipped_parse} not parsed", skipped_moved.len(), skipped_moved); + let blocks_live = map.sites.iter().filter(|s| live(s) && !s.log).count(); + let _ = writeln!(out, "{} sites in the tables; {} in reachable functions (outside logging macros); {} reached by the suites", map.sites.len(), blocks_live, map.sites.iter().filter(|s| reached.contains(&s.id)).count()); + + // Branch arms. + let mut per_kind: BTreeMap = BTreeMap::new(); + let mut arm_gaps: BTreeMap<(String, String), Vec> = BTreeMap::new(); + let mut all = Totals::default(); + for d in &scan.decisions { + let cond_sites = map.probe(&d.file, d.cond.0, d.cond.1, d.floor); + let reachable = cond_sites.first().is_some_and(|s| live(s)); + for t in [per_kind.entry(d.kind).or_default(), &mut all] { + t.decisions += 1; + } + for arm in &d.arms { + let inside: Vec<&sitemap::Site> = match arm.span { + Some((a, b)) => map.probe(&d.file, a, b, (u32::MAX, 0)), + None => Vec::new(), + }; + let arm_live = inside.first().map_or(reachable, |s| live(s)); + let measurable = !inside.is_empty(); + let panics = measurable && inside.iter().all(|s| s.panics || s.log); + let taken = inside.iter().any(|s| reached.contains(&s.id)); + for t in [per_kind.entry(d.kind).or_default(), &mut all] { + t.arms += 1; + if arm.span.is_some() { t.explicit += 1 } else { t.implicit += 1 } + if arm_live { + t.reachable_arms += 1; + if panics { + t.panic_only += 1; + } else if measurable { + t.measurable += 1; + t.taken += taken as usize; + } + } + } + if arm_live && measurable && !panics && !taken { + let krate = d.file.split('/').nth(1).unwrap_or("").to_owned(); + let (line, _) = arm.span.unwrap().0; + arm_gaps.entry((krate, d.file.clone())).or_default().push(format!("- {line} {:?} arm: `{}`", d.kind, inside[0].snippet.chars().take(80).collect::())); + } + } + } + let pct = |a: usize, b: usize| 100.0 * a as f64 / b.max(1) as f64; + let _ = writeln!(out, "\nbranch arms: {} decisions, {} arms ({} with a body, {} implicit)", all.decisions, all.arms, all.explicit, all.implicit); + let _ = writeln!( + out, + " in reachable functions: {} arms; {} measurable from block sites (a site starts in the body), {} panic-only; taken {} ({:.1}%); implicit arms wait for M2", + all.reachable_arms, all.measurable, all.panic_only, all.taken, pct(all.taken, all.measurable) + ); + let _ = writeln!(out, " arms per reachable site: {:.2} (the plan estimated about 0.5 new arm sites per block)", all.reachable_arms as f64 / blocks_live.max(1) as f64); + let _ = writeln!(out, " {:14} {:>9} {:>9} {:>10} {:>9} {:>7}", "kind", "decisions", "arms", "measurable", "taken", "%"); + for (k, t) in &per_kind { + let _ = writeln!(out, " {:14} {:9} {:9} {:10} {:9} {:7.1}", format!("{k:?}"), t.decisions, t.arms, t.measurable, t.taken, pct(t.taken, t.measurable)); + } + + // Configuration branches. + #[derive(Default)] + struct Cfg { + decisions: usize, + consulted: usize, + first: usize, + other: usize, + both: usize, + } + let mut cfg: BTreeMap<(&str, String), Cfg> = BTreeMap::new(); + let mut cfg_gaps: Vec = Vec::new(); + for d in scan.decisions.iter().filter(|d| !d.config.is_empty()) { + let cond_sites = map.probe(&d.file, d.cond.0, d.cond.1, d.floor); + if !cond_sites.first().is_some_and(|s| live(s)) { + continue; + } + let consulted = cond_sites.iter().any(|s| reached.contains(&s.id)); + let arm_taken = |i: usize| -> Option { + let (a, b) = d.arms.get(i)?.span?; + let inside = map.probe(&d.file, a, b, (u32::MAX, 0)); + (!inside.is_empty()).then(|| inside.iter().any(|s| reached.contains(&s.id))) + }; + let (first, other) = (arm_taken(0).unwrap_or(false), (1..d.arms.len()).any(|i| arm_taken(i).unwrap_or(false))); + for c in &d.config { + let e = cfg.entry((c.0, c.1.clone())).or_default(); + e.decisions += 1; + e.consulted += consulted as usize; + e.first += first as usize; + e.other += other as usize; + e.both += (first && other) as usize; + } + if consulted && !(first && other) { + let (line, _) = d.cond.0; + let names: Vec = d.config.iter().map(|c| format!("{} {}", c.0, c.1)).collect(); + cfg_gaps.push(format!("- {}:{line} {:?} on {}: {}", d.file, d.kind, names.join(", "), if first { "only the first arm seen" } else if other { "only the other arms seen" } else { "consulted, no arm body reached" })); + } + } + let mut by_kind: BTreeMap<&str, (usize, usize, usize, usize)> = BTreeMap::new(); + for ((k, _), c) in &cfg { + let e = by_kind.entry(k).or_default(); + e.0 += 1; + e.1 += c.decisions; + e.2 += c.consulted; + e.3 += c.both; + } + let _ = writeln!(out, "\nconfiguration branches (decisions in reachable functions whose condition reads a gate, option, edition or target property):"); + let _ = writeln!(out, " {:9} {:>6} {:>10} {:>10} {:>16}", "kind", "names", "decisions", "consulted", "both arms seen"); + for (k, (n, d, c, b)) in &by_kind { + let _ = writeln!(out, " {k:9} {n:6} {d:10} {c:10} {b:16}"); + } + + // Feature-gate check sites. + let mut gate: BTreeMap = features.iter().map(|f| (f.clone(), (0, 0))).collect(); + for c in &scan.feature_calls { + let sites = map.probe(&c.file, c.span.0, c.span.1, c.floor); + let e = gate.entry(c.name.clone()).or_default(); + e.0 += 1; + e.1 += sites.iter().any(|s| reached.contains(&s.id)) as usize; + } + let generic: Vec<&String> = gate.iter().filter(|(f, v)| v.0 == 0 && referenced.contains(*f)).map(|(f, _)| f).collect(); + let unchecked: Vec<&String> = gate.iter().filter(|(f, v)| v.0 == 0 && !referenced.contains(*f)).map(|(f, _)| f).collect(); + let with_sites = gate.values().filter(|v| v.0 > 0).count(); + let consulted = gate.values().filter(|v| v.1 > 0).count(); + let _ = writeln!( + out, + "\nfeature gates: {} unstable features; {} have accessor call sites in the compiler ({} call sites); {} were consulted by a measured run (on or off: M4 records which); {} with call sites never consulted", + features.len(), + with_sites, + scan.feature_calls.len(), + consulted, + with_sites - consulted + ); + let _ = writeln!( + out, + " {} more are checked only through generic code that names them (`sym::name` in a gate table or the parser's gated spans, `Features::name` pointers, an attribute parser's `unstable!(name)`): block coverage cannot say whether those were consulted; {} are not named in the compiler that way (a lint declaration's `@feature_gate` identifier, a library const-stability attribute, rustdoc, or nothing)", + generic.len(), + unchecked.len() + ); + + // Delayed bugs. + let mut delayed_gaps: BTreeMap> = BTreeMap::new(); + let (mut d_live, mut d_reached) = (0, 0); + for c in &scan.delayed { + let sites = map.probe(&c.file, c.span.0, c.span.1, c.floor); + if !sites.first().is_some_and(|s| live(s)) { + continue; + } + d_live += 1; + if sites.iter().any(|s| reached.contains(&s.id)) { + d_reached += 1; + } else { + delayed_gaps.entry(c.file.split('/').nth(1).unwrap_or("").to_owned()).or_default().push(format!("- {}:{} `{}`", c.file, c.span.0 .0, c.name)); + } + } + let _ = writeln!(out, "\ndelayed-bug sites: {} calls; {d_live} in reachable functions; {d_reached} reached ({:.1}%)", scan.delayed.len(), pct(d_reached, d_live)); + + // Keyed engine sites. + let qsrc = std::fs::read_to_string(inp.rust.join("compiler/rustc_middle/src/queries.rs")).unwrap_or_default(); + let qs = queries(&qsrc); + let count = |m: &str| qs.iter().filter(|(_, v)| v.iter().any(|x| x.starts_with(m))).count(); + let engine: Vec<&sitemap::Site> = map + .sites + .iter() + .filter(|s| live(s) && (s.path.starts_with("rustc_query_impl::execution::") || s.path.starts_with("rustc_middle::dep_graph::graph::"))) + .collect(); + let disk = engine.iter().filter(|s| s.path.contains("load_from_disk") || s.path.contains("try_load")).count(); + let feed = engine.iter().filter(|s| s.path.contains("feed")).count(); + let cached = count("cache_on_disk"); + let feedable = count("feedable"); + let possible: usize = qs + .iter() + .map(|(_, m)| { + engine.len() - if m.iter().any(|x| x.starts_with("cache_on_disk")) { 0 } else { disk } - if m.iter().any(|x| x == "feedable") { 0 } else { feed } + }) + .sum(); + let _ = writeln!( + out, + "\nkeyed engine sites: {} queries ({} eval_always, {cached} cached on disk, {feedable} feedable, {} anon); {} engine sites in reachable functions ({disk} on the load-from-disk path, {feed} on the feeding path); query × allowed engine site: {possible} possible keyed sites", + qs.len(), + count("eval_always"), + count("anon"), + engine.len() + ); + + // Metadata tables. + let tables = rmeta_tables(&std::fs::read_to_string(inp.rust.join("compiler/rustc_metadata/src/rmeta/mod.rs")).unwrap_or_default()); + let _ = writeln!(out, "\nmetadata tables: {} in define_tables! ({} defaulted, {} optional)", tables.len(), tables.iter().filter(|t| t.1).count(), tables.iter().filter(|t| !t.1).count()); + print!("{out}"); + + if let Some(dir) = &args.out { + std::fs::create_dir_all(dir)?; + let total: usize = arm_gaps.values().map(Vec::len).sum(); + let mut text = format!("# Arms never taken (estimate from block sites, M1): {total}\n\nArms with a body in reachable functions, not panic-only, where no site inside the body ran.\n\n"); + let mut by_crate: HashMap<&str, usize> = HashMap::new(); + for ((k, _), v) in &arm_gaps { + *by_crate.entry(k).or_default() += v.len(); + } + let mut crates: Vec<(&&str, &usize)> = by_crate.iter().collect(); + crates.sort_by(|a, b| b.1.cmp(a.1)); + for (krate, n) in crates { + let _ = writeln!(text, "## {krate} ({n})\n"); + let mut files: Vec<(&(String, String), &Vec)> = arm_gaps.iter().filter(|((k, _), _)| k == krate).collect(); + files.sort_by(|a, b| b.1.len().cmp(&a.1.len())); + for ((_, file), v) in files { + let _ = writeln!(text, "### {file} ({})\n\n{}\n", v.len(), v.join("\n")); + } + } + std::fs::write(dir.join("gaps-arms.md"), text)?; + let mut text = format!("# Configuration decisions not seen both ways: {}\n\n", cfg_gaps.len()); + let _ = writeln!(text, "| kind | name | decisions | consulted | first arm | other arms | both |\n|---|---|---:|---:|---:|---:|---:|"); + for ((k, n), c) in &cfg { + let _ = writeln!(text, "| {k} | {n} | {} | {} | {} | {} | {} |", c.decisions, c.consulted, c.first, c.other, c.both); + } + let _ = writeln!(text, "\n## Decisions\n\n{}", cfg_gaps.join("\n")); + std::fs::write(dir.join("gaps-config.md"), text)?; + let mut text = String::from("# Feature gates\n\n| feature | accessor call sites | consulted sites | checked |\n|---|---:|---:|---|\n"); + for (f, (n, r)) in &gate { + let how = if *n > 0 { "directly" } else if referenced.contains(f) { "generically" } else { "not named in the compiler" }; + let _ = writeln!(text, "| {f} | {n} | {r} | {how} |"); + } + std::fs::write(dir.join("gaps-features.md"), text)?; + let mut text = format!("# Delayed-bug sites never reached: {}\n\n", d_live - d_reached); + for (k, v) in &delayed_gaps { + let _ = writeln!(text, "## {k} ({})\n\n{}\n", v.len(), v.join("\n")); + } + std::fs::write(dir.join("gaps-delayed.md"), text)?; + } + if let Some(j) = &args.json { + let kinds: BTreeMap = per_kind.iter().map(|(k, v)| (format!("{k:?}"), v)).collect(); + std::fs::write( + j, + serde_json::to_string_pretty(&serde_json::json!({ + "arms": all, "arms_by_kind": kinds, + "config_names": cfg.len(), "features": features.len(), "features_with_sites": with_sites, "features_consulted": consulted, + "delayed_sites": scan.delayed.len(), "delayed_live": d_live, "delayed_reached": d_reached, + "queries": qs.len(), "engine_sites": engine.len(), "keyed_possible": possible, "rmeta_tables": tables.len(), + "live_sites": blocks_live, + }))?, + )?; + } + Ok(ExitCode::SUCCESS) +} diff --git a/crates/mirth-lab/src/tools/diag_coverage.rs b/crates/mirth-lab/src/tools/diag_coverage.rs new file mode 100644 index 0000000..ab951df --- /dev/null +++ b/crates/mirth-lab/src/tools/diag_coverage.rs @@ -0,0 +1,399 @@ +//! Diagnostics coverage (docs/coverage-plan.md, rank 4): which of the compiler's error codes, lints, +//! diagnostic structs and suggestion applicabilities a corpus makes it emit, against all that exist. +//! +//! `--collect` compiles each standalone UI test with `--error-format=json` (the sweep driver: +//! resumable, `--jobs`) and keeps, per test, each diagnostic's level, code (an error code or a +//! lint name), message, child messages and labels, and suggestion applicabilities. Without it, +//! the report folds one or more such `results.jsonl` files against: +//! +//! - error codes: `rustc_error_codes/src/error_codes/E*.md` (those marked no longer emitted apart); +//! - lints: `rustc -W help` (by default level); +//! - diagnostic structs: every `#[derive(Diagnostic)]` and `#[derive(Subdiagnostic)]` item or +//! variant with an inline message (`#[diag("…")]`, `#[note("…")]`, …), matched against the +//! emitted messages with each `{$arg}` (and each `{$x -> …}` selector) as a wildcard. A template +//! that is only an argument (`{$msg}`) matches anything and is left out as unmeasurable. + +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::fmt::Write as _; +use std::path::{Path, PathBuf}; +use std::process::{Command, ExitCode}; +use std::sync::LazyLock; + +use mirth_lab::driver::{self, Record, Sweep}; +use mirth_lab::rustc::{self, Compile, Status}; +use mirth_lab::sitemap::home; +use mirth_lab::uitest::{self, Test}; +use regex::{Regex, RegexSet}; +use serde::{Deserialize, Serialize}; +use syn::spanned::Spanned; + +#[derive(clap::Args, Debug)] +pub struct Args { + /// The compiler (for --collect, and for the lint list). + #[arg(long, default_value_os_t = home("mirth-work/campaign/rustc/bin/rustc"))] + rustc: PathBuf, + /// Compile the corpus and save its diagnostics to /results.jsonl. + #[arg(long)] + collect: bool, + #[command(flatten)] + sweep: Sweep, + /// Collected results to report on (default: /results.jsonl). + #[arg(long)] + data: Vec, + /// The rust checkout (error code docs, diagnostic structs). + #[arg(long, default_value_os_t = home("mirth-work/rust"))] + rust: PathBuf, + /// Write the gap lists here. + #[arg(long)] + out: Option, +} + +#[derive(Serialize, Deserialize, Default)] +struct Diag { + level: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + code: String, + message: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + children: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + applicability: Vec, +} + +#[derive(Serialize, Deserialize)] +struct Rec { + test: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + skip: Option, + #[serde(default)] + diags: Vec, +} + +impl Record for Rec { + fn findings(&self) -> Vec { + Vec::new() + } + fn test(&self) -> &str { + &self.test + } +} + +fn collect_one(args: &Args, test: &Test) -> Rec { + let dir = driver::scratch_dir(&args.sweep); + let c = Compile::new(&args.rustc, &test.path, dir.path(), &test.flags, test.edition()).emit("metadata").json().timeout(120).run(); + let mut rec = Rec { test: test.rel.clone(), skip: None, diags: Vec::new() }; + if matches!(c.status, Status::Timeout) { + rec.skip = Some("timeout".into()); + return rec; + } + for d in rustc::diagnostics(&c.stderr) { + let mut children = Vec::new(); + let mut appl = BTreeSet::new(); + for node in d.walk() { + if !std::ptr::eq(node, &d) { + children.push(node.message.clone()); + } + for s in &node.spans { + if let Some(l) = &s.label { + children.push(l.clone()); + } + if s.suggested_replacement.is_some() { + appl.insert(s.suggestion_applicability.clone().unwrap_or_else(|| "Unspecified".into())); + } + } + } + children.sort(); + children.dedup(); + rec.diags.push(Diag { level: d.level.clone(), code: d.code().to_owned(), message: d.message.clone(), children, applicability: appl.into_iter().collect() }); + } + rec +} + +/// A message template as a regex: literal text, with `{$arg}` and `{$x -> …}` as wildcards. +fn template_regex(t: &str) -> Option { + let mut out = String::from("(?s)^"); + let mut literal = 0; + let mut rest = t; + while let Some(i) = rest.find('{') { + out.push_str(®ex::escape(&rest[..i])); + literal += rest[..i].trim().len(); + // Skip to the matching brace (selectors nest braces). + let mut depth = 0; + let mut end = rest.len(); + for (k, ch) in rest[i..].char_indices() { + match ch { + '{' => depth += 1, + '}' => { + depth -= 1; + if depth == 0 { + end = i + k + 1; + break; + } + } + _ => {} + } + } + out.push_str(".*?"); + rest = &rest[end.min(rest.len())..]; + } + literal += rest.trim().len(); + out.push_str(®ex::escape(rest)); + out.push('$'); + (literal >= 4).then_some(out) +} + +struct Template { + kind: &'static str, + file: String, + line: usize, + name: String, + text: String, +} + +/// The inline messages of `#[derive(Diagnostic)]` and `#[derive(Subdiagnostic)]` items. +fn templates(rust: &Path) -> (Vec