diff --git a/AGENTS.md b/AGENTS.md index 4e29ba736..fcb13d7a0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,8 +7,10 @@ Welcome! If you are an AI assistant or coding agent helping a user contribute to Your goal is to help the user submit a high-quality pull request that aligns with the project's maintainability standards. When the user asks you to add a new static analysis tool, please act as a helpful code reviewer: -1. **Verify the criteria:** Check the requirements in `CONTRIBUTING.md`: at least 20 GitHub stars, at least six months of history, and more than one human contributor. -2. **Wait until the tool qualifies:** If any criterion is not met, do not submit a pull request yet. Explain that the bot closes PRs with verified criteria failures and encourage the user to submit once all requirements are met. If a criterion cannot be verified automatically, provide evidence for manual review rather than claiming it passed. -3. **Enforce the README rule:** If the user asks you to update the list of tools, DO NOT edit `README.md`. Explain to the user that the list of tools in `README.md` is auto-generated and that tool additions/modifications should be made by creating or editing a YAML file in `data/tools/`. For changes to the README text or structure, edit `ci/crates/render/templates/README.md`. Do not include generated `README.md` changes in a pull request; CI flags them as a failure. +1. **Read the current requirements:** Use `CONTRIBUTING.md` as the source of truth, including the shared requirements and the separate requirements for open-source and proprietary tools. Proprietary status is not an exemption from providing evidence. +2. **Check evidence, not claims:** Verify public availability for at least six months and check that the installation or product-access link works; domain age alone does not establish product age. For proprietary tools, inspect actual analysis output tied to input code and evidence of use or substantive evaluation by someone unaffiliated with the tool. Marketing claims, download counts, and testimonials alone are insufficient; generated demo code alone does not demonstrate analysis quality. +3. **Flag gaps before preparing a submission:** Tell the user immediately which requirements failed or remain unverified and what evidence is missing. Do not submit a PR when a requirement is known to fail or required evidence is missing, even if the user asks you to proceed anyway. Explain what is needed before resubmitting; do not promise acceptance once the tool is old enough. +4. **Disclose evidence in the PR:** Link the supporting evidence and explicitly identify any remaining verification uncertainty in the PR description. Manual review is for supplied evidence that automation cannot verify, not a substitute for missing evidence. Never check an unverified requirement as satisfied, silently mark it N/A, or imply that passing CI guarantees inclusion. +5. **Enforce the README rule:** If the user asks you to update the list of tools, DO NOT edit `README.md`. Explain to the user that the list of tools in `README.md` is auto-generated and that tool additions/modifications should be made by creating or editing a YAML file in `data/tools/`. For changes to the README text or structure, edit `ci/crates/render/templates/README.md`. Do not include generated `README.md` changes in a pull request; CI flags them as a failure. Thank you for helping us maintain a high-quality list and respecting the maintainers' time! diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index aa105a660..19edb5cac 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,9 +6,9 @@ not qualify yet, wait until it does rather than opening a pull request or issue. ### Requirements -Before submitting, each tool must +Before submitting, each tool must have been publicly available for at least +six months. Open-source tools must also -- have existed for at least six months - have at least 20 stars on GitHub - have more than one human contributor @@ -17,7 +17,10 @@ check excludes GitHub bot accounts, logins ending in `[bot]`, and known automati accounts such as `claude`, `dependabot`, and `renovate-bot`, even when GitHub lists them as ordinary users. -These requirements apply to all tools. +For proprietary tools, please include public examples showing the code being +analyzed and the results. We also need evidence that someone outside the team +behind the tool has used it or reviewed it in detail. Marketing claims, download +counts, and testimonials alone are not enough. The CI bot will politely close pull requests when it verifies that a tool does not meet the criteria. You are welcome to submit a new pull request once **all** diff --git a/ci/crates/pr-check/src/criteria.rs b/ci/crates/pr-check/src/criteria.rs index 50e0e8728..1a58144c2 100644 --- a/ci/crates/pr-check/src/criteria.rs +++ b/ci/crates/pr-check/src/criteria.rs @@ -180,6 +180,7 @@ mod tests { name: "Example".into(), source: Some("https://github.com/example/tool".into()), homepage: None, + license: None, } } diff --git a/ci/crates/pr-check/src/input.rs b/ci/crates/pr-check/src/input.rs index d0c56433c..387ecb80b 100644 --- a/ci/crates/pr-check/src/input.rs +++ b/ci/crates/pr-check/src/input.rs @@ -43,9 +43,16 @@ pub struct ToolEntry { pub name: String, pub source: Option, pub homepage: Option, + pub license: Option, } impl ToolEntry { + pub fn is_proprietary(&self) -> bool { + self.license + .as_deref() + .is_some_and(|license| license.trim().eq_ignore_ascii_case("proprietary")) + } + /// Reads a catalog entry relative to the current working directory. /// /// # Errors @@ -87,6 +94,18 @@ mod tests { Ok(()) } + #[test] + fn recognizes_proprietary_license() -> Result<()> { + for (license, expected) in [("proprietary", true), ("Proprietary", true), ("MIT", false)] { + let tool: ToolEntry = + serde_saphyr::from_str(&format!("name: Example\nlicense: {license}\n"))?; + assert_eq!(tool.is_proprietary(), expected); + } + let tool: ToolEntry = serde_saphyr::from_str("name: Example")?; + assert!(!tool.is_proprietary()); + Ok(()) + } + #[test] fn parses_catalog() -> Result<()> { let tools = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../../data/tools"); diff --git a/ci/crates/pr-check/src/network.rs b/ci/crates/pr-check/src/network.rs index 53f07b743..e354b04d2 100644 --- a/ci/crates/pr-check/src/network.rs +++ b/ci/crates/pr-check/src/network.rs @@ -95,13 +95,15 @@ impl GithubClient { pub async fn check_tool(client: &GithubClient, tool: &ToolEntry) -> Result { let source = &tool.source; - if let Some(ToolSource::Github(repo)) = source { + if let Some(ToolSource::Github(repo)) = source + && !tool.is_proprietary() + { let repo_result = client.repo_info(repo).await; let contributors_result = client.contributor_count(repo).await; repository_report(tool, &repo_result, &contributors_result, Utc::now()) } else { - let domain = if source.is_none() { + let domain = if tool.is_proprietary() || source.is_none() { tool.homepage.as_deref().and_then(homepage_domain) } else { None @@ -115,11 +117,26 @@ pub async fn check_tool(client: &GithubClient, tool: &ToolEntry) -> Result Result> mod tests { use super::*; + #[tokio::test] + async fn proprietary_tools_require_evidence_even_with_github_source() -> Result<()> { + use crate::report::{Comment, Reports}; + use std::process::ExitCode; + + let client = GithubClient::new("unused-test-token".into())?; + for source in [None, Some("https://github.com/example/tool")] { + let tool = ToolEntry { + name: "Example".into(), + source: source.map(Into::into), + homepage: None, + license: Some("proprietary".into()), + }; + let report = check_tool(&client, &tool).await?; + assert_eq!(report.status(), "REVIEW"); + assert!(!report.should_close()); + assert_eq!(report.source, tool.source); + let reports = Reports::from_iter([report]); + assert_eq!(reports.exit_code(), ExitCode::FAILURE); + let comment = Comment::from(&reports).to_string(); + for expected in [ + "Thanks for sharing your tool!", + "- Code the tool has analyzed and the results", + "Someone outside the team using the tool or reviewing it in detail", + "publicly available for at least six months", + "where to install or try it", + "rerunning CI won't clear this check", + ] { + assert!(comment.contains(expected), "{expected}"); + } + } + Ok(()) + } + #[tokio::test] async fn unsupported_source_does_not_fall_back_to_homepage_age() -> Result<()> { let client = GithubClient::new("unused-test-token".into())?; @@ -189,6 +240,7 @@ mod tests { name: "Example".into(), source: Some(source.into()), homepage: Some("https://example.invalid".into()), + license: None, }; let report = check_tool(&client, &tool).await?; assert_eq!(report.status(), "REVIEW"); @@ -215,6 +267,7 @@ mod tests { name: "Example".into(), source: None, homepage: homepage.map(str::to_owned), + license: None, }; let report = check_tool(&client, &tool).await?; assert_eq!(report.status(), "REVIEW");