From 543daa63eaa7590043c9cabac2c28761872c6861 Mon Sep 17 00:00:00 2001 From: Gareth Paul Jones Date: Wed, 19 Aug 2026 20:02:41 +0000 Subject: [PATCH 1/3] Harden agent compatibility scoring --- .github/workflows/validate-plugins.yml | 9 + .../.cursor-plugin/plugin.json | 9 +- agent-compatibility/CHANGELOG.md | 23 +- agent-compatibility/README.md | 83 +- .../agents/compatibility-scan-review.md | 72 +- .../agents/docs-reliability-review.md | 69 +- agent-compatibility/agents/startup-review.md | 91 ++- .../agents/validation-review.md | 89 +- .../skills/check-agent-compatibility/SKILL.md | 71 +- .../scripts/run-deterministic-scan.mjs | 760 ++++++++++++++++++ .../scripts/synthesize-results.mjs | 454 +++++++++++ .../test/deterministic-scan.test.mjs | 210 +++++ .../cloudflare-worker/package.json | 12 + .../cloudflare-worker/src/worker.ts | 5 + .../cloudflare-worker/wrangler.toml | 3 + .../genuine-cli/bin/genuine-cli.mjs | 3 + .../genuine-cli/package.json | 7 + .../test/plugin-contract.test.mjs | 227 ++++++ .../test/synthesize-results.test.mjs | 401 +++++++++ 19 files changed, 2412 insertions(+), 186 deletions(-) create mode 100644 agent-compatibility/skills/check-agent-compatibility/scripts/run-deterministic-scan.mjs create mode 100644 agent-compatibility/skills/check-agent-compatibility/scripts/synthesize-results.mjs create mode 100644 agent-compatibility/test/deterministic-scan.test.mjs create mode 100644 agent-compatibility/test/fixtures/deterministic-scan/cloudflare-worker/package.json create mode 100644 agent-compatibility/test/fixtures/deterministic-scan/cloudflare-worker/src/worker.ts create mode 100644 agent-compatibility/test/fixtures/deterministic-scan/cloudflare-worker/wrangler.toml create mode 100644 agent-compatibility/test/fixtures/deterministic-scan/genuine-cli/bin/genuine-cli.mjs create mode 100644 agent-compatibility/test/fixtures/deterministic-scan/genuine-cli/package.json create mode 100644 agent-compatibility/test/plugin-contract.test.mjs create mode 100644 agent-compatibility/test/synthesize-results.test.mjs diff --git a/.github/workflows/validate-plugins.yml b/.github/workflows/validate-plugins.yml index 03e8e192..fed18b87 100644 --- a/.github/workflows/validate-plugins.yml +++ b/.github/workflows/validate-plugins.yml @@ -6,6 +6,12 @@ on: - ".cursor-plugin/marketplace.json" - "**/plugin.json" - "schemas/**" + - "agent-compatibility/agents/**" + - "agent-compatibility/skills/**" + - "agent-compatibility/test/**" + - "agent-compatibility/README.md" + - "agent-compatibility/CHANGELOG.md" + - ".github/workflows/validate-plugins.yml" jobs: validate: @@ -22,3 +28,6 @@ jobs: - name: Validate plugin definitions run: node scripts/validate-plugins.mjs + + - name: Test agent compatibility contracts + run: node --test agent-compatibility/test/*.test.mjs diff --git a/agent-compatibility/.cursor-plugin/plugin.json b/agent-compatibility/.cursor-plugin/plugin.json index a4c4aab1..c3ef9d04 100644 --- a/agent-compatibility/.cursor-plugin/plugin.json +++ b/agent-compatibility/.cursor-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "agent-compatibility", "displayName": "Agent Compatibility", - "version": "1.0.0", + "version": "1.1.0", "description": "CLI-backed repo compatibility scans plus agents that audit startup, validation, and docs against reality.", "author": { "name": "Cursor", @@ -21,12 +21,7 @@ "validation" ], "category": "developer-tools", - "tags": [ - "agents", - "compatibility", - "quality", - "workflow" - ], + "tags": ["agents", "compatibility", "quality", "workflow"], "skills": "./skills/", "agents": "./agents/" } diff --git a/agent-compatibility/CHANGELOG.md b/agent-compatibility/CHANGELOG.md index f726ed9a..aa22f7e1 100644 --- a/agent-compatibility/CHANGELOG.md +++ b/agent-compatibility/CHANGELOG.md @@ -1,11 +1,22 @@ # Changelog -All notable changes to this plugin will be documented here. +All notable changes to this plugin are documented here. ## Unreleased -- Renamed the full-pass skill to `check-agent-compatibility`. -- Renamed `deterministic-scan-review` to `compatibility-scan-review`. -- Renamed `docs-reality-review` to `docs-reliability-review`. -- Clarified the score model so `Agent Compatibility Score` is the final blended score and `Deterministic Compatibility Score` is the raw CLI score. -- Tightened the README, marketplace copy, and agent wording for public release. +## 1.1.0 - 2026-08-19 + +- Pinned the deterministic scanner to `agent-compatibility@0.1.7`. +- Added an executable scanner guard that validates the pinned version, scanned path, output shape, and Cloudflare Worker classification signals. +- Added a fail-closed result synthesizer as the sole owner of score validation, degraded states, and 70/30 arithmetic. +- Made the synthesizer reject missing evidence, mismatched targets, and stateful results without isolated execution provenance. +- Made startup and validation writable only inside isolated copies, with deploy, migration, credential, and paid-test boundaries. +- Added explicit target, budget, mutation, and evidence handoffs for every subagent. +- Changed specialist output to structured JSON with command outcomes and evidence. +- Added contract and fixture tests plus CI coverage for agent, skill, and helper-script changes. + +## 1.0.0 - 2026-03-25 + +- Added the full compatibility pass with deterministic, startup, validation, and docs-reliability reviews. +- Added the 70/30 deterministic and workflow score model. +- Added marketplace metadata and usage documentation. diff --git a/agent-compatibility/README.md b/agent-compatibility/README.md index 5dcd861d..428cc6c1 100644 --- a/agent-compatibility/README.md +++ b/agent-compatibility/README.md @@ -1,86 +1,113 @@ # Agent Compatibility -Cursor plugin for checking how well a repo holds up under agent workflows. It pairs the published `agent-compatibility` CLI with focused reviews for startup, validation, and docs reliability. +Cursor plugin for checking how well a repository holds up under agent workflows. It combines a pinned deterministic scanner with observed startup, validation, and docs-reliability reviews. -By default, the full pass returns one overall score and one short list of the highest-leverage fixes. If the user wants the full breakdown, the agents can expose the component scores and the reasoning behind them. +The default result is one score and a short list of evidence-backed fixes. Ask for a breakdown to see component scores, commands, scanner version, and file references. ## What it includes -- `check-agent-compatibility`: full compatibility pass -- `compatibility-scan-review`: raw CLI-backed scan -- `startup-review`: cold-start and bootstrap review -- `validation-review`: small-change verification review -- `docs-reliability-review`: docs reliability review +- `check-agent-compatibility`: orchestrates the full pass +- `compatibility-scan-review`: runs and validates the deterministic scan +- `startup-review`: tests cold bootstrap and startup in an isolated copy +- `validation-review`: tests the narrowest credible verification loop in an isolated copy +- `docs-reliability-review`: checks docs against repository interfaces without changing state + +## Reliability model + +The full pass uses one target root throughout: + +1. Run the pinned scanner through an executable guard that verifies its version, scanned path, output shape, and repository classification. +2. Run startup, validation, and docs reviews in parallel with the scanner result as context. +3. Require structured results with a matching canonical target, command outcomes, file evidence, and isolated execution provenance from every stateful specialist. +4. Retry malformed specialist output once instead of guessing missing values. +5. Pass the four lane results through an executable synthesizer that validates states and owns all score arithmetic. +6. Refuse to compute the blended score when the deterministic scan is unavailable or its classification conflicts with obvious repository signals. + +Startup and validation use writable isolated copies because install, build, test, and start commands often create state. They must not deploy, publish, migrate data, use production credentials, or run paid or live tests. The docs review remains read-only. ## Score model -- `Agent Compatibility Score`: final blended score shown to the user -- `Deterministic Compatibility Score`: raw score from the published CLI -- `Startup Compatibility Score`: how much guesswork it takes to boot the repo -- `Validation Loop Score`: how practical it is to verify a small change -- `Docs Reliability Score`: how closely the docs match the real setup path +- `Agent Compatibility Score`: final blended score +- `Deterministic Compatibility Score`: raw score from the pinned CLI +- `Startup Compatibility Score`: observed cold-start friction +- `Validation Loop Score`: observed small-change verification quality +- `Docs Reliability Score`: documented paths compared with real repository interfaces -The final score blends the deterministic scan with the workflow checks: +When every component is usable: ```text +workflow = round((startup + validation + docs) / 3) Agent Compatibility Score = round((deterministic * 0.7) + (workflow * 0.3)) ``` -The CLI also reports an accelerator layer for committed agent tooling. That extra context informs recommendations, but it does not inflate the deterministic compatibility score itself. +If the deterministic scan is unavailable or its classification is unreliable, the plugin reports `Agent Compatibility Score: unavailable` and shows a clearly labeled workflow-only score. It never invents the missing deterministic value or silently changes the weighting. + +The scanner's accelerator layer informs recommendations but does not inflate the deterministic score. ## How to use it -Use `check-agent-compatibility` when you want the full pass. That skill fans out to the four review agents above, then returns a compact result: +Use `check-agent-compatibility` for the full pass. A successful result stays compact: ```md ## Agent Compatibility Score: 72/100 Top fixes + - First issue - Second issue ``` -Ask for a breakdown if you want the component scores or the weighting. - ## CLI notes -The plugin does not bundle the scanner. It runs the published npm package when needed. +Plugin version 1.1.0 pins scanner version 0.1.7. Updating the scanner requires a plugin version change and contract-test update. -Default scan (compact terminal dashboard): +The plugin invokes the scanner through `skills/check-agent-compatibility/scripts/run-deterministic-scan.mjs`; direct commands below are for manual inspection. It computes final scores with `scripts/synthesize-results.mjs`, which fails closed on malformed or inconsistent lane results. + +Default scan: ```bash -npx -y agent-compatibility@latest . +npx -y agent-compatibility@0.1.7 . ``` JSON output: ```bash -npx -y agent-compatibility@latest --json . +npx -y agent-compatibility@0.1.7 --json . ``` Markdown output: ```bash -npx -y agent-compatibility@latest --md . +npx -y agent-compatibility@0.1.7 --md . ``` -Plain text output: +Plain-text output: ```bash -npx -y agent-compatibility@latest --text . +npx -y agent-compatibility@0.1.7 --text . ``` -Config override for ignored paths or weight overrides: +Config override for ignored paths or check weights: + +```bash +npx -y agent-compatibility@0.1.7 . --config ./agent-compatibility.config.json +``` + +The scanner is heuristic. It scores repository signals and surfaces likely friction; it is not a general code-quality verdict. + +## Validate the plugin + +Run the contract suite from the marketplace repository root: ```bash -npx -y agent-compatibility@latest . --config ./agent-compatibility.config.json +node --test agent-compatibility/test/*.test.mjs ``` -The scanner is heuristic. It scores repo signals and surfaces likely friction, but it is not a full quality verdict on the codebase. +The suite checks execution permissions, scanner pinning, executable guard fixtures, fail-closed score synthesis, degraded scoring, subagent handoffs, structured output, classification safeguards, side-effect boundaries, and CI coverage. ## Local install -If you want to use this plugin directly, symlink this directory into: +Symlink this directory into: ```bash ~/.cursor/plugins/local/agent-compatibility diff --git a/agent-compatibility/agents/compatibility-scan-review.md b/agent-compatibility/agents/compatibility-scan-review.md index 97d2419f..0980629b 100644 --- a/agent-compatibility/agents/compatibility-scan-review.md +++ b/agent-compatibility/agents/compatibility-scan-review.md @@ -1,40 +1,58 @@ --- name: compatibility-scan-review -description: Run the agent-compatibility CLI and return the raw repository score with its main problems +description: Run the pinned agent-compatibility CLI, validate its repository classification, and return structured evidence. model: fast -readonly: true +readonly: false --- # Compatibility scan review -Runs the published scanner and reports the raw repository score. - -## Trigger - -Use when the task is specifically to run the published `agent-compatibility` scanner and report the raw compatibility result. +Run the deterministic scanner without modifying the target repository. ## Workflow -1. Try the published scanner first with `npx -y agent-compatibility@latest --json ""`. -2. If you are clearly working inside the scanner source repo and the published package path fails for an environment reason, fall back to the local scanner entrypoint. -3. Only say the scanner is unavailable after you have actually tried the published package, and the local fallback when it is clearly available. -4. Prefer JSON when you need structured reasoning. Prefer Markdown when the user wants a direct report. -5. Keep the scanner's real score, summary direction, and problem ordering. -6. Do not bundle in startup, validation, or docs-reliability judgments. Those belong to separate agents. - -## Output +1. Require canonical absolute `Target root` and `Scanner helper` paths from the parent task. +2. Confirm `Scanner helper` ends in `scripts/run-deterministic-scan.mjs`, then run `node "" ""` exactly once. The helper runs `agent-compatibility@0.1.7`, validates the scanned path, and checks the scanner classification against obvious repository signals. +3. Return the helper's JSON exactly. Do not reinterpret its score, classification, reliability, command outcomes, or failure status. +4. If the helper process exits nonzero but emits valid JSON, return that JSON; `unavailable` is evidence about the tool or environment, not a repository score of zero. +5. If the helper cannot be read or emits invalid JSON, return `unavailable` using the schema below. Do not run the scanner directly or substitute a different package version. +6. Do not make startup, validation-loop, or docs-reliability judgments. -Reply in **plain text only** (no markdown fences, no `#` headings, no emphasis syntax). Use this layout: +Npm cache writes are allowed. Do not change files in the target repository. -First line: `Deterministic Compatibility Score: /100` - -Then a short summary paragraph. - -Then the line `Problems` followed by one bullet per line using `- `. +## Output -- Use the compatibility scan's real score. -- Keep accelerator context separate from the deterministic compatibility score itself. -- Include both rubric issues and accelerator issues when they matter. -- If there are no meaningful problems, under Problems write `- None.` -- Do not treat scanner availability as a defect in the target repo. -- If the scanner truly cannot be run, say that the deterministic scan is unavailable because of the tool environment, not because the repo lacks a compatibility CLI. +Return JSON only, with no markdown fence: + +Allowed `status` values are `complete`, `unreliable`, and `unavailable`. Allowed command `outcome` values are `passed`, `failed`, and `blocked`. +An obviously wrong classification is returned as `"unreliable"`, with the scanner's evidence preserved. + +```json +{ + "status": "complete", + "scoreName": "Deterministic Compatibility Score", + "score": 84, + "scannerVersion": "0.1.7", + "targetRoot": "/absolute/path", + "scannedPath": "/absolute/path", + "classification": "application", + "classificationReliable": true, + "summary": "Short evidence-based summary.", + "evidence": ["scannerVersion: 0.1.7", "scannedPath: /absolute/path"], + "problems": [ + { + "title": "Problem title", + "evidence": ["file:line or scanner evidence"], + "remediation": "Concrete fix" + } + ], + "commands": [ + { + "command": "npx -y agent-compatibility@0.1.7 --json ", + "outcome": "passed" + } + ] +} +``` + +Always return `targetRoot`, `summary`, and at least one `evidence` string. Use `null` for `score`, `scannerVersion`, or `classification` when unavailable. Use an empty `problems` array when no meaningful deterministic problem exists. diff --git a/agent-compatibility/agents/docs-reliability-review.md b/agent-compatibility/agents/docs-reliability-review.md index f199aaaf..0b6c90c9 100644 --- a/agent-compatibility/agents/docs-reliability-review.md +++ b/agent-compatibility/agents/docs-reliability-review.md @@ -1,44 +1,57 @@ --- name: docs-reliability-review -description: Check whether the documented setup and run paths reliably lead to the real working path +description: Check whether documented setup, run, and validation paths match the repository's real interfaces. model: fast readonly: true --- # Docs reliability review -Follows the written setup path and reports where the docs drift from reality. - -## Trigger - -Use when the user wants to know whether the repo documentation is actually trustworthy for an agent starting fresh. +Measure whether a cold agent can trust the written setup and workflow guidance. ## Workflow -1. If a compatibility scan result is already available from the parent task, use it as context. Otherwise run the compatibility scan once. -2. Read the obvious documentation surfaces: `README`, setup docs, env docs, and contribution or agent guidance. -3. Follow the documented setup and run path as literally as practical. -4. Note where docs are accurate, stale, incomplete, or misleading. -5. Pick a specific score instead of a round bucket. Start from these anchors and move a few points if the evidence clearly warrants it: - - around `93/100` if the docs lead to the working path with little or no correction. - - around `84/100` if the docs drift in places but an agent can still get to the right setup or run path without much guesswork. - - around `68/100` if the docs are stale enough that the agent has to reconstruct important steps from the tree or CI. - - around `27/100` if the docs point the agent down the wrong path or omit key steps you need to proceed. - - around `12/100` if the real path depends on private docs or internal context that is not available in the repo. -6. Prefer a specific score such as `81`, `85`, or `92` over a multiple of ten when that is the more honest read. +1. Require `Target root`, `Deterministic scan result`, `Time budget`, `Allowed mutations`, and `Required evidence` from the parent. +2. Read the README, setup and environment docs, contribution guidance, agent instructions, manifests, and root task definitions. +3. Trace documented install, run, and validation commands to their real scripts or targets without changing repository state. +4. Use the passed deterministic result as context. Do not install or rerun the scanner. +5. Record exact mismatches, missing prerequisites, stale names, unsupported claims, and commands with no real target. +6. Score the damage caused by drift. Minor wording differences should not drag an otherwise reliable path into the midrange. +7. Return status `complete` when the review produced a defensible score. Use `unavailable` only when the target or required files cannot be inspected. -## Output +## Scoring anchors -Reply in **plain text only** (no markdown fences, no `#` headings, no emphasis syntax). Use this layout: +- About `93`: docs lead to the working path with little or no correction. +- About `84`: limited drift exists, but recovery takes little guesswork. +- About `68`: important steps must be reconstructed from the tree or CI. +- About `27`: docs point down the wrong path or omit required steps. +- About `12`: the real path depends on unavailable private context. -First line: `Docs Reliability Score: /100` +Choose a specific score supported by file references. -Then a short summary paragraph. - -Then the line `Problems` followed by one bullet per line using `- `. +## Output -- Base the score on what happened when you followed the docs. -- Build Problems from real mismatches, omissions, or misleading guidance. -- If the repo is blocked on secrets or infrastructure, say so plainly and still use the same output shape. -- Minor drift or stale references should not drag a good repo into the mid-60s if the real path is still easy to recover. -- Score the damage from the drift, not the mere existence of drift. +Return JSON only, with no markdown fence: + +Allowed `status` values are `complete` and `unavailable`. + +```json +{ + "status": "complete", + "scoreName": "Docs Reliability Score", + "score": 84, + "targetRoot": "/absolute/target/path", + "summary": "Short evidence-based summary.", + "evidence": ["README.md:20 maps to package.json#scripts.test"], + "problems": [ + { + "title": "Problem title", + "evidence": ["file:line"], + "remediation": "Concrete fix" + } + ], + "commands": [] +} +``` + +Always return `targetRoot`, `summary`, and at least one `evidence` string. Use `null` for `score` only when status is `unavailable`. diff --git a/agent-compatibility/agents/startup-review.md b/agent-compatibility/agents/startup-review.md index a6880ae6..8f0e66ed 100644 --- a/agent-compatibility/agents/startup-review.md +++ b/agent-compatibility/agents/startup-review.md @@ -1,51 +1,76 @@ --- name: startup-review -description: Try to bootstrap and start a repository like a cold agent, then report where the path breaks down +description: Bootstrap and start a repository in an isolated copy, then score the observed cold-start path. model: fast -readonly: true +readonly: false --- # Startup review -Tries the cold-start path and reports how much work it takes to get the repo running. +Measure whether a cold agent can reach the repository's documented first success. -## Trigger +## Safety boundary -Use when the user wants to know whether a repo is actually easy to start, not just whether it claims to be. +- Create a dedicated temporary copy outside `Target root`; never share it with another review. If isolation is unavailable, do not run a command that could modify tracked files. +- Do not run deploy, release, publish, migration, destructive reset, or external data-write commands. +- Do not run paid or live tests, or anything documented as costing money. +- Do not request or use production credentials. Treat missing secrets or accounts as observed startup friction. +- Do not modify tracked files. Record any untracked build output or dependency directories created by the startup path. +- Stop processes you start and do not leave ports or services running. ## Workflow -1. If a compatibility scan result is already available from the parent task, use it as context. Otherwise run the compatibility scan once. -2. Read the obvious startup surfaces: `README`, scripts, toolchain files, env examples, and workflow docs. -3. Pick the most likely bootstrap path and startup command. -4. Try to reach first success inside a fixed time budget. -5. If the first path fails, allow a small amount of recovery and note what you had to infer. -6. Do not infer a startup failure from a lockfile, a bound port, or an existing repo-local process by itself. -7. Only call startup blocked or failed when your own startup attempt fails, or when the documented startup path cannot be completed within the budget. -8. Pick a specific score instead of a round bucket. Start from these anchors and move a few points if the evidence clearly warrants it: - - around `93/100` if the main startup path works inside the time budget, even if it needs ordinary local prerequisites such as Docker or a database. - - around `84/100` if the repo starts, but only after some digging, a recovery step, or heavier setup than the docs suggest. - - around `68/100` if a startup path probably exists but stays too manual, too ambiguous, or too expensive for normal agent use. - - around `27/100` if you cannot get a credible startup path working from the repo and docs you have. - - around `12/100` if the path is blocked on secrets, accounts, or infrastructure you cannot reasonably access. -9. Prefer a specific score such as `82`, `85`, or `91` over a multiple of ten when that is the more honest read. -10. Return the result in the same plain-text report shape as the deterministic scan. +1. Require `Target root`, `Deterministic scan result`, `Time budget`, `Allowed mutations`, and `Required evidence` from the parent. +2. Create and canonicalize a dedicated `executionRoot` copy outside `Target root`. Record `isolation` as `isolated-copy`; never run stateful commands in `Target root`. +3. Capture the starting `git status --short` in the isolated copy when the target is a Git checkout. +4. Read the README, scripts, toolchain files, environment examples, and workflow docs. +5. Pick the most likely documented bootstrap and startup path. Run it within the supplied time budget. +6. Permit one recovery attempt when the first path fails. Record every inferred step. +7. Verify the success condition implied by the docs. Do not require HTTP when the documented runtime is not an HTTP service. +8. Compare final tracked-file status with the starting status. A startup command that unexpectedly changes tracked files is a problem. +9. Return status `complete` when the review produced a defensible score, including a low score caused by inaccessible secrets or infrastructure. Use `unavailable` only for a tool or environment failure that prevents evaluation; in that case use `null` for `executionRoot` and `unavailable` for `isolation` if no copy was created. -## Output +## Scoring anchors + +- About `93`: the main path works within budget with only ordinary prerequisites. +- About `84`: it works after limited digging or one recovery step. +- About `68`: a credible path exists but remains manual, ambiguous, or expensive. +- About `27`: no credible path works from the repository and docs. +- About `12`: the path is blocked on secrets, accounts, or inaccessible infrastructure. -Reply in **plain text only** (no markdown fences, no `#` headings, no emphasis syntax). Use this layout: +Choose a specific score supported by the evidence. + +## Output -First line: `Startup Compatibility Score: /100` +Return JSON only, with no markdown fence: -Then a short summary paragraph. +Allowed `status` values are `complete` and `unavailable`. Allowed command `outcome` values are `passed`, `failed`, and `blocked`. -Then the line `Problems` followed by one bullet per line using `- `. +```json +{ + "status": "complete", + "scoreName": "Startup Compatibility Score", + "score": 84, + "targetRoot": "/absolute/target/path", + "executionRoot": "/temporary/isolated/copy", + "isolation": "isolated-copy", + "summary": "Short evidence-based summary.", + "evidence": ["npm run dev reached the documented success condition"], + "problems": [ + { + "title": "Problem title", + "evidence": ["command outcome or file:line"], + "remediation": "Concrete fix" + } + ], + "commands": [ + { + "command": "documented command", + "outcome": "passed", + "evidence": "short observed result" + } + ] +} +``` -- Base the score on what happened when you actually tried to start the repo. -- Build Problems from the real startup friction you observed. -- If the repo is blocked on secrets, accounts, or external infra, say that plainly and still use the same output shape. -- Do not assume a Next.js lockfile or a port that does not answer HTTP immediately is a repo problem. -- Do not require an HTTP response unless the documented startup path clearly implies one and you actually started that path yourself. -- If the environment starts successfully, treat that as a strong result. Record the friction, but do not score it like a near-failure. -- Treat Docker, local services, and other standard dev prerequisites as friction, not failure. -- Error-message quality is secondary here unless it actually prevents startup or recovery. +Always return `targetRoot`, `summary`, and at least one `evidence` string. Use `null` for `score` only when status is `unavailable`. diff --git a/agent-compatibility/agents/validation-review.md b/agent-compatibility/agents/validation-review.md index 1630597e..2eeb935d 100644 --- a/agent-compatibility/agents/validation-review.md +++ b/agent-compatibility/agents/validation-review.md @@ -1,50 +1,75 @@ --- name: validation-review -description: Assess whether an agent can verify a small change without guessing or running an unnecessarily heavy loop +description: Run the narrowest credible validation loop in an isolated copy and score its usefulness for small changes. model: fast -readonly: true +readonly: false --- # Validation review -Checks whether an agent can verify a small change without falling back to a full-repo loop. +Measure whether an agent can verify a small change without guessing or defaulting to an unnecessarily heavy loop. -## Trigger +## Safety boundary -Use when the user wants to know whether an agent can safely verify its own work in a repo. +- Create a dedicated temporary copy outside `Target root`; never share it with another review. If isolation is unavailable, do not run a command that could modify tracked files. +- Do not run deploy, release, publish, migration, destructive reset, or external data-write commands. +- Do not run paid or live tests, or anything documented as costing money. +- Do not request or use production credentials. +- Do not modify tracked files. Record generated files, caches, dependency directories, and other untracked output. ## Workflow -1. If a compatibility scan result is already available from the parent task, use it as context. Otherwise run the compatibility scan once. -2. Inspect the repo's declared test, lint, check, and typecheck paths. -3. Decide whether there is a practical scoped loop for a small change. -4. Try the most relevant validation path. -5. Judge whether the result is: - - targeted - - actionable - - noisy - - too expensive for normal iteration -6. Pick a specific score instead of a round bucket. Start from these anchors and move a few points if the evidence clearly warrants it: - - around `93/100` if there is a repeatable validation path and it gives useful signal, even if it is broader than ideal. - - around `84/100` if validation works but is heavier than it should be, repo-wide, or split across a few commands. - - around `68/100` if a valid loop probably exists but picking the right one takes guesswork or the output is too noisy to trust quickly. - - around `27/100` if there is no practical validation loop you can actually use. - - around `12/100` if the loop is blocked on secrets, accounts, or infrastructure you cannot reasonably access. -7. Prefer a specific score such as `83`, `86`, or `91` over a multiple of ten when that is the more honest read. -8. Return the result in the same plain-text report shape as the deterministic scan. +1. Require `Target root`, `Deterministic scan result`, `Time budget`, `Allowed mutations`, and `Required evidence` from the parent. +2. Create and canonicalize a dedicated `executionRoot` copy outside `Target root`. Record `isolation` as `isolated-copy`; never run stateful commands in `Target root`. +3. Capture the starting `git status --short` in the isolated copy when the target is a Git checkout. +4. Inspect declared test, lint, format-check, typecheck, and task-runner paths. +5. Choose the narrowest representative validation command for a small change. Prefer a documented file, package, or test target over a full-repository suite. +6. Run the command within the supplied time budget and assess whether the result is targeted, actionable, trustworthy, and affordable for normal iteration. +7. If no scoped command exists, run the lightest credible broader check and record the extra cost as friction. +8. Compare final tracked-file status with the starting status. Unexpected tracked-file changes are a problem. +9. Return status `complete` when the review produced a defensible score. Use `unavailable` only for a tool or environment failure that prevents evaluation; in that case use `null` for `executionRoot` and `unavailable` for `isolation` if no copy was created. -## Output +## Scoring anchors + +- About `93`: a repeatable scoped loop gives useful signal. +- About `84`: validation is reliable but broader or more fragmented than ideal. +- About `68`: the loop exists but selection or output requires material guesswork. +- About `27`: no practical validation loop can be run. +- About `12`: validation depends on secrets, accounts, or inaccessible infrastructure. -Reply in **plain text only** (no markdown fences, no `#` headings, no emphasis syntax). Use this layout: +Choose a specific score supported by the evidence. + +## Output -First line: `Validation Loop Score: /100` +Return JSON only, with no markdown fence: -Then a short summary paragraph. +Allowed `status` values are `complete` and `unavailable`. Allowed command `outcome` values are `passed`, `failed`, and `blocked`. -Then the line `Problems` followed by one bullet per line using `- `. +```json +{ + "status": "complete", + "scoreName": "Validation Loop Score", + "score": 84, + "targetRoot": "/absolute/target/path", + "executionRoot": "/temporary/isolated/copy", + "isolation": "isolated-copy", + "summary": "Short evidence-based summary.", + "evidence": ["targeted test command passed"], + "problems": [ + { + "title": "Problem title", + "evidence": ["command outcome or file:line"], + "remediation": "Concrete fix" + } + ], + "commands": [ + { + "command": "validation command", + "outcome": "passed", + "evidence": "short observed result" + } + ] +} +``` -- Base the score on the loop you actually tried. -- Build Problems from the real validation friction you observed. -- Prefer concrete issues like "only full-repo test path exists" over generic quality advice. -- Do not score a repo in the mid-60s just because the loop is heavy. If an agent can still verify changes reliably, keep it in the good range and note the cost. -- Noisy logs and extra warnings matter only when they hide the actual validation result. +Always return `targetRoot`, `summary`, and at least one `evidence` string. Use `null` for `score` only when status is `unavailable`. diff --git a/agent-compatibility/skills/check-agent-compatibility/SKILL.md b/agent-compatibility/skills/check-agent-compatibility/SKILL.md index 6d88d4a5..28dbf623 100644 --- a/agent-compatibility/skills/check-agent-compatibility/SKILL.md +++ b/agent-compatibility/skills/check-agent-compatibility/SKILL.md @@ -1,46 +1,67 @@ --- name: check-agent-compatibility -description: Run the full repository compatibility pass: scanner score, startup path, validation loop, and docs reliability. +description: Run an evidence-backed repository compatibility pass across deterministic signals, startup, validation, and docs reliability. --- # Check agent compatibility ## Trigger -Use when the user wants the full compatibility pass for a repo. +Use when the user wants the full agent-compatibility review for a repository. ## Workflow -1. Launch `compatibility-scan-review` to run the CLI and capture the raw repository score and main issues. -2. Launch `startup-review` to verify whether the repo can actually be booted by an agent. -3. Launch `validation-review` to check whether an agent can verify a small change without an unnecessarily heavy loop. -4. Launch `docs-reliability-review` to see whether the documented setup and run paths reliably match reality. -5. Use one subagent per task. Do not collapse these checks into one agent prompt. -6. Compute an internal workflow score as the rounded average of: - - `Startup Compatibility Score` - - `Validation Loop Score` - - `Docs Reliability Score` -7. Compute an `Agent Compatibility Score` as: - - `round((deterministic_score * 0.7) + (workflow_score * 0.3))` -8. Synthesize the results into one final response. - -When scoring internally, use specific non-round workflow scores for the behavioral checks rather than coarse round buckets. If startup, validation, or docs mostly work, treat them as good-with-friction rather than defaulting to the mid-60s. Do not create a low workflow score just because logs are noisy or the error text is rough. +1. Resolve the requested repository to one canonical absolute `Target root`. Use that same path for every task. Resolve `Skill root` as the directory containing this `SKILL.md`; helper paths below are relative to that directory. +2. Launch `compatibility-scan-review` first. Its task prompt must contain: + - `Target root`: the canonical path. + - `Scanner helper`: `/scripts/run-deterministic-scan.mjs`. + - `Deterministic scan result`: `not available yet`. + - `Time budget`: 3 minutes. + - `Allowed mutations`: npm cache writes needed to run the pinned scanner; no target-repository changes. + - `Required evidence`: the scanner helper's complete JSON result. +3. Validate the returned JSON and retry the same subagent once if it is malformed or misses a required field. Preserve `unreliable` and `unavailable` as real statuses. +4. Launch `startup-review`, `validation-review`, and `docs-reliability-review` in parallel, one subagent per task. Require startup and validation to create separate temporary project copies outside `Target root`. Every task prompt must contain: + - `Target root`: the same canonical path. + - `Deterministic scan result`: the complete scan JSON, including status and classification reliability. + - `Time budget`: 10 minutes for startup, 8 minutes for validation, and 5 minutes for docs. + - `Allowed mutations`: isolated-checkout writes only for startup and validation; none for docs. Never allow deploys, paid tests, production credentials, migrations, or external data changes. + - `Required evidence`: exact `targetRoot`, commands attempted and outcomes, file references, observed friction, and reasons for the score. Startup and validation must also return canonical `executionRoot` and `isolation` provenance. +5. Validate each returned JSON and retry malformed output once. A tool or environment failure is `unavailable`; never convert it to a repository score of zero. +6. Create a JSON object with exactly four top-level fields: `deterministic`, `startup`, `validation`, and `docs`. Each value is the corresponding complete specialist result. Write it to a temporary file outside `Target root`. +7. Run `node "/scripts/synthesize-results.mjs" ""`, then remove the temporary file. Use the synthesizer output exactly for score availability, component scores, statuses, and arithmetic. If the helper rejects the results, retry the malformed specialist once; if validation still fails, report no aggregate score. +8. You must not compute the 70/30 blend yourself or substitute a different formula. With an unusable deterministic result and three usable workflow lanes, the synthesizer returns degraded workflow-only evidence. When any workflow lane is unavailable, it returns no aggregate score; do not impute its score. +9. Prioritize only fixes backed by the returned evidence. Deduplicate overlapping problems and prefer fixes that improve more than one lane. + +Use specific workflow scores rather than coarse buckets. Ordinary prerequisites and noisy logs are friction, not failure, unless they prevent the documented path from working. ## Output -Respond in markdown, but keep it minimal. Do not use fenced code blocks. +Keep the default response compact. + +When all four results are usable: + +```text +## Agent Compatibility Score: N/100 -Show only one score, as a level-two heading: `## Agent Compatibility Score: N/100`. Do not show how it was computed, including weights, formula, deterministic score, workflow score, per-check scores, or arithmetic, unless the user explicitly asks for a breakdown. +Top fixes +- First evidence-backed fix +- Second evidence-backed fix +``` -Then a flat, prioritized list labeled `Top fixes` with one issue per line, each line starting with `- `. +When the deterministic scan is unavailable or its classification is unreliable: -If the deterministic scanner cannot be run because of tool environment issues, say that separately and do not treat it as a repo defect or penalize the repo. Fold deterministic and behavioral findings into that one list instead of separate sections. Focus on the fixes that would most improve real agent workflows. Do not include a separate summary unless the user asks for more detail. +```text +## Agent Compatibility Score: unavailable -Example shape: +## Workflow Compatibility Score: N/100 -## Agent Compatibility Score: 72/100 +The deterministic score was not used: . Top fixes -- First issue -- Second issue -- Third issue +- First evidence-backed fix +- Second evidence-backed fix +``` + +If any workflow lane is unavailable, report `## Agent Compatibility Score: unavailable`, omit `Workflow Compatibility Score: N/100`, name the unavailable lane, and do not impute its score. + +Render the score headings from the synthesizer output. Show scanner version, component scores, statuses, arithmetic, and supporting evidence only when the user asks for a breakdown. diff --git a/agent-compatibility/skills/check-agent-compatibility/scripts/run-deterministic-scan.mjs b/agent-compatibility/skills/check-agent-compatibility/scripts/run-deterministic-scan.mjs new file mode 100644 index 00000000..051fa211 --- /dev/null +++ b/agent-compatibility/skills/check-agent-compatibility/scripts/run-deterministic-scan.mjs @@ -0,0 +1,760 @@ +#!/usr/bin/env node + +import { execFile } from "node:child_process"; +import { + existsSync, + readFileSync, + readdirSync, + realpathSync, + statSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { isAbsolute, join, relative, resolve, sep } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { promisify } from "node:util"; + +export const SCANNER_VERSION = "0.1.7"; +export const SCANNER_SPECIFIER = `agent-compatibility@${SCANNER_VERSION}`; + +const SCORE_NAME = "Deterministic Compatibility Score"; +const execFileAsync = promisify(execFile); +const sourceExtensions = [ + "js", + "cjs", + "mjs", + "ts", + "cts", + "mts", + "py", + "rb", + "sh", + "bash", + "zsh", + "go", + "rs", + "php", + "ex", + "exs", +]; +const cliFrameworkPackages = [ + "@oclif/core", + "cac", + "citty", + "clipanion", + "commander", + "meow", + "sade", + "yargs", +]; + +function isRecord(value) { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function readText(path) { + try { + return readFileSync(path, "utf8"); + } catch { + return null; + } +} + +function readJsonObject(path) { + const content = readText(path); + if (content === null) { + return null; + } + + try { + const parsed = JSON.parse(content); + return isRecord(parsed) ? parsed : null; + } catch { + return null; + } +} + +function relativePath(root, path) { + return relative(root, path).split(sep).join("/"); +} + +function uniqueSorted(values) { + return [...new Set(values)].sort(); +} + +function executableLookingFile(name) { + if (!name.includes(".")) { + return true; + } + + const extension = name.split(".").at(-1)?.toLowerCase(); + return sourceExtensions.includes(extension); +} + +function directoryEntrypoints(targetRoot, directoryName) { + const directory = join(targetRoot, directoryName); + if (!existsSync(directory)) { + return []; + } + + const found = []; + const visit = (currentDirectory, depth) => { + let entries; + try { + entries = readdirSync(currentDirectory, { withFileTypes: true }); + } catch { + return; + } + + for (const entry of entries) { + if (entry.name.startsWith(".")) { + continue; + } + + const path = join(currentDirectory, entry.name); + if (entry.isFile() && executableLookingFile(entry.name)) { + found.push(relativePath(targetRoot, path)); + } else if (entry.isDirectory() && depth < 2) { + visit(path, depth + 1); + } + } + }; + + visit(directory, 0); + return found; +} + +function existingSourceEntrypoints(targetRoot, stems) { + const found = []; + for (const stem of stems) { + for (const extension of sourceExtensions) { + const path = join(targetRoot, `${stem}.${extension}`); + if (existsSync(path)) { + found.push(relativePath(targetRoot, path)); + } + } + } + return found; +} + +function packageSignals(targetRoot) { + const packageJson = readJsonObject(join(targetRoot, "package.json")); + if (packageJson === null) { + return { + cli: [], + cloudflareDependencies: [], + cloudflareScripts: [], + }; + } + + const cli = []; + if ( + (typeof packageJson.bin === "string" && packageJson.bin.length > 0) || + (isRecord(packageJson.bin) && Object.keys(packageJson.bin).length > 0) + ) { + cli.push("package.json#bin"); + } + + const scripts = isRecord(packageJson.scripts) ? packageJson.scripts : {}; + const cloudflareScripts = []; + for (const [name, command] of Object.entries(scripts)) { + if (typeof command !== "string") { + continue; + } + + if ( + /^cli(?::|$)/i.test(name) || + /(?:^|[\s;&|])(?:node|bun|tsx|ts-node|deno\s+run)\s+(?:\.\/)?(?:src\/)?(?:cli\.[cm]?[jt]s|bin\/|cmd\/)/i.test( + command, + ) + ) { + cli.push(`package.json#scripts.${name}`); + } + + if ( + /\bwrangler(?:@[\w.-]+)?\s+(?:dev|deploy|tail|versions)\b/i.test(command) + ) { + cloudflareScripts.push(`package.json#scripts.${name}`); + } + } + + const runtimeDependencies = { + ...(isRecord(packageJson.dependencies) ? packageJson.dependencies : {}), + ...(isRecord(packageJson.optionalDependencies) + ? packageJson.optionalDependencies + : {}), + }; + for (const framework of cliFrameworkPackages) { + if (Object.hasOwn(runtimeDependencies, framework)) { + cli.push(`package.json#dependencies.${framework}`); + } + } + + const allDependencies = { + ...runtimeDependencies, + ...(isRecord(packageJson.devDependencies) + ? packageJson.devDependencies + : {}), + }; + const cloudflareDependencies = [ + "wrangler", + "@cloudflare/workers-types", + "@cloudflare/vitest-pool-workers", + ] + .filter((dependency) => Object.hasOwn(allDependencies, dependency)) + .map((dependency) => `package.json#dependencies.${dependency}`); + + return { + cli, + cloudflareDependencies, + cloudflareScripts, + }; +} + +function nonNodeCliSignals(targetRoot) { + const signals = []; + const pyproject = readText(join(targetRoot, "pyproject.toml")); + if ( + pyproject !== null && + /^\[(?:project\.scripts|tool\.poetry\.scripts)\]\s*$/m.test(pyproject) + ) { + signals.push("pyproject.toml#scripts"); + } + + const setupPy = readText(join(targetRoot, "setup.py")); + if (setupPy !== null && /console_scripts\s*[=:]/.test(setupPy)) { + signals.push("setup.py#console_scripts"); + } + + const cargoToml = readText(join(targetRoot, "Cargo.toml")); + if (cargoToml !== null && /^\[\[bin\]\]\s*$/m.test(cargoToml)) { + signals.push("Cargo.toml#[[bin]]"); + } + + return signals; +} + +function cliFrameworkImportSignals(targetRoot) { + const candidates = existingSourceEntrypoints(targetRoot, [ + "index", + "src/index", + "main", + "src/main", + ]); + const signals = []; + + for (const candidate of candidates) { + const content = readText(join(targetRoot, candidate)); + if (content === null) { + continue; + } + + for (const framework of cliFrameworkPackages) { + const escapedFramework = framework.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const importPattern = new RegExp( + `(?:from\\s+|require\\(\\s*)["']${escapedFramework}(?:["'/])`, + ); + if (importPattern.test(content)) { + signals.push(`${candidate}#imports.${framework}`); + } + } + } + + return signals; +} + +function wranglerConfigSignals(targetRoot) { + const configNames = ["wrangler.toml", "wrangler.json", "wrangler.jsonc"]; + const configs = configNames.filter((name) => + existsSync(join(targetRoot, name)), + ); + const entrypoints = []; + + for (const config of configs) { + const content = readText(join(targetRoot, config)); + if (content === null) { + continue; + } + + let main = null; + if (config === "wrangler.toml") { + main = content.match(/^\s*main\s*=\s*["']([^"']+)["']/m)?.[1] ?? null; + } else if (config === "wrangler.json") { + const parsed = readJsonObject(join(targetRoot, config)); + main = typeof parsed?.main === "string" ? parsed.main : null; + } + + if (main !== null && existsSync(resolve(targetRoot, main))) { + entrypoints.push(`${config}#main`); + } + } + + return { configs, entrypoints }; +} + +export function buildScannerCommands(targetRoot) { + return { + version: { + command: "npx", + args: ["-y", SCANNER_SPECIFIER, "--version"], + }, + scan: { + command: "npx", + args: ["-y", SCANNER_SPECIFIER, "--json", targetRoot], + }, + }; +} + +export function inspectRepositorySignals(targetRoot) { + const packageResult = packageSignals(targetRoot); + const directoryCliEntrypoints = [ + ...directoryEntrypoints(targetRoot, "bin"), + ...directoryEntrypoints(targetRoot, "cmd"), + ]; + const sourceCliEntrypoints = existingSourceEntrypoints(targetRoot, [ + "cli", + "src/cli", + ]); + const nonNodeSignals = nonNodeCliSignals(targetRoot); + const frameworkImports = cliFrameworkImportSignals(targetRoot); + const cliEntrypoints = uniqueSorted([ + ...packageResult.cli, + ...directoryCliEntrypoints, + ...sourceCliEntrypoints, + ...nonNodeSignals, + ...frameworkImports, + ]); + + const wrangler = wranglerConfigSignals(targetRoot); + const conventionalWorkerEntrypoints = existingSourceEntrypoints(targetRoot, [ + "worker", + "src/worker", + ]); + const cloudflareWorker = uniqueSorted([ + ...wrangler.configs, + ...wrangler.entrypoints, + ...packageResult.cloudflareDependencies, + ...packageResult.cloudflareScripts, + ...conventionalWorkerEntrypoints, + ]); + const cloudflareSignalGroups = [ + wrangler.configs, + [...wrangler.entrypoints, ...conventionalWorkerEntrypoints], + packageResult.cloudflareDependencies, + packageResult.cloudflareScripts, + ].filter((group) => group.length > 0).length; + + return { + targetRoot, + isCloudflareWorker: cloudflareSignalGroups >= 2, + hasCliEntrypoint: + packageResult.cli.length > 0 || + directoryCliEntrypoints.length > 0 || + sourceCliEntrypoints.length > 0 || + nonNodeSignals.length > 0, + hasCliSignal: cliEntrypoints.length > 0, + cloudflareWorker, + cli: cliEntrypoints, + }; +} + +function classificationKind(classification) { + if (typeof classification === "string") { + return classification.trim().toLowerCase(); + } + if (isRecord(classification) && typeof classification.kind === "string") { + return classification.kind.trim().toLowerCase(); + } + return null; +} + +export function assessClassification(classification, signals) { + const kind = classificationKind(classification); + if (kind === null) { + return { + reliable: false, + reason: "The scanner did not report a repository classification.", + evidence: [], + }; + } + + if ( + kind === "cli" && + signals.isCloudflareWorker === true && + signals.hasCliSignal !== true + ) { + return { + reliable: false, + reason: + 'The scanner reported "cli", but strong repository signals identify a Cloudflare Worker and no package bin, bin/cmd entrypoint, CLI script, or CLI framework signal was found.', + evidence: [...signals.cloudflareWorker], + }; + } + + return { + reliable: true, + reason: null, + evidence: [], + }; +} + +export function validateScannedPath(targetRoot, scannedPath) { + return ( + typeof targetRoot === "string" && + typeof scannedPath === "string" && + isAbsolute(targetRoot) && + isAbsolute(scannedPath) && + resolve(targetRoot) === resolve(scannedPath) + ); +} + +function scannerProblems(recommendations) { + if (!Array.isArray(recommendations)) { + return []; + } + + return recommendations.filter(isRecord).map((recommendation) => ({ + title: + typeof recommendation.title === "string" + ? recommendation.title + : typeof recommendation.checkId === "string" + ? recommendation.checkId + : "Scanner recommendation", + evidence: Array.isArray(recommendation.evidence) + ? recommendation.evidence.filter((value) => typeof value === "string") + : [], + remediation: + typeof recommendation.remediation === "string" + ? recommendation.remediation + : "Review the scanner evidence and add the missing repository signal.", + })); +} + +function unavailableResult({ + targetRoot, + scannerVersion = null, + summary, + commands = [], +}) { + return { + status: "unavailable", + scoreName: SCORE_NAME, + score: null, + scannerVersion, + targetRoot, + scannedPath: null, + classification: null, + classificationReliable: false, + classificationEvidence: [], + summary, + evidence: [summary], + problems: [], + commands, + }; +} + +export function evaluateScannerOutput({ + targetRoot, + scannerVersion, + scanOutput, + signals, +}) { + if (!isRecord(scanOutput)) { + return unavailableResult({ + targetRoot, + scannerVersion, + summary: "Pinned scanner returned JSON with an invalid top-level value.", + }); + } + + const score = scanOutput.overallScore; + if ( + typeof score !== "number" || + !Number.isFinite(score) || + score < 0 || + score > 100 + ) { + return unavailableResult({ + targetRoot, + scannerVersion, + summary: "Pinned scanner JSON did not contain a valid overallScore.", + }); + } + + const kind = classificationKind(scanOutput.classification); + const classificationAssessment = assessClassification( + scanOutput.classification, + signals, + ); + const pathMatches = validateScannedPath(targetRoot, scanOutput.scannedPath); + const reliabilityProblems = []; + if (!pathMatches) { + reliabilityProblems.push( + `Scanner scannedPath ${JSON.stringify(scanOutput.scannedPath ?? null)} does not match target root ${JSON.stringify(targetRoot)}.`, + ); + } + if (!classificationAssessment.reliable) { + reliabilityProblems.push(classificationAssessment.reason); + } + + const reliable = reliabilityProblems.length === 0; + const maturity = + typeof scanOutput.maturity === "string" ? ` (${scanOutput.maturity})` : ""; + const summary = reliable + ? `Scanner ${scannerVersion} classified the repository as ${kind} and scored it ${score}/100${maturity}.` + : `Deterministic scan is unreliable: ${reliabilityProblems.join(" ")}`; + + return { + status: reliable ? "complete" : "unreliable", + scoreName: SCORE_NAME, + score, + scannerVersion, + targetRoot, + scannedPath: + typeof scanOutput.scannedPath === "string" + ? scanOutput.scannedPath + : null, + classification: kind, + classificationReliable: reliable, + classificationEvidence: classificationAssessment.evidence, + summary, + evidence: [ + `scannerVersion: ${scannerVersion}`, + `scannedPath: ${scanOutput.scannedPath}`, + `classification: ${kind}`, + ...classificationAssessment.evidence, + ], + problems: scannerProblems(scanOutput.recommendations), + commands: [], + }; +} + +function canonicalTargetRoot(targetRoot) { + if (typeof targetRoot !== "string" || targetRoot.trim() === "") { + throw new Error("A target root argument is required."); + } + + const absolutePath = resolve(targetRoot); + const stats = statSync(absolutePath); + if (!stats.isDirectory()) { + throw new Error(`Target root is not a directory: ${absolutePath}`); + } + return realpathSync(absolutePath); +} + +function compactFailure(value) { + if (typeof value !== "string") { + return null; + } + const compact = value.trim().replace(/\s+/g, " "); + return compact.length > 0 ? compact.slice(0, 1_000) : null; +} + +function commandFailure(result) { + return ( + compactFailure(result?.error) ?? + compactFailure(result?.stderr) ?? + (Number.isInteger(result?.exitCode) + ? `command exited with code ${result.exitCode}` + : "command could not be executed") + ); +} + +function shellDisplay(command) { + const safeArgument = (argument) => + /^[A-Za-z0-9@%_+=:,./-]+$/.test(argument) + ? argument + : JSON.stringify(argument); + return [command.command, ...command.args.map(safeArgument)].join(" "); +} + +async function executeScannerCommand(command) { + try { + const { stdout, stderr } = await execFileAsync( + command.command, + command.args, + { + cwd: tmpdir(), + encoding: "utf8", + maxBuffer: 16 * 1024 * 1024, + timeout: 180_000, + windowsHide: true, + }, + ); + return { ok: true, stdout, stderr, exitCode: 0 }; + } catch (error) { + return { + ok: false, + stdout: typeof error?.stdout === "string" ? error.stdout : "", + stderr: typeof error?.stderr === "string" ? error.stderr : "", + exitCode: Number.isInteger(error?.code) ? error.code : null, + error: error instanceof Error ? error.message : String(error), + }; + } +} + +async function safelyExecute(executeCommand, command) { + try { + const result = await executeCommand(command); + if (!isRecord(result) || typeof result.ok !== "boolean") { + return { + ok: false, + stdout: "", + stderr: "", + exitCode: null, + error: "command runner returned an invalid result", + }; + } + return result; + } catch (error) { + return { + ok: false, + stdout: "", + stderr: "", + exitCode: null, + error: error instanceof Error ? error.message : String(error), + }; + } +} + +function reportedVersion(stdout) { + if (typeof stdout !== "string") { + return null; + } + return stdout.match(/\b\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?\b/)?.[0] ?? null; +} + +export async function runDeterministicScan( + targetRootArgument, + { executeCommand = executeScannerCommand } = {}, +) { + let targetRoot; + try { + targetRoot = canonicalTargetRoot(targetRootArgument); + } catch (error) { + return unavailableResult({ + targetRoot: + typeof targetRootArgument === "string" + ? resolve(targetRootArgument) + : null, + summary: `Target root validation failed: ${error instanceof Error ? error.message : String(error)}`, + }); + } + + const scannerCommands = buildScannerCommands(targetRoot); + const commands = []; + const versionRun = await safelyExecute( + executeCommand, + scannerCommands.version, + ); + if (!versionRun.ok) { + const detail = commandFailure(versionRun); + commands.push({ + command: shellDisplay(scannerCommands.version), + outcome: "failed", + detail, + }); + return unavailableResult({ + targetRoot, + summary: `Pinned scanner version check failed: ${detail}`, + commands, + }); + } + + const scannerVersion = reportedVersion(versionRun.stdout); + if (scannerVersion !== SCANNER_VERSION) { + const detail = `expected ${SCANNER_VERSION}, received ${scannerVersion ?? "no version"}`; + commands.push({ + command: shellDisplay(scannerCommands.version), + outcome: "failed", + detail, + }); + return unavailableResult({ + targetRoot, + scannerVersion, + summary: `Pinned scanner version check failed: ${detail}`, + commands, + }); + } + commands.push({ + command: shellDisplay(scannerCommands.version), + outcome: "passed", + }); + + const scanRun = await safelyExecute(executeCommand, scannerCommands.scan); + if (!scanRun.ok) { + const detail = commandFailure(scanRun); + commands.push({ + command: shellDisplay(scannerCommands.scan), + outcome: "failed", + detail, + }); + return unavailableResult({ + targetRoot, + scannerVersion, + summary: `Pinned scanner execution failed: ${detail}`, + commands, + }); + } + + let scanOutput; + try { + scanOutput = JSON.parse(scanRun.stdout); + } catch { + const detail = "scanner stdout was not valid JSON"; + commands.push({ + command: shellDisplay(scannerCommands.scan), + outcome: "failed", + detail, + }); + return unavailableResult({ + targetRoot, + scannerVersion, + summary: `Pinned scanner execution failed: ${detail}`, + commands, + }); + } + + const result = evaluateScannerOutput({ + targetRoot, + scannerVersion, + scanOutput, + signals: inspectRepositorySignals(targetRoot), + }); + if (result.status === "unavailable") { + commands.push({ + command: shellDisplay(scannerCommands.scan), + outcome: "failed", + detail: result.summary, + }); + } else { + commands.push({ + command: shellDisplay(scannerCommands.scan), + outcome: "passed", + }); + } + + return { ...result, commands }; +} + +function isDirectExecution() { + if (typeof process.argv[1] !== "string") { + return false; + } + + try { + return ( + pathToFileURL(realpathSync(resolve(process.argv[1]))).href === + pathToFileURL(realpathSync(fileURLToPath(import.meta.url))).href + ); + } catch { + return false; + } +} + +if (isDirectExecution()) { + const result = await runDeterministicScan(process.argv[2]); + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); + if (result.status === "unavailable") { + process.exitCode = 1; + } +} diff --git a/agent-compatibility/skills/check-agent-compatibility/scripts/synthesize-results.mjs b/agent-compatibility/skills/check-agent-compatibility/scripts/synthesize-results.mjs new file mode 100644 index 00000000..9519cc72 --- /dev/null +++ b/agent-compatibility/skills/check-agent-compatibility/scripts/synthesize-results.mjs @@ -0,0 +1,454 @@ +import { realpathSync } from "node:fs"; +import { readFile } from "node:fs/promises"; +import { isAbsolute, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const laneDefinitions = { + deterministic: { + scoreName: "Deterministic Compatibility Score", + statuses: ["complete", "unreliable", "unavailable"], + }, + startup: { + scoreName: "Startup Compatibility Score", + statuses: ["complete", "unavailable"], + }, + validation: { + scoreName: "Validation Loop Score", + statuses: ["complete", "unavailable"], + }, + docs: { + scoreName: "Docs Reliability Score", + statuses: ["complete", "unavailable"], + }, +}; + +const laneNames = Object.keys(laneDefinitions); +const workflowLaneNames = ["startup", "validation", "docs"]; +const statefulLaneNames = ["startup", "validation"]; +const commandOutcomes = ["passed", "failed", "blocked"]; + +export class ResultsValidationError extends Error { + constructor(message) { + super(message); + this.name = "ResultsValidationError"; + } +} + +function failValidation(message) { + throw new ResultsValidationError(message); +} + +function isRecord(value) { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + return false; + } + + const prototype = Object.getPrototypeOf(value); + return prototype === Object.prototype || prototype === null; +} + +function validateNonEmptyString(value, field) { + if (typeof value !== "string" || value.trim() === "") { + failValidation(`${field} must be a non-empty string.`); + } + return value; +} + +function canonicalPath(path) { + try { + return realpathSync(path); + } catch { + return resolve(path); + } +} + +function validateStringEvidence(value, field) { + if (!Array.isArray(value) || value.length === 0) { + failValidation(`${field} must be a non-empty array of strings.`); + } + + return value.map((item, index) => + validateNonEmptyString(item, `${field}[${index}]`), + ); +} + +function validateProblems(value, laneName) { + if (!Array.isArray(value)) { + failValidation(`${laneName}.problems must be an array.`); + } + + return value.map((problem, index) => { + const field = `${laneName}.problems[${index}]`; + if (!isRecord(problem)) { + failValidation(`${field} must be an object.`); + } + if (!Array.isArray(problem.evidence)) { + failValidation(`${field}.evidence must be an array.`); + } + + return { + ...problem, + title: validateNonEmptyString(problem.title, `${field}.title`), + evidence: problem.evidence.map((item, evidenceIndex) => + validateNonEmptyString(item, `${field}.evidence[${evidenceIndex}]`), + ), + remediation: validateNonEmptyString( + problem.remediation, + `${field}.remediation`, + ), + }; + }); +} + +function validateCommands(value, laneName) { + if (!Array.isArray(value)) { + failValidation(`${laneName}.commands must be an array.`); + } + + return value.map((command, index) => { + const field = `${laneName}.commands[${index}]`; + if (!isRecord(command)) { + failValidation(`${field} must be an object.`); + } + if (!commandOutcomes.includes(command.outcome)) { + failValidation( + `${field}.outcome must be one of: ${commandOutcomes.join(", ")}.`, + ); + } + + return { + ...command, + command: validateNonEmptyString(command.command, `${field}.command`), + }; + }); +} + +function validateLane(laneName, lane) { + const definition = laneDefinitions[laneName]; + + if (!isRecord(lane)) { + failValidation(`${laneName} must be an object.`); + } + + if (lane.scoreName !== definition.scoreName) { + failValidation(`${laneName}.scoreName must be "${definition.scoreName}".`); + } + + if (!definition.statuses.includes(lane.status)) { + failValidation( + `${laneName}.status must be one of: ${definition.statuses.join(", ")}.`, + ); + } + + if (lane.status === "unavailable") { + if (lane.score !== null) { + failValidation( + `${laneName}.score must be null when status is unavailable.`, + ); + } + } else if ( + typeof lane.score !== "number" || + !Number.isFinite(lane.score) || + lane.score < 0 || + lane.score > 100 + ) { + failValidation( + `${laneName}.score must be a finite number from 0 through 100.`, + ); + } + + if (laneName === "deterministic") { + if (typeof lane.classificationReliable !== "boolean") { + failValidation("deterministic.classificationReliable must be a boolean."); + } + + const expectedReliability = lane.status === "complete"; + if (lane.classificationReliable !== expectedReliability) { + failValidation( + "deterministic.classificationReliable must be " + + `${expectedReliability} when status is ${lane.status}.`, + ); + } + } + + const reportedTargetRoot = validateNonEmptyString( + lane.targetRoot, + `${laneName}.targetRoot`, + ); + if (!isAbsolute(reportedTargetRoot)) { + failValidation(`${laneName}.targetRoot must be an absolute path.`); + } + const targetRoot = canonicalPath(reportedTargetRoot); + + const normalized = { + ...lane, + targetRoot, + summary: validateNonEmptyString(lane.summary, `${laneName}.summary`), + evidence: validateStringEvidence(lane.evidence, `${laneName}.evidence`), + problems: validateProblems(lane.problems, laneName), + commands: validateCommands(lane.commands, laneName), + }; + + if (statefulLaneNames.includes(laneName)) { + const executionRoot = lane.executionRoot; + const hasExecutionRoot = + typeof executionRoot === "string" && executionRoot.trim() !== ""; + + if (lane.status !== "unavailable") { + if (!hasExecutionRoot || !isAbsolute(executionRoot)) { + failValidation(`${laneName}.executionRoot must be an absolute path.`); + } + const canonicalExecutionRoot = canonicalPath(executionRoot); + if (canonicalExecutionRoot === targetRoot) { + failValidation( + `${laneName}.executionRoot must differ from targetRoot.`, + ); + } + if (lane.isolation !== "isolated-copy") { + failValidation(`${laneName}.isolation must be "isolated-copy".`); + } + normalized.executionRoot = canonicalExecutionRoot; + } else { + if (hasExecutionRoot && !isAbsolute(executionRoot)) { + failValidation( + `${laneName}.executionRoot must be null or an absolute path.`, + ); + } + if (!hasExecutionRoot && executionRoot !== null) { + failValidation( + `${laneName}.executionRoot must be null or an absolute path.`, + ); + } + if (!["isolated-copy", "unavailable"].includes(lane.isolation)) { + failValidation( + `${laneName}.isolation must be "isolated-copy" or "unavailable".`, + ); + } + normalized.executionRoot = hasExecutionRoot + ? canonicalPath(executionRoot) + : null; + } + } + + return normalized; +} + +export function validateResults(input) { + if (!isRecord(input)) { + failValidation("Input must be an object containing the four result lanes."); + } + + for (const laneName of laneNames) { + if (!Object.hasOwn(input, laneName)) { + failValidation(`missing lane "${laneName}".`); + } + } + + for (const laneName of Object.keys(input)) { + if (!Object.hasOwn(laneDefinitions, laneName)) { + failValidation(`unexpected lane "${laneName}".`); + } + } + + const lanes = Object.fromEntries( + laneNames.map((laneName) => [ + laneName, + validateLane(laneName, input[laneName]), + ]), + ); + const targetRoot = lanes.deterministic.targetRoot; + for (const laneName of workflowLaneNames) { + if (lanes[laneName].targetRoot !== targetRoot) { + failValidation("all lanes must use the same targetRoot."); + } + } + if ( + lanes.startup.executionRoot !== null && + lanes.startup.executionRoot === lanes.validation.executionRoot + ) { + failValidation("startup and validation must use a separate executionRoot."); + } + + return lanes; +} + +function baseSynthesis(components) { + return { + schemaVersion: 1, + status: null, + agentCompatibilityScore: null, + workflowCompatibilityScore: null, + components, + unavailableWorkflowLanes: [], + reason: null, + }; +} + +export function synthesizeResults(input) { + const components = validateResults(input); + const synthesis = baseSynthesis(components); + const unavailableWorkflowLanes = workflowLaneNames.filter( + (laneName) => components[laneName].status === "unavailable", + ); + + if (unavailableWorkflowLanes.length > 0) { + return { + ...synthesis, + status: "unavailable", + unavailableWorkflowLanes, + reason: { + code: "WORKFLOW_LANES_UNAVAILABLE", + message: + "No aggregate was computed because workflow lanes are unavailable: " + + `${unavailableWorkflowLanes.join(", ")}.`, + }, + }; + } + + const workflowCompatibilityScore = Math.round( + workflowLaneNames.reduce( + (total, laneName) => total + components[laneName].score, + 0, + ) / workflowLaneNames.length, + ); + + if (components.deterministic.status === "unreliable") { + return { + ...synthesis, + status: "degraded", + workflowCompatibilityScore, + reason: { + code: "DETERMINISTIC_UNRELIABLE", + message: + "The deterministic classification is unreliable; only workflow evidence was aggregated.", + }, + }; + } + + if (components.deterministic.status === "unavailable") { + return { + ...synthesis, + status: "degraded", + workflowCompatibilityScore, + reason: { + code: "DETERMINISTIC_UNAVAILABLE", + message: + "The deterministic scan is unavailable; only workflow evidence was aggregated.", + }, + }; + } + + return { + ...synthesis, + status: "complete", + agentCompatibilityScore: Math.round( + components.deterministic.score * 0.7 + workflowCompatibilityScore * 0.3, + ), + workflowCompatibilityScore, + }; +} + +class CliInputError extends Error { + constructor(code, message) { + super(message); + this.name = "CliInputError"; + this.code = code; + } +} + +async function readStdin() { + let input = ""; + process.stdin.setEncoding("utf8"); + + for await (const chunk of process.stdin) { + input += chunk; + } + + return input; +} + +async function readCliInput(arguments_) { + if (arguments_.length > 1) { + throw new CliInputError( + "INVALID_USAGE", + "Pass one JSON file path, '-' for stdin, or pipe JSON with no argument.", + ); + } + + const inputPath = arguments_[0]; + if (inputPath === undefined || inputPath === "-") { + if (inputPath === undefined && process.stdin.isTTY) { + throw new CliInputError( + "INVALID_USAGE", + "Pass one JSON file path, '-' for stdin, or pipe JSON with no argument.", + ); + } + + return readStdin(); + } + + try { + return await readFile(inputPath, "utf8"); + } catch { + throw new CliInputError( + "INPUT_READ_ERROR", + `Unable to read input file: ${inputPath}`, + ); + } +} + +function parseJson(input) { + try { + return JSON.parse(input); + } catch { + throw new CliInputError("INVALID_JSON", "Input is not valid JSON."); + } +} + +function writeJson(stream, value) { + stream.write(`${JSON.stringify(value, null, 2)}\n`); +} + +async function runCli() { + try { + const input = await readCliInput(process.argv.slice(2)); + writeJson(process.stdout, synthesizeResults(parseJson(input))); + } catch (error) { + const cliError = + error instanceof ResultsValidationError + ? new CliInputError("INVALID_RESULTS", error.message) + : error; + + const code = + cliError instanceof CliInputError ? cliError.code : "INTERNAL_ERROR"; + const message = + cliError instanceof CliInputError + ? cliError.message + : "Result synthesis failed unexpectedly."; + + writeJson(process.stderr, { + status: "invalid", + error: { code, message }, + }); + process.exitCode = 1; + } +} + +function isMainModule() { + if (process.argv[1] === undefined) { + return false; + } + + try { + return ( + realpathSync(resolve(process.argv[1])) === + realpathSync(fileURLToPath(import.meta.url)) + ); + } catch { + return false; + } +} + +if (isMainModule()) { + await runCli(); +} diff --git a/agent-compatibility/test/deterministic-scan.test.mjs b/agent-compatibility/test/deterministic-scan.test.mjs new file mode 100644 index 00000000..7d13da0d --- /dev/null +++ b/agent-compatibility/test/deterministic-scan.test.mjs @@ -0,0 +1,210 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, realpathSync, rmSync, symlinkSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, resolve } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +import { + assessClassification, + buildScannerCommands, + evaluateScannerOutput, + inspectRepositorySignals, + runDeterministicScan, +} from "../skills/check-agent-compatibility/scripts/run-deterministic-scan.mjs"; + +const testDirectory = dirname(fileURLToPath(import.meta.url)); +const fixturesRoot = resolve(testDirectory, "fixtures/deterministic-scan"); +const cloudflareWorkerRoot = realpathSync( + resolve(fixturesRoot, "cloudflare-worker"), +); +const genuineCliRoot = realpathSync(resolve(fixturesRoot, "genuine-cli")); +const skillRoot = resolve(testDirectory, "../skills/check-agent-compatibility"); + +function scannerOutput(targetRoot, kind) { + return { + scannedPath: targetRoot, + overallScore: 81, + maturity: "Solid", + classification: { + kind, + reasons: ["fixture classification"], + }, + recommendations: [ + { + title: "Add a validation command", + remediation: "Expose validation from the repository root.", + evidence: ["no validation command found"], + }, + ], + }; +} + +test("pins both scanner commands to agent-compatibility 0.1.7", () => { + const commands = buildScannerCommands(genuineCliRoot); + + assert.deepEqual(commands.version, { + command: "npx", + args: ["-y", "agent-compatibility@0.1.7", "--version"], + }); + assert.deepEqual(commands.scan, { + command: "npx", + args: ["-y", "agent-compatibility@0.1.7", "--json", genuineCliRoot], + }); +}); + +test("rejects a CLI classification for a Cloudflare Worker without a CLI entrypoint", () => { + const signals = inspectRepositorySignals(cloudflareWorkerRoot); + const assessment = assessClassification("cli", signals); + + assert.equal(signals.isCloudflareWorker, true); + assert.equal(signals.hasCliEntrypoint, false); + assert.equal(assessment.reliable, false); + assert.match(assessment.reason, /Cloudflare Worker/i); + assert.ok(assessment.evidence.includes("wrangler.toml")); +}); + +test("preserves a CLI classification backed by package bin metadata", () => { + const signals = inspectRepositorySignals(genuineCliRoot); + const assessment = assessClassification("cli", signals); + + assert.equal(signals.hasCliEntrypoint, true); + assert.ok(signals.cli.includes("package.json#bin")); + assert.equal(assessment.reliable, true); + assert.equal(assessment.reason, null); +}); + +test("marks a mismatched scannedPath unreliable while retaining scanner evidence", () => { + const signals = inspectRepositorySignals(genuineCliRoot); + const result = evaluateScannerOutput({ + targetRoot: genuineCliRoot, + scannerVersion: "0.1.7", + scanOutput: scannerOutput(cloudflareWorkerRoot, "cli"), + signals, + }); + + assert.equal(result.status, "unreliable"); + assert.equal(result.classificationReliable, false); + assert.equal(result.score, 81); + assert.equal(result.problems[0].title, "Add a validation command"); + assert.match(result.summary, /scannedPath/i); +}); + +test("returns a complete result for a matching, well-supported CLI scan", () => { + const signals = inspectRepositorySignals(genuineCliRoot); + const result = evaluateScannerOutput({ + targetRoot: genuineCliRoot, + scannerVersion: "0.1.7", + scanOutput: scannerOutput(genuineCliRoot, "cli"), + signals, + }); + + assert.equal(result.status, "complete"); + assert.equal(result.classification, "cli"); + assert.equal(result.classificationReliable, true); + assert.equal(result.scannedPath, genuineCliRoot); +}); + +test("runs the version check before the scan and returns structured JSON", async () => { + const invocations = []; + const executeCommand = async (command) => { + invocations.push(command); + if (command.args.at(-1) === "--version") { + return { ok: true, stdout: "0.1.7\n", stderr: "", exitCode: 0 }; + } + + return { + ok: true, + stdout: JSON.stringify(scannerOutput(genuineCliRoot, "cli")), + stderr: "", + exitCode: 0, + }; + }; + + const result = await runDeterministicScan(genuineCliRoot, { + executeCommand, + }); + + assert.deepEqual( + invocations.map((command) => command.args), + [ + ["-y", "agent-compatibility@0.1.7", "--version"], + ["-y", "agent-compatibility@0.1.7", "--json", genuineCliRoot], + ], + ); + assert.equal(result.status, "complete"); + assert.deepEqual( + result.commands.map((command) => command.outcome), + ["passed", "passed"], + ); + assert.doesNotThrow(() => JSON.stringify(result)); +}); + +test("normalizes scanner execution failures instead of throwing", async () => { + const executeCommand = async (command) => { + if (command.args.at(-1) === "--version") { + return { ok: true, stdout: "0.1.7\n", stderr: "", exitCode: 0 }; + } + + return { + ok: false, + stdout: "", + stderr: "npm registry unavailable", + exitCode: 1, + error: "npm registry unavailable", + }; + }; + + const result = await runDeterministicScan(genuineCliRoot, { + executeCommand, + }); + + assert.deepEqual(result, { + status: "unavailable", + scoreName: "Deterministic Compatibility Score", + score: null, + scannerVersion: "0.1.7", + targetRoot: genuineCliRoot, + scannedPath: null, + classification: null, + classificationReliable: false, + classificationEvidence: [], + summary: "Pinned scanner execution failed: npm registry unavailable", + evidence: ["Pinned scanner execution failed: npm registry unavailable"], + problems: [], + commands: [ + { + command: "npx -y agent-compatibility@0.1.7 --version", + outcome: "passed", + }, + { + command: `npx -y agent-compatibility@0.1.7 --json ${genuineCliRoot}`, + outcome: "failed", + detail: "npm registry unavailable", + }, + ], + }); +}); + +test("executes through the documented symlinked plugin install", (t) => { + const temporaryDirectory = mkdtempSync( + resolve(tmpdir(), "deterministic-scan-symlink-"), + ); + const linkedSkillRoot = resolve( + temporaryDirectory, + "check-agent-compatibility", + ); + t.after(() => rmSync(temporaryDirectory, { recursive: true, force: true })); + symlinkSync(skillRoot, linkedSkillRoot, "dir"); + + const run = spawnSync( + process.execPath, + [resolve(linkedSkillRoot, "scripts/run-deterministic-scan.mjs")], + { encoding: "utf8" }, + ); + + assert.equal(run.status, 1); + assert.equal(run.stderr, ""); + assert.equal(JSON.parse(run.stdout).status, "unavailable"); +}); diff --git a/agent-compatibility/test/fixtures/deterministic-scan/cloudflare-worker/package.json b/agent-compatibility/test/fixtures/deterministic-scan/cloudflare-worker/package.json new file mode 100644 index 00000000..8382d2a7 --- /dev/null +++ b/agent-compatibility/test/fixtures/deterministic-scan/cloudflare-worker/package.json @@ -0,0 +1,12 @@ +{ + "name": "cloudflare-worker-fixture", + "private": true, + "type": "module", + "scripts": { + "dev": "wrangler dev", + "deploy": "wrangler deploy" + }, + "devDependencies": { + "wrangler": "^4.0.0" + } +} diff --git a/agent-compatibility/test/fixtures/deterministic-scan/cloudflare-worker/src/worker.ts b/agent-compatibility/test/fixtures/deterministic-scan/cloudflare-worker/src/worker.ts new file mode 100644 index 00000000..a5c59f13 --- /dev/null +++ b/agent-compatibility/test/fixtures/deterministic-scan/cloudflare-worker/src/worker.ts @@ -0,0 +1,5 @@ +export default { + fetch() { + return new Response("ok"); + }, +}; diff --git a/agent-compatibility/test/fixtures/deterministic-scan/cloudflare-worker/wrangler.toml b/agent-compatibility/test/fixtures/deterministic-scan/cloudflare-worker/wrangler.toml new file mode 100644 index 00000000..3c7c0453 --- /dev/null +++ b/agent-compatibility/test/fixtures/deterministic-scan/cloudflare-worker/wrangler.toml @@ -0,0 +1,3 @@ +name = "cloudflare-worker-fixture" +main = "src/worker.ts" +compatibility_date = "2026-08-19" diff --git a/agent-compatibility/test/fixtures/deterministic-scan/genuine-cli/bin/genuine-cli.mjs b/agent-compatibility/test/fixtures/deterministic-scan/genuine-cli/bin/genuine-cli.mjs new file mode 100644 index 00000000..dabd075c --- /dev/null +++ b/agent-compatibility/test/fixtures/deterministic-scan/genuine-cli/bin/genuine-cli.mjs @@ -0,0 +1,3 @@ +#!/usr/bin/env node + +process.stdout.write("genuine CLI\n"); diff --git a/agent-compatibility/test/fixtures/deterministic-scan/genuine-cli/package.json b/agent-compatibility/test/fixtures/deterministic-scan/genuine-cli/package.json new file mode 100644 index 00000000..2a846cb6 --- /dev/null +++ b/agent-compatibility/test/fixtures/deterministic-scan/genuine-cli/package.json @@ -0,0 +1,7 @@ +{ + "name": "genuine-cli-fixture", + "type": "module", + "bin": { + "genuine-cli": "./bin/genuine-cli.mjs" + } +} diff --git a/agent-compatibility/test/plugin-contract.test.mjs b/agent-compatibility/test/plugin-contract.test.mjs new file mode 100644 index 00000000..26f1e2ae --- /dev/null +++ b/agent-compatibility/test/plugin-contract.test.mjs @@ -0,0 +1,227 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; + +const testDirectory = dirname(fileURLToPath(import.meta.url)); +const repositoryRoot = resolve(testDirectory, "../.."); +const pluginRoot = resolve(repositoryRoot, "agent-compatibility"); + +function readPluginFile(relativePath) { + return readFileSync(resolve(pluginRoot, relativePath), "utf8"); +} + +function readRepositoryFile(relativePath) { + return readFileSync(resolve(repositoryRoot, relativePath), "utf8"); +} + +function frontmatterValue(content, field) { + const frontmatter = content.match(/^---\n([\s\S]*?)\n---/); + assert.ok(frontmatter, "expected YAML frontmatter"); + + const match = frontmatter[1].match(new RegExp(`^${field}:\\s*(.+)$`, "m")); + return match?.[1].trim(); +} + +const agentPaths = [ + "agents/compatibility-scan-review.md", + "agents/startup-review.md", + "agents/validation-review.md", + "agents/docs-reliability-review.md", +]; + +test("agents that install, start, or validate are writable while docs review stays read-only", () => { + for (const path of agentPaths.slice(0, 3)) { + const content = readPluginFile(path); + assert.equal(frontmatterValue(content, "readonly"), "false", path); + } + + const docsReview = readPluginFile("agents/docs-reliability-review.md"); + assert.equal(frontmatterValue(docsReview, "readonly"), "true"); +}); + +test("the published scanner is pinned everywhere it can be executed", () => { + const files = [ + "README.md", + "agents/compatibility-scan-review.md", + "skills/check-agent-compatibility/SKILL.md", + ]; + + for (const path of files) { + const content = readPluginFile(path); + assert.doesNotMatch(content, /agent-compatibility@latest/, path); + } + + assert.match(readPluginFile("README.md"), /agent-compatibility@0\.1\.7/); + assert.match( + readPluginFile("agents/compatibility-scan-review.md"), + /agent-compatibility@0\.1\.7/, + ); +}); + +test("the orchestrator uses executable helpers for scanning and score synthesis", () => { + const skill = readPluginFile("skills/check-agent-compatibility/SKILL.md"); + const scanReview = readPluginFile("agents/compatibility-scan-review.md"); + + assert.match(skill, /scripts\/run-deterministic-scan\.mjs/); + assert.match(skill, /scripts\/synthesize-results\.mjs/); + assert.match(scanReview, /run-deterministic-scan\.mjs/); + assert.match(skill, /use the synthesizer output exactly/i); +}); + +test("the orchestrator has an explicit degraded result when the deterministic scan is unusable", () => { + const skill = readPluginFile("skills/check-agent-compatibility/SKILL.md"); + + assert.match(skill, /Agent Compatibility Score: unavailable/); + assert.match(skill, /Workflow Compatibility Score: N\/100/); + assert.match(skill, /must not compute the 70\/30 blend/i); +}); + +test("the orchestrator refuses a blended score when a workflow lane is unavailable", () => { + const skill = readPluginFile("skills/check-agent-compatibility/SKILL.md"); + + assert.match( + skill, + /workflow lane is unavailable[^\n]+Agent Compatibility Score: unavailable/i, + ); + assert.match(skill, /do not impute its score/i); +}); + +test("every delegated review receives the target, evidence contract, and bounded execution context", () => { + const skill = readPluginFile("skills/check-agent-compatibility/SKILL.md"); + + for (const phrase of [ + "Target root", + "Deterministic scan result", + "Time budget", + "Allowed mutations", + "Required evidence", + ]) { + assert.match(skill, new RegExp(phrase, "i"), phrase); + } + + assert.match( + skill, + /launch .*startup-review.*validation-review.*docs-reliability-review.*parallel/is, + ); +}); + +test("specialists return a machine-checkable result with evidence and command outcomes", () => { + for (const path of agentPaths) { + const content = readPluginFile(path); + + for (const key of [ + '"status"', + '"scoreName"', + '"score"', + '"summary"', + '"targetRoot"', + '"evidence"', + '"problems"', + ]) { + assert.match(content, new RegExp(key), `${path}: ${key}`); + } + } + + for (const path of agentPaths.slice(0, 3)) { + assert.match(readPluginFile(path), /"commands"/, path); + } +}); + +test("stateful specialists report isolated execution provenance", () => { + for (const path of [ + "agents/startup-review.md", + "agents/validation-review.md", + ]) { + const content = readPluginFile(path); + assert.match(content, /"executionRoot"/, path); + assert.match(content, /"isolation": "isolated-copy"/, path); + } +}); + +test("specialist output examples are valid JSON", () => { + for (const path of agentPaths) { + const content = readPluginFile(path); + const example = content.match(/```json\n([\s\S]*?)\n```/); + assert.ok(example, `${path}: missing JSON example`); + + const parsed = JSON.parse(example[1]); + assert.equal(typeof parsed.status, "string", path); + assert.doesNotMatch( + parsed.status, + /\|/, + `${path}: status must be a concrete value`, + ); + assert.equal(typeof parsed.scoreName, "string", path); + assert.ok(Array.isArray(parsed.problems), path); + assert.ok(Array.isArray(parsed.commands), path); + for (const command of parsed.commands) { + assert.doesNotMatch( + command.outcome, + /\|/, + `${path}: command outcome must be a concrete value`, + ); + } + } +}); + +test("the scan specialist rejects an obviously wrong repository classification", () => { + const scanReview = readPluginFile("agents/compatibility-scan-review.md"); + + assert.match(scanReview, /classification/i); + assert.match(scanReview, /obvious repository signals/i); + assert.match(scanReview, /"unreliable"/); +}); + +test("stateful specialists define isolation and side-effect boundaries", () => { + for (const path of [ + "agents/startup-review.md", + "agents/validation-review.md", + ]) { + const content = readPluginFile(path); + assert.match(content, /isolated/i, path); + assert.match(content, /do not (run|perform).*deploy/i, path); + assert.match(content, /paid|costs money/i, path); + assert.match(content, /tracked files/i, path); + } +}); + +test("CI runs contract tests whenever the plugin behavior can change", () => { + const workflow = readRepositoryFile(".github/workflows/validate-plugins.yml"); + + for (const watchedPath of [ + "agent-compatibility/agents/**", + "agent-compatibility/skills/**", + "agent-compatibility/test/**", + "agent-compatibility/README.md", + "agent-compatibility/CHANGELOG.md", + ".github/workflows/validate-plugins.yml", + ]) { + assert.match( + workflow, + new RegExp(watchedPath.replaceAll("*", "\\*")), + watchedPath, + ); + } + + assert.match( + workflow, + /node --test agent-compatibility\/test\/\*\.test\.mjs/, + ); +}); + +test("release documentation matches the manifest version", () => { + const manifest = JSON.parse(readPluginFile(".cursor-plugin/plugin.json")); + const readme = readPluginFile("README.md"); + const changelog = readPluginFile("CHANGELOG.md"); + + assert.match( + readme, + new RegExp(`Plugin version ${manifest.version.replaceAll(".", "\\.")}`), + ); + assert.match( + changelog, + new RegExp(`^## ${manifest.version.replaceAll(".", "\\.")} - `, "m"), + ); +}); diff --git a/agent-compatibility/test/synthesize-results.test.mjs b/agent-compatibility/test/synthesize-results.test.mjs new file mode 100644 index 00000000..d10954df --- /dev/null +++ b/agent-compatibility/test/synthesize-results.test.mjs @@ -0,0 +1,401 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; + +import { + synthesizeResults, + validateResults, +} from "../skills/check-agent-compatibility/scripts/synthesize-results.mjs"; + +const testDirectory = dirname(fileURLToPath(import.meta.url)); +const scriptPath = resolve( + testDirectory, + "../skills/check-agent-compatibility/scripts/synthesize-results.mjs", +); + +const scoreNames = { + deterministic: "Deterministic Compatibility Score", + startup: "Startup Compatibility Score", + validation: "Validation Loop Score", + docs: "Docs Reliability Score", +}; +const targetRoot = "/workspace/target-repository"; + +function completeResults(overrides = {}) { + const results = { + deterministic: { + status: "complete", + scoreName: scoreNames.deterministic, + score: 81, + classificationReliable: true, + targetRoot, + summary: "Pinned scan completed.", + evidence: ["scanner 0.1.7 scanned the canonical target"], + problems: [], + commands: [{ command: "pinned scan", outcome: "passed" }], + }, + startup: { + status: "complete", + scoreName: scoreNames.startup, + score: 90, + targetRoot, + executionRoot: "/tmp/startup-isolated-copy", + isolation: "isolated-copy", + summary: "Startup completed in isolation.", + evidence: ["startup command passed"], + problems: [], + commands: [{ command: "startup", outcome: "passed" }], + }, + validation: { + status: "complete", + scoreName: scoreNames.validation, + score: 80, + targetRoot, + executionRoot: "/tmp/validation-isolated-copy", + isolation: "isolated-copy", + summary: "Validation completed in isolation.", + evidence: ["validation command passed"], + problems: [], + commands: [{ command: "validate", outcome: "passed" }], + }, + docs: { + status: "complete", + scoreName: scoreNames.docs, + score: 70, + targetRoot, + summary: "Documentation was traced to repository interfaces.", + evidence: ["README.md:10 maps to package.json#scripts.test"], + problems: [], + commands: [], + }, + }; + + for (const [lane, values] of Object.entries(overrides)) { + results[lane] = { ...results[lane], ...values }; + } + + return results; +} + +test("computes the workflow average and eligible 70/30 aggregate", () => { + const input = completeResults(); + const before = structuredClone(input); + + assert.deepEqual(synthesizeResults(input), { + schemaVersion: 1, + status: "complete", + agentCompatibilityScore: 81, + workflowCompatibilityScore: 80, + components: before, + unavailableWorkflowLanes: [], + reason: null, + }); + assert.deepEqual( + input, + before, + "synthesis must not mutate specialist results", + ); +}); + +test("rounds the workflow score before applying the 70/30 blend", () => { + const result = synthesizeResults( + completeResults({ + deterministic: { score: 0 }, + startup: { score: 0 }, + validation: { score: 14 }, + docs: { score: 0 }, + }), + ); + + assert.equal(result.workflowCompatibilityScore, 5); + assert.equal(result.agentCompatibilityScore, 2); +}); + +test("returns workflow-only evidence when deterministic classification is unreliable", () => { + const result = synthesizeResults( + completeResults({ + deterministic: { + status: "unreliable", + classificationReliable: false, + }, + }), + ); + + assert.equal(result.status, "degraded"); + assert.equal(result.agentCompatibilityScore, null); + assert.equal(result.workflowCompatibilityScore, 80); + assert.deepEqual(result.reason, { + code: "DETERMINISTIC_UNRELIABLE", + message: + "The deterministic classification is unreliable; only workflow evidence was aggregated.", + }); +}); + +test("returns workflow-only evidence when the deterministic scan is unavailable", () => { + const result = synthesizeResults( + completeResults({ + deterministic: { + status: "unavailable", + score: null, + classificationReliable: false, + }, + }), + ); + + assert.equal(result.status, "degraded"); + assert.equal(result.agentCompatibilityScore, null); + assert.equal(result.workflowCompatibilityScore, 80); + assert.equal(result.reason.code, "DETERMINISTIC_UNAVAILABLE"); +}); + +test("returns no aggregate when any workflow lane is unavailable", () => { + const result = synthesizeResults( + completeResults({ + startup: { status: "unavailable", score: null }, + docs: { status: "unavailable", score: null }, + }), + ); + + assert.equal(result.status, "unavailable"); + assert.equal(result.agentCompatibilityScore, null); + assert.equal(result.workflowCompatibilityScore, null); + assert.deepEqual(result.unavailableWorkflowLanes, ["startup", "docs"]); + assert.deepEqual(result.reason, { + code: "WORKFLOW_LANES_UNAVAILABLE", + message: + "No aggregate was computed because workflow lanes are unavailable: startup, docs.", + }); +}); + +test("validates exact lane names, statuses, and score bounds", () => { + assert.throws( + () => validateResults({ ...completeResults(), surprise: {} }), + /unexpected lane "surprise"/, + ); + assert.throws( + () => + validateResults( + completeResults({ startup: { scoreName: "Startup-ish Score" } }), + ), + /startup\.scoreName/, + ); + assert.throws( + () => + validateResults(completeResults({ validation: { status: "failed" } })), + /validation\.status/, + ); + assert.throws( + () => validateResults(completeResults({ docs: { score: 101 } })), + /docs\.score/, + ); + assert.throws( + () => validateResults(completeResults({ startup: { score: "90" } })), + /startup\.score/, + ); +}); + +test("rejects inconsistent unavailable and deterministic states", () => { + assert.throws( + () => + validateResults( + completeResults({ startup: { status: "unavailable", score: 0 } }), + ), + /startup\.score must be null/, + ); + assert.throws( + () => + validateResults( + completeResults({ + deterministic: { + status: "unreliable", + classificationReliable: true, + }, + }), + ), + /deterministic\.classificationReliable must be false/, + ); + assert.throws( + () => + validateResults( + completeResults({ + deterministic: { + status: "complete", + classificationReliable: false, + }, + }), + ), + /deterministic\.classificationReliable must be true/, + ); +}); + +test("rejects evidence-free, wrong-target, and non-isolated results", () => { + const bare = { + deterministic: { + status: "complete", + scoreName: scoreNames.deterministic, + score: 81, + classificationReliable: true, + }, + startup: { + status: "complete", + scoreName: scoreNames.startup, + score: 90, + }, + validation: { + status: "complete", + scoreName: scoreNames.validation, + score: 80, + }, + docs: { + status: "complete", + scoreName: scoreNames.docs, + score: 70, + }, + }; + + assert.throws(() => validateResults(bare), /targetRoot|summary|evidence/); + assert.throws( + () => + validateResults( + completeResults({ docs: { targetRoot: "/workspace/wrong-target" } }), + ), + /same targetRoot/, + ); + assert.throws( + () => + validateResults( + completeResults({ startup: { executionRoot: targetRoot } }), + ), + /executionRoot/, + ); + assert.throws( + () => + validateResults( + completeResults({ startup: { executionRoot: `${targetRoot}/.` } }), + ), + /executionRoot/, + ); + assert.throws( + () => + validateResults( + completeResults({ + startup: { executionRoot: "/tmp/shared-isolated-copy" }, + validation: { executionRoot: "/tmp/shared-isolated-copy/." }, + }), + ), + /separate executionRoot/, + ); + assert.throws( + () => validateResults(completeResults({ validation: { evidence: [] } })), + /evidence/, + ); +}); + +test("retains validated target, evidence, and isolation provenance", () => { + const result = synthesizeResults(completeResults()); + + assert.equal(result.components.docs.targetRoot, targetRoot); + assert.deepEqual(result.components.docs.evidence, [ + "README.md:10 maps to package.json#scripts.test", + ]); + assert.equal( + result.components.startup.executionRoot, + "/tmp/startup-isolated-copy", + ); + assert.equal(result.components.startup.isolation, "isolated-copy"); +}); + +test("CLI reads valid JSON from stdin and emits structured JSON", () => { + const run = spawnSync(process.execPath, [scriptPath], { + encoding: "utf8", + input: JSON.stringify(completeResults()), + }); + + assert.equal(run.status, 0, run.stderr); + assert.equal(run.stderr, ""); + assert.deepEqual( + JSON.parse(run.stdout), + synthesizeResults(completeResults()), + ); +}); + +test("CLI reads one JSON file path", (t) => { + const temporaryDirectory = mkdtempSync( + resolve(tmpdir(), "synthesize-results-"), + ); + const inputPath = resolve(temporaryDirectory, "results.json"); + t.after(() => rmSync(temporaryDirectory, { recursive: true, force: true })); + writeFileSync(inputPath, JSON.stringify(completeResults())); + + const run = spawnSync(process.execPath, [scriptPath, inputPath], { + encoding: "utf8", + }); + + assert.equal(run.status, 0, run.stderr); + assert.deepEqual( + JSON.parse(run.stdout), + synthesizeResults(completeResults()), + ); +}); + +test("CLI fails closed with structured errors for malformed input", () => { + const run = spawnSync(process.execPath, [scriptPath, "-"], { + encoding: "utf8", + input: "{not json", + }); + + assert.equal(run.status, 1); + assert.equal(run.stdout, ""); + assert.deepEqual(JSON.parse(run.stderr), { + status: "invalid", + error: { + code: "INVALID_JSON", + message: "Input is not valid JSON.", + }, + }); +}); + +test("CLI fails closed with structured errors for invalid result data", () => { + const run = spawnSync(process.execPath, [scriptPath], { + encoding: "utf8", + input: JSON.stringify( + completeResults({ validation: { status: "failed" } }), + ), + }); + + assert.equal(run.status, 1); + assert.equal(run.stdout, ""); + const error = JSON.parse(run.stderr); + assert.equal(error.status, "invalid"); + assert.equal(error.error.code, "INVALID_RESULTS"); + assert.match(error.error.message, /validation\.status/); +}); + +test("CLI executes through the documented symlinked plugin install", (t) => { + const temporaryDirectory = mkdtempSync( + resolve(tmpdir(), "synthesize-results-symlink-"), + ); + const skillRoot = resolve(scriptPath, "../.."); + const linkedSkillRoot = resolve( + temporaryDirectory, + "check-agent-compatibility", + ); + t.after(() => rmSync(temporaryDirectory, { recursive: true, force: true })); + symlinkSync(skillRoot, linkedSkillRoot, "dir"); + + const run = spawnSync( + process.execPath, + [resolve(linkedSkillRoot, "scripts/synthesize-results.mjs")], + { encoding: "utf8", input: JSON.stringify(completeResults()) }, + ); + + assert.equal(run.status, 0, run.stderr); + assert.deepEqual( + JSON.parse(run.stdout), + synthesizeResults(completeResults()), + ); +}); From 0a3637243e6a354497a1df37949c7c4f765175ff Mon Sep 17 00:00:00 2001 From: Gareth Paul Jones Date: Wed, 19 Aug 2026 20:10:09 +0000 Subject: [PATCH 2/3] Support Windows scanner execution --- .github/workflows/validate-plugins.yml | 5 +- agent-compatibility/CHANGELOG.md | 1 + agent-compatibility/README.md | 2 +- .../scripts/run-deterministic-scan.mjs | 83 +++++++++++++++++-- .../test/deterministic-scan.test.mjs | 30 ++++++- .../test/plugin-contract.test.mjs | 1 + .../test/synthesize-results.test.mjs | 6 +- 7 files changed, 117 insertions(+), 11 deletions(-) diff --git a/.github/workflows/validate-plugins.yml b/.github/workflows/validate-plugins.yml index fed18b87..51773acd 100644 --- a/.github/workflows/validate-plugins.yml +++ b/.github/workflows/validate-plugins.yml @@ -15,7 +15,10 @@ on: jobs: validate: - runs-on: ubuntu-latest + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 diff --git a/agent-compatibility/CHANGELOG.md b/agent-compatibility/CHANGELOG.md index aa22f7e1..2d23fd41 100644 --- a/agent-compatibility/CHANGELOG.md +++ b/agent-compatibility/CHANGELOG.md @@ -8,6 +8,7 @@ All notable changes to this plugin are documented here. - Pinned the deterministic scanner to `agent-compatibility@0.1.7`. - Added an executable scanner guard that validates the pinned version, scanned path, output shape, and Cloudflare Worker classification signals. +- Made pinned scanner execution shell-free on Windows by resolving npm's `npx-cli.js` entrypoint through Node. - Added a fail-closed result synthesizer as the sole owner of score validation, degraded states, and 70/30 arithmetic. - Made the synthesizer reject missing evidence, mismatched targets, and stateful results without isolated execution provenance. - Made startup and validation writable only inside isolated copies, with deploy, migration, credential, and paid-test boundaries. diff --git a/agent-compatibility/README.md b/agent-compatibility/README.md index 428cc6c1..4eb890cb 100644 --- a/agent-compatibility/README.md +++ b/agent-compatibility/README.md @@ -61,7 +61,7 @@ Top fixes Plugin version 1.1.0 pins scanner version 0.1.7. Updating the scanner requires a plugin version change and contract-test update. -The plugin invokes the scanner through `skills/check-agent-compatibility/scripts/run-deterministic-scan.mjs`; direct commands below are for manual inspection. It computes final scores with `scripts/synthesize-results.mjs`, which fails closed on malformed or inconsistent lane results. +The plugin invokes the scanner through `skills/check-agent-compatibility/scripts/run-deterministic-scan.mjs`; direct commands below are for manual inspection. On Windows, the helper runs npm's JavaScript entrypoint through `node.exe` instead of invoking the `npx.cmd` shell shim. It computes final scores with `scripts/synthesize-results.mjs`, which fails closed on malformed or inconsistent lane results. Default scan: diff --git a/agent-compatibility/skills/check-agent-compatibility/scripts/run-deterministic-scan.mjs b/agent-compatibility/skills/check-agent-compatibility/scripts/run-deterministic-scan.mjs index 051fa211..80a2ec78 100644 --- a/agent-compatibility/skills/check-agent-compatibility/scripts/run-deterministic-scan.mjs +++ b/agent-compatibility/skills/check-agent-compatibility/scripts/run-deterministic-scan.mjs @@ -9,7 +9,14 @@ import { statSync, } from "node:fs"; import { tmpdir } from "node:os"; -import { isAbsolute, join, relative, resolve, sep } from "node:path"; +import { + isAbsolute, + join, + relative, + resolve, + sep, + win32 as windowsPath, +} from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { promisify } from "node:util"; @@ -47,6 +54,45 @@ const cliFrameworkPackages = [ "yargs", ]; +export function resolveNpxRunner({ + platform = process.platform, + nodeExecutable = process.execPath, + npmExecPath = process.env.npm_execpath ?? null, + pathExists = existsSync, +} = {}) { + if (platform !== "win32") { + return { command: "npx", prefixArgs: [] }; + } + + const candidates = []; + if (typeof npmExecPath === "string" && npmExecPath.trim() !== "") { + candidates.push( + windowsPath.join(windowsPath.dirname(npmExecPath), "npx-cli.js"), + ); + } + candidates.push( + windowsPath.join( + windowsPath.dirname(nodeExecutable), + "node_modules", + "npm", + "bin", + "npx-cli.js", + ), + ); + + const npxCli = [...new Set(candidates)].find((candidate) => { + try { + return pathExists(candidate); + } catch { + return false; + } + }); + + return npxCli === undefined + ? null + : { command: nodeExecutable, prefixArgs: [npxCli] }; +} + function isRecord(value) { return value !== null && typeof value === "object" && !Array.isArray(value); } @@ -291,15 +337,30 @@ function wranglerConfigSignals(targetRoot) { return { configs, entrypoints }; } -export function buildScannerCommands(targetRoot) { +export function buildScannerCommands( + targetRoot, + { runner = resolveNpxRunner() } = {}, +) { + if (runner === null) { + throw new Error( + "Unable to locate npm's npx-cli.js for safe Windows execution.", + ); + } + return { version: { - command: "npx", - args: ["-y", SCANNER_SPECIFIER, "--version"], + command: runner.command, + args: [...runner.prefixArgs, "-y", SCANNER_SPECIFIER, "--version"], }, scan: { - command: "npx", - args: ["-y", SCANNER_SPECIFIER, "--json", targetRoot], + command: runner.command, + args: [ + ...runner.prefixArgs, + "-y", + SCANNER_SPECIFIER, + "--json", + targetRoot, + ], }, }; } @@ -640,7 +701,15 @@ export async function runDeterministicScan( }); } - const scannerCommands = buildScannerCommands(targetRoot); + let scannerCommands; + try { + scannerCommands = buildScannerCommands(targetRoot); + } catch (error) { + return unavailableResult({ + targetRoot, + summary: error instanceof Error ? error.message : String(error), + }); + } const commands = []; const versionRun = await safelyExecute( executeCommand, diff --git a/agent-compatibility/test/deterministic-scan.test.mjs b/agent-compatibility/test/deterministic-scan.test.mjs index 7d13da0d..e0c7154b 100644 --- a/agent-compatibility/test/deterministic-scan.test.mjs +++ b/agent-compatibility/test/deterministic-scan.test.mjs @@ -11,6 +11,7 @@ import { buildScannerCommands, evaluateScannerOutput, inspectRepositorySignals, + resolveNpxRunner, runDeterministicScan, } from "../skills/check-agent-compatibility/scripts/run-deterministic-scan.mjs"; @@ -54,6 +55,29 @@ test("pins both scanner commands to agent-compatibility 0.1.7", () => { }); }); +test("runs npx through node on Windows without a command shell", () => { + const nodeExecutable = "C:\\Program Files\\nodejs\\node.exe"; + const npxCli = + "C:\\Program Files\\nodejs\\node_modules\\npm\\bin\\npx-cli.js"; + const runner = resolveNpxRunner({ + platform: "win32", + nodeExecutable, + npmExecPath: null, + pathExists: (path) => path === npxCli, + }); + + assert.deepEqual(runner, { + command: nodeExecutable, + prefixArgs: [npxCli], + }); + const commands = buildScannerCommands(genuineCliRoot, { runner }); + assert.deepEqual(commands.version, { + command: nodeExecutable, + args: [npxCli, "-y", "agent-compatibility@0.1.7", "--version"], + }); + assert.equal(Object.hasOwn(commands.version, "shell"), false); +}); + test("rejects a CLI classification for a Cloudflare Worker without a CLI entrypoint", () => { const signals = inspectRepositorySignals(cloudflareWorkerRoot); const assessment = assessClassification("cli", signals); @@ -196,7 +220,11 @@ test("executes through the documented symlinked plugin install", (t) => { "check-agent-compatibility", ); t.after(() => rmSync(temporaryDirectory, { recursive: true, force: true })); - symlinkSync(skillRoot, linkedSkillRoot, "dir"); + symlinkSync( + skillRoot, + linkedSkillRoot, + process.platform === "win32" ? "junction" : "dir", + ); const run = spawnSync( process.execPath, diff --git a/agent-compatibility/test/plugin-contract.test.mjs b/agent-compatibility/test/plugin-contract.test.mjs index 26f1e2ae..b659a6ee 100644 --- a/agent-compatibility/test/plugin-contract.test.mjs +++ b/agent-compatibility/test/plugin-contract.test.mjs @@ -209,6 +209,7 @@ test("CI runs contract tests whenever the plugin behavior can change", () => { workflow, /node --test agent-compatibility\/test\/\*\.test\.mjs/, ); + assert.match(workflow, /windows-latest/); }); test("release documentation matches the manifest version", () => { diff --git a/agent-compatibility/test/synthesize-results.test.mjs b/agent-compatibility/test/synthesize-results.test.mjs index d10954df..5311aaa5 100644 --- a/agent-compatibility/test/synthesize-results.test.mjs +++ b/agent-compatibility/test/synthesize-results.test.mjs @@ -385,7 +385,11 @@ test("CLI executes through the documented symlinked plugin install", (t) => { "check-agent-compatibility", ); t.after(() => rmSync(temporaryDirectory, { recursive: true, force: true })); - symlinkSync(skillRoot, linkedSkillRoot, "dir"); + symlinkSync( + skillRoot, + linkedSkillRoot, + process.platform === "win32" ? "junction" : "dir", + ); const run = spawnSync( process.execPath, From cbe450b2318de412f79e465c565a91cb9ae9d31a Mon Sep 17 00:00:00 2001 From: Gareth Paul Jones Date: Wed, 19 Aug 2026 20:14:43 +0000 Subject: [PATCH 3/3] Make compatibility tests cross-platform --- .../scripts/run-deterministic-scan.mjs | 4 ++-- .../test/deterministic-scan.test.mjs | 19 ++++++++++++++- .../test/synthesize-results.test.mjs | 23 +++++++++++-------- 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/agent-compatibility/skills/check-agent-compatibility/scripts/run-deterministic-scan.mjs b/agent-compatibility/skills/check-agent-compatibility/scripts/run-deterministic-scan.mjs index 80a2ec78..e4cf5e5e 100644 --- a/agent-compatibility/skills/check-agent-compatibility/scripts/run-deterministic-scan.mjs +++ b/agent-compatibility/skills/check-agent-compatibility/scripts/run-deterministic-scan.mjs @@ -686,7 +686,7 @@ function reportedVersion(stdout) { export async function runDeterministicScan( targetRootArgument, - { executeCommand = executeScannerCommand } = {}, + { executeCommand = executeScannerCommand, runner = resolveNpxRunner() } = {}, ) { let targetRoot; try { @@ -703,7 +703,7 @@ export async function runDeterministicScan( let scannerCommands; try { - scannerCommands = buildScannerCommands(targetRoot); + scannerCommands = buildScannerCommands(targetRoot, { runner }); } catch (error) { return unavailableResult({ targetRoot, diff --git a/agent-compatibility/test/deterministic-scan.test.mjs b/agent-compatibility/test/deterministic-scan.test.mjs index e0c7154b..131aa2f9 100644 --- a/agent-compatibility/test/deterministic-scan.test.mjs +++ b/agent-compatibility/test/deterministic-scan.test.mjs @@ -22,6 +22,7 @@ const cloudflareWorkerRoot = realpathSync( ); const genuineCliRoot = realpathSync(resolve(fixturesRoot, "genuine-cli")); const skillRoot = resolve(testDirectory, "../skills/check-agent-compatibility"); +const unixNpxRunner = { command: "npx", prefixArgs: [] }; function scannerOutput(targetRoot, kind) { return { @@ -43,7 +44,9 @@ function scannerOutput(targetRoot, kind) { } test("pins both scanner commands to agent-compatibility 0.1.7", () => { - const commands = buildScannerCommands(genuineCliRoot); + const commands = buildScannerCommands(genuineCliRoot, { + runner: unixNpxRunner, + }); assert.deepEqual(commands.version, { command: "npx", @@ -78,6 +81,18 @@ test("runs npx through node on Windows without a command shell", () => { assert.equal(Object.hasOwn(commands.version, "shell"), false); }); +test("resolves the real npm JavaScript entrypoint on hosted Windows", (t) => { + if (process.platform !== "win32") { + t.skip("Windows-only runtime assertion"); + return; + } + + const runner = resolveNpxRunner(); + assert.ok(runner, "expected npm's npx-cli.js beside the Node installation"); + assert.equal(runner.command, process.execPath); + assert.match(runner.prefixArgs[0], /npx-cli\.js$/i); +}); + test("rejects a CLI classification for a Cloudflare Worker without a CLI entrypoint", () => { const signals = inspectRepositorySignals(cloudflareWorkerRoot); const assessment = assessClassification("cli", signals); @@ -148,6 +163,7 @@ test("runs the version check before the scan and returns structured JSON", async const result = await runDeterministicScan(genuineCliRoot, { executeCommand, + runner: unixNpxRunner, }); assert.deepEqual( @@ -182,6 +198,7 @@ test("normalizes scanner execution failures instead of throwing", async () => { const result = await runDeterministicScan(genuineCliRoot, { executeCommand, + runner: unixNpxRunner, }); assert.deepEqual(result, { diff --git a/agent-compatibility/test/synthesize-results.test.mjs b/agent-compatibility/test/synthesize-results.test.mjs index 5311aaa5..e68bfa65 100644 --- a/agent-compatibility/test/synthesize-results.test.mjs +++ b/agent-compatibility/test/synthesize-results.test.mjs @@ -23,7 +23,9 @@ const scoreNames = { validation: "Validation Loop Score", docs: "Docs Reliability Score", }; -const targetRoot = "/workspace/target-repository"; +const targetRoot = resolve(tmpdir(), "target-repository"); +const startupExecutionRoot = resolve(tmpdir(), "startup-isolated-copy"); +const validationExecutionRoot = resolve(tmpdir(), "validation-isolated-copy"); function completeResults(overrides = {}) { const results = { @@ -43,7 +45,7 @@ function completeResults(overrides = {}) { scoreName: scoreNames.startup, score: 90, targetRoot, - executionRoot: "/tmp/startup-isolated-copy", + executionRoot: startupExecutionRoot, isolation: "isolated-copy", summary: "Startup completed in isolation.", evidence: ["startup command passed"], @@ -55,7 +57,7 @@ function completeResults(overrides = {}) { scoreName: scoreNames.validation, score: 80, targetRoot, - executionRoot: "/tmp/validation-isolated-copy", + executionRoot: validationExecutionRoot, isolation: "isolated-copy", summary: "Validation completed in isolation.", evidence: ["validation command passed"], @@ -261,7 +263,9 @@ test("rejects evidence-free, wrong-target, and non-isolated results", () => { assert.throws( () => validateResults( - completeResults({ docs: { targetRoot: "/workspace/wrong-target" } }), + completeResults({ + docs: { targetRoot: resolve(tmpdir(), "wrong-target") }, + }), ), /same targetRoot/, ); @@ -283,8 +287,10 @@ test("rejects evidence-free, wrong-target, and non-isolated results", () => { () => validateResults( completeResults({ - startup: { executionRoot: "/tmp/shared-isolated-copy" }, - validation: { executionRoot: "/tmp/shared-isolated-copy/." }, + startup: { executionRoot: resolve(tmpdir(), "shared-isolated-copy") }, + validation: { + executionRoot: resolve(tmpdir(), "shared-isolated-copy", "."), + }, }), ), /separate executionRoot/, @@ -302,10 +308,7 @@ test("retains validated target, evidence, and isolation provenance", () => { assert.deepEqual(result.components.docs.evidence, [ "README.md:10 maps to package.json#scripts.test", ]); - assert.equal( - result.components.startup.executionRoot, - "/tmp/startup-isolated-copy", - ); + assert.equal(result.components.startup.executionRoot, startupExecutionRoot); assert.equal(result.components.startup.isolation, "isolated-copy"); });