From 81b99a58d83917a2db8837fec8621f813c987286 Mon Sep 17 00:00:00 2001 From: Bao Tran Date: Tue, 14 Jul 2026 11:52:14 +0700 Subject: [PATCH 01/14] feat: add one-command readiness upload - Analyze the current repository and upload its readiness snapshot by default\n- Reject legacy readiness arguments to keep the workflow unambiguous\n- Preserve the existing local backend override outside this commit --- src/index.tsx | 9 + src/lib/help.ts | 1 + src/lib/readiness.ts | 400 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 410 insertions(+) create mode 100644 src/lib/readiness.ts diff --git a/src/index.tsx b/src/index.tsx index c27e8ce..4b7b253 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -7,6 +7,7 @@ import { forwardToCodegraph } from "@/lib/codegraph.js"; import { printHelp, printVersion } from "@/lib/help.js"; import { initLogging } from "@/lib/log.js"; import { runLogs } from "@/lib/logs.js"; +import { runReadiness } from "@/lib/readiness.js"; import { ensureNodeSqliteOrReexec } from "@/lib/reexec.js"; import { ensureFreshGatewayKey } from "@/lib/refresh.js"; import { @@ -211,6 +212,14 @@ switch (command) { process.exit(runLogs(args)); break; } + case "readiness": { + if (args.length > 0) { + console.error("Usage: codev readiness"); + process.exit(1); + } + process.exit(await runReadiness()); + break; + } case "restore": { const agent = args[0]; if (agent === undefined) { diff --git a/src/lib/help.ts b/src/lib/help.ts index 9c3e141..576f6d3 100644 --- a/src/lib/help.ts +++ b/src/lib/help.ts @@ -16,6 +16,7 @@ Commands: upload Export and upload logs to the monitor module (--force, -f re-uploads every conversation) model Switch the default model + readiness Analyze and upload readiness for the current repository restore [agent] Restore an agent's pre-CoDev config from its *.backup (no arg processes every agent) logs Show the last run from the diagnostic log diff --git a/src/lib/readiness.ts b/src/lib/readiness.ts new file mode 100644 index 0000000..fd14f13 --- /dev/null +++ b/src/lib/readiness.ts @@ -0,0 +1,400 @@ +import { spawnSync } from "node:child_process"; +import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { join, relative } from "node:path"; +import { login } from "@/lib/auth.js"; +import { BACKEND_URL } from "@/lib/const.js"; + +interface Criterion { + numerator: number | null; + denominator: number; + rationale: string; +} + +type Report = Record; + +const APP_CRITERIA = [ + "lint_config", + "type_check", + "formatter", + "pre_commit_hooks", + "strict_typing", + "naming_consistency", + "cyclomatic_complexity", + "dead_code_detection", + "duplicate_code_detection", + "code_modularization", + "n_plus_one_detection", + "heavy_dependency_detection", + "unused_dependencies_detection", + "unit_tests_exist", + "integration_tests_exist", + "unit_tests_runnable", + "test_performance_tracking", + "flaky_test_detection", + "test_coverage_thresholds", + "test_naming_conventions", + "test_isolation", + "api_schema_docs", + "database_schema", + "structured_logging", + "distributed_tracing", + "metrics_collection", + "code_quality_metrics", + "error_tracking_contextualized", + "alerting_configured", + "deployment_observability", + "health_checks", + "circuit_breakers", + "profiling_instrumentation", + "dast_scanning", + "pii_handling", + "log_scrubbing", + "product_analytics_instrumentation", + "error_to_insight_pipeline", +] as const; + +const REPO_CRITERIA = [ + "large_file_detection", + "tech_debt_tracking", + "build_cmd_doc", + "deps_pinned", + "vcs_cli_tools", + "automated_pr_review", + "agentic_development", + "fast_ci_feedback", + "build_performance_tracking", + "deployment_frequency", + "single_command_setup", + "feature_flag_infrastructure", + "release_notes_automation", + "progressive_rollout", + "rollback_automation", + "monorepo_tooling", + "version_drift_detection", + "release_automation", + "dead_feature_flag_detection", + "agents_md", + "readme", + "automated_doc_generation", + "skills", + "documentation_freshness", + "service_flow_documented", + "agents_md_validation", + "devcontainer", + "env_template", + "local_services_setup", + "devcontainer_runnable", + "runbooks_documented", + "branch_protection", + "secret_scanning", + "codeowners", + "automated_security_review", + "dependency_update_automation", + "gitignore_comprehensive", + "privacy_compliance", + "secrets_management", + "min_release_age", + "issue_templates", + "issue_labeling_system", + "backlog_health", + "pr_templates", +] as const; + +const ALL_CRITERIA = [...APP_CRITERIA, ...REPO_CRITERIA]; + +interface AppInfo { + description: string; + path: string; +} + +function walk(dir: string, maxDepth = 3, depth = 0): string[] { + if (depth > maxDepth || !existsSync(dir)) return []; + const out: string[] = []; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if ( + [ + ".git", + "node_modules", + ".next", + "dist", + "build", + ".venv", + "venv", + ].includes(entry.name) + ) + continue; + const path = join(dir, entry.name); + out.push(path); + if (entry.isDirectory()) out.push(...walk(path, maxDepth, depth + 1)); + } + return out; +} + +function hasAny(root: string, patterns: RegExp[]) { + return walk(root).some((path) => + patterns.some((pattern) => pattern.test(relative(root, path))), + ); +} + +function fileText(path: string) { + try { + return readFileSync(path, "utf8"); + } catch { + return ""; + } +} + +function detectApps(root: string): Record { + const apps: Record = {}; + for (const path of walk(root, 2)) { + if ( + !existsSync(join(path, "package.json")) && + !existsSync(join(path, "requirements.txt")) && + !existsSync(join(path, "pyproject.toml")) + ) + continue; + const rel = relative(root, path) || "."; + const pkg = fileText(join(path, "package.json")); + const name = pkg ? (JSON.parse(pkg).name ?? rel) : rel; + apps[rel] = { path, description: `${name} application` }; + } + if (Object.keys(apps).length === 0) { + apps["."] = { path: root, description: "Repository root application" }; + } + return apps; +} + +function scoreApps( + apps: Record, + check: (path: string) => boolean, + label: string, +): Criterion { + const entries = Object.entries(apps); + const passed = entries.filter(([, app]) => check(app.path)); + return { + numerator: passed.length, + denominator: entries.length, + rationale: `${passed.length}/${entries.length} apps pass ${label}.`, + }; +} + +function criterion(numerator: number | null, rationale: string): Criterion { + return { numerator, denominator: 1, rationale }; +} + +function git(args: string[], cwd: string) { + const proc = spawnSync("git", args, { cwd, encoding: "utf8" }); + return proc.status === 0 ? proc.stdout.trim() : ""; +} + +function repoUrl(root: string) { + return git(["config", "--get", "remote.origin.url"], root) || root; +} + +function buildReport(root: string, apps: Record): Report { + const report = Object.fromEntries( + ALL_CRITERIA.map((id) => [ + id, + { + numerator: 0, + denominator: 1, + rationale: "No matching evidence found.", + }, + ]), + ) as Report; + + report.lint_config = scoreApps( + apps, + (p) => + hasAny(p, [ + /eslint\.config\./, + /\.eslintrc/, + /biome\.json/, + /ruff\.toml/, + /pyproject\.toml/, + ]), + "linter config", + ); + report.type_check = scoreApps( + apps, + (p) => + hasAny(p, [ + /tsconfig\.json/, + /pyrightconfig\.json/, + /mypy\.ini/, + /pyproject\.toml/, + ]), + "type checking", + ); + report.formatter = scoreApps( + apps, + (p) => + hasAny(p, [/prettier/, /biome\.json/, /ruff\.toml/, /pyproject\.toml/]), + "formatter config", + ); + report.pre_commit_hooks = scoreApps( + apps, + (p) => hasAny(p, [/^\.husky\//, /\.pre-commit-config\.yaml/]), + "pre-commit hooks", + ); + report.unit_tests_exist = scoreApps( + apps, + (p) => hasAny(p, [/\.test\./, /\.spec\./, /^tests\//, /test_.*\.py$/]), + "unit tests", + ); + report.unit_tests_runnable = scoreApps( + apps, + (p) => + fileText(join(p, "package.json")).includes('"test"') || + hasAny(p, [/pytest\.ini/, /pyproject\.toml/]), + "test command", + ); + report.database_schema = scoreApps( + apps, + (p) => hasAny(p, [/migrations\//, /\.sql$/]), + "database schema files", + ); + report.health_checks = scoreApps( + apps, + (p) => /\/health|healthcheck/i.test(walk(p, 2).map(fileText).join("\n")), + "health checks", + ); + report.structured_logging = scoreApps( + apps, + (p) => + /pino|winston|logrus|logging\.|structlog|logger/i.test( + walk(p, 2).map(fileText).join("\n"), + ), + "structured logging", + ); + + report.agents_md = criterion( + existsSync(join(root, "AGENTS.md")) ? 1 : 0, + existsSync(join(root, "AGENTS.md")) + ? "AGENTS.md exists." + : "AGENTS.md not found.", + ); + report.readme = criterion( + existsSync(join(root, "README.md")) ? 1 : 0, + existsSync(join(root, "README.md")) + ? "README.md exists." + : "README.md not found.", + ); + report.deps_pinned = criterion( + hasAny(root, [ + /package-lock\.json/, + /pnpm-lock\.yaml/, + /bun\.lock/, + /go\.sum/, + /uv\.lock/, + /poetry\.lock/, + ]) + ? 1 + : 0, + "Dependency lockfile scan completed.", + ); + report.env_template = criterion( + hasAny(root, [/\.env\.example$/, /\.env\.template$/]) ? 1 : 0, + "Environment template scan completed.", + ); + report.gitignore_comprehensive = criterion( + fileText(join(root, ".gitignore")).includes(".env") ? 1 : 0, + ".gitignore secret pattern scan completed.", + ); + report.devcontainer = criterion( + hasAny(root, [/^\.devcontainer\/devcontainer\.json$/]) ? 1 : 0, + "Devcontainer scan completed.", + ); + report.issue_templates = criterion( + hasAny(root, [/^\.github\/ISSUE_TEMPLATE\//]) ? 1 : 0, + "Issue template scan completed.", + ); + report.pr_templates = criterion( + hasAny(root, [/pull_request_template/i]) ? 1 : 0, + "PR template scan completed.", + ); + report.codeowners = criterion( + hasAny(root, [/(^|\/)CODEOWNERS$/]) ? 1 : 0, + "CODEOWNERS scan completed.", + ); + report.release_automation = criterion( + hasAny(root, [ + /^\.github\/workflows\/.*release/i, + /^\.github\/workflows\/.*deploy/i, + ]) + ? 1 + : 0, + "Release workflow scan completed.", + ); + report.agentic_development = criterion( + /factory-droid|droid|claude|codex/i.test( + git(["log", "--oneline", "-50"], root), + ) + ? 1 + : 0, + "Recent commit authorship scan completed.", + ); + report.skills = criterion( + hasAny(root, [ + /(^|\/)\.factory\/skills\//, + /(^|\/)\.agents\/skills\//, + /(^|\/)\.claude\/skills\//, + ]) + ? 1 + : 0, + "Agent skill directory scan completed.", + ); + + for (const id of ALL_CRITERIA) { + const item = report[id]; + if ( + item?.rationale === "No matching evidence found." && + APP_CRITERIA.includes(id as (typeof APP_CRITERIA)[number]) + ) { + report[id] = { + numerator: null, + denominator: Object.keys(apps).length, + rationale: "Not evaluated by the deterministic MVP analyzer.", + }; + } + } + return report; +} + +export async function runReadiness() { + const root = process.cwd(); + const apps = detectApps(root); + const report = buildReport(root, apps); + const payload = { + repoUrl: repoUrl(root), + branch: git(["branch", "--show-current"], root), + commitHash: git(["rev-parse", "HEAD"], root), + apps: Object.fromEntries( + Object.entries(apps).map(([key, app]) => [ + key, + { description: app.description }, + ]), + ), + modelUsed: { id: "codev-deterministic-readiness" }, + report, + }; + + const auth = await login(console.error, () => {}); + const res = await fetch(`${BACKEND_URL}/readiness/reports`, { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Bearer ${auth.access_token}`, + }, + body: JSON.stringify(payload), + }); + if (!res.ok) { + console.error( + `Readiness upload failed (${res.status}): ${await res.text()}`, + ); + return 1; + } + const data = (await res.json()) as { report?: { id?: string } }; + console.log(`Stored readiness report ${data.report?.id ?? ""}`); + return 0; +} From 125f3a8f5994b10766af32f86a7d427e8851f1c5 Mon Sep 17 00:00:00 2001 From: Bao Tran Date: Wed, 15 Jul 2026 12:42:52 +0700 Subject: [PATCH 02/14] feat: add agent-driven readiness evaluation --- README.md | 21 + src/ReadinessApp.tsx | 124 ++++++ src/components/ReadinessAgentSelect.tsx | 85 ++++ src/index.tsx | 16 +- src/lib/const.ts | 3 +- src/lib/readiness-agent.ts | 499 +++++++++++++++++++++ src/lib/readiness-config.ts | 44 ++ src/lib/readiness-contract.ts | 438 +++++++++++++++++++ src/lib/readiness.ts | 557 +++++++++--------------- tests/ReadinessApp.test.tsx | 40 ++ tests/lib/readiness-contract.test.ts | 288 ++++++++++++ 11 files changed, 1748 insertions(+), 367 deletions(-) create mode 100644 src/ReadinessApp.tsx create mode 100644 src/components/ReadinessAgentSelect.tsx create mode 100644 src/lib/readiness-agent.ts create mode 100644 src/lib/readiness-config.ts create mode 100644 src/lib/readiness-contract.ts create mode 100644 tests/ReadinessApp.test.tsx create mode 100644 tests/lib/readiness-contract.test.ts diff --git a/README.md b/README.md index 5d85fa4..ff408a1 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,27 @@ Details worth knowing: - **Retention** is automatic: files older than 14 days are pruned, and the directory is capped at 50 MB. - **Tuning:** `CODEV_LOG_LEVEL` (`debug` by default; `silent` disables logging), `CODEV_LOG_DIR` relocates the directory. +## Agent readiness + +Run `codev readiness` from a Git repository to choose Claude Code, Codex, or +OpenCode and start a headless, read-only readiness evaluation. CoDev validates +the agent's evidence against its versioned rubric, computes scores locally, +rejects scans that change the working tree, and uploads the validated report. +The report is a local agent evaluation, not a server-verified audit. + +During development use `bun dev readiness`. Factory fixtures and benchmark +utilities live in the separate top-level `agent-readiness-benchmarks` package; +they are not shipped with or imported by the CLI. + +Readiness uses each harness's configured default model for Claude Code and +Codex; `codev readiness --model ` provides a one-off override for +those harnesses. OpenCode readiness always uses +`aigateway/MiniMax/MiniMax-M2.7`. Repair, timeout, and output limits are bounded +product safeguards rather than user configuration. The selected harness must +first be configured through `codev install`. +The rubric version is deliberately not configurable: it identifies the actual +report schema and scoring rubric shared by CLI, proxy, and dashboard. + ## Development ```bash diff --git a/src/ReadinessApp.tsx b/src/ReadinessApp.tsx new file mode 100644 index 0000000..b0d80b7 --- /dev/null +++ b/src/ReadinessApp.tsx @@ -0,0 +1,124 @@ +import { Box, Text, useApp } from "ink"; +import Spinner from "ink-spinner"; +import { useCallback, useEffect, useMemo, useState } from "react"; +import { Banner } from "@/components/Banner.js"; +import { Frame } from "@/components/Frame.js"; +import { + ReadinessAgentSelect, + readinessAgentSelectTitle, +} from "@/components/ReadinessAgentSelect.js"; +import { Step } from "@/components/Step.js"; +import { + type ReadinessOptions, + type ReadinessRunResult, + runReadiness, +} from "@/lib/readiness.js"; +import { + isAgentAvailable, + READINESS_AGENTS, + type ReadinessAgent, +} from "@/lib/readiness-agent.js"; + +type Phase = "select" | "running" | "done" | "failed"; + +interface ReadinessAppProps { + available?: Record; + run?: typeof runReadiness; + options?: ReadinessOptions; +} + +export function ReadinessApp({ + available, + run = runReadiness, + options = {}, +}: ReadinessAppProps) { + const { exit } = useApp(); + const detected = useMemo( + () => + available ?? + (Object.fromEntries( + READINESS_AGENTS.map((agent) => [agent, isAgentAvailable(agent)]), + ) as Record), + [available], + ); + const [phase, setPhase] = useState("select"); + const [agent, setAgent] = useState(null); + const [progress, setProgress] = useState("Preparing readiness scan"); + const [result, setResult] = useState(null); + const hasAvailableAgent = READINESS_AGENTS.some( + (candidate) => detected[candidate], + ); + + const selectAgent = useCallback( + (choice: ReadinessAgent) => { + setAgent(choice); + setPhase("running"); + run(choice, setProgress, options) + .then((next) => { + setResult(next); + setPhase(next.exitCode === 0 ? "done" : "failed"); + }) + .catch((error) => { + setResult({ + exitCode: 1, + message: error instanceof Error ? error.message : String(error), + }); + setPhase("failed"); + }); + }, + [run, options], + ); + + useEffect(() => { + if (phase !== "done") return; + const timer = setTimeout(() => exit(), 50); + return () => clearTimeout(timer); + }, [phase, exit]); + + return ( + + + + {!hasAvailableAgent && ( + + No supported coding agent is available. Run `codev install` first. + + )} + + + + {phase !== "select" && ( + Evaluate repository} + > + {phase === "running" ? ( + + + + + {` ${progress}...`} + + ) : ( + + {phase === "done" ? "✓ " : "✗ "} + {result?.message} + + )} + + )} + {phase === "failed" && ( + Fix the issue above and rerun `codev readiness`. + )} + + + ); +} diff --git a/src/components/ReadinessAgentSelect.tsx b/src/components/ReadinessAgentSelect.tsx new file mode 100644 index 0000000..ab90b5f --- /dev/null +++ b/src/components/ReadinessAgentSelect.tsx @@ -0,0 +1,85 @@ +import { Box, Text, useInput } from "ink"; +import { useState } from "react"; +import { + READINESS_AGENTS, + type ReadinessAgent, +} from "@/lib/readiness-agent.js"; + +const LABELS: Record = { + claude: "Claude Code", + codex: "Codex", + opencode: "OpenCode", +}; + +interface ReadinessAgentSelectProps { + available: Record; + selected?: ReadinessAgent | null; + readOnly?: boolean; + onSelect: (agent: ReadinessAgent) => void; +} + +export function ReadinessAgentSelect({ + available, + selected = null, + readOnly = false, + onSelect, +}: ReadinessAgentSelectProps) { + const firstAvailable = Math.max( + 0, + READINESS_AGENTS.findIndex((agent) => available[agent]), + ); + const [cursor, setCursor] = useState(firstAvailable); + + useInput( + (_input, key) => { + if (key.upArrow || key.downArrow) { + const direction = key.upArrow ? -1 : 1; + for (let step = 1; step <= READINESS_AGENTS.length; step++) { + const next = + (cursor + direction * step + READINESS_AGENTS.length) % + READINESS_AGENTS.length; + const candidate = READINESS_AGENTS[next]; + if (candidate && available[candidate]) { + setCursor(next); + break; + } + } + } else if (key.return) { + const choice = READINESS_AGENTS[cursor]; + if (choice && available[choice]) onSelect(choice); + } + }, + { isActive: !readOnly }, + ); + + return ( + + {READINESS_AGENTS.map((agent, index) => { + const enabled = available[agent]; + const chosen = selected === agent; + const active = !readOnly && cursor === index && enabled; + return ( + + + {chosen ? "●" : "○"} + + + + {LABELS[agent]} + {enabled ? "" : " (unavailable)"} + + + ); + })} + + ); +} + +export function readinessAgentSelectTitle(readOnly = false) { + return ( + + {"Choose coding agent "} + {!readOnly && (↑/↓ to move, Enter to confirm)} + + ); +} diff --git a/src/index.tsx b/src/index.tsx index 4b7b253..d404c23 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -7,7 +7,6 @@ import { forwardToCodegraph } from "@/lib/codegraph.js"; import { printHelp, printVersion } from "@/lib/help.js"; import { initLogging } from "@/lib/log.js"; import { runLogs } from "@/lib/logs.js"; -import { runReadiness } from "@/lib/readiness.js"; import { ensureNodeSqliteOrReexec } from "@/lib/reexec.js"; import { ensureFreshGatewayKey } from "@/lib/refresh.js"; import { @@ -28,6 +27,7 @@ import { } from "@/lib/shims.js"; import { runUploadDaemon, spawnUploadDaemon } from "@/lib/upload.js"; import { ModelApp } from "@/ModelApp.js"; +import { ReadinessApp } from "@/ReadinessApp.js"; import { RemoveApp } from "@/RemoveApp.js"; import { UpdateApp } from "@/UpdateApp.js"; import { UploadApp } from "@/UploadApp.js"; @@ -213,11 +213,19 @@ switch (command) { break; } case "readiness": { - if (args.length > 0) { - console.error("Usage: codev readiness"); + const modelIndex = args.indexOf("--model"); + const model = modelIndex >= 0 ? args[modelIndex + 1] : undefined; + if ((modelIndex >= 0 && !model) || args.length !== (model ? 2 : 0)) { + console.error("Usage: codev readiness [--model ]"); + process.exit(1); + } + const { waitUntilExit } = render(); + try { + await waitUntilExit(); + process.exit(0); + } catch { process.exit(1); } - process.exit(await runReadiness()); break; } case "restore": { diff --git a/src/lib/const.ts b/src/lib/const.ts index ceb4498..69e4be5 100644 --- a/src/lib/const.ts +++ b/src/lib/const.ts @@ -4,7 +4,8 @@ import { join } from "node:path"; import pkg from "../../package.json" with { type: "json" }; const BASE_URL = atob("aHR0cHM6Ly9uZXRtaW5kLnZpZXR0ZWwudm4="); -export const BACKEND_URL = `${BASE_URL}/codev-proxy`; +// export const BACKEND_URL = `${BASE_URL}/codev-proxy`; +export const BACKEND_URL = `http://localhost:8787`; export const SSO_URL = `${BASE_URL}/sso-wrapper`; export const LOGIN_SUCCESS_URL = `${BASE_URL}/codev-landing-page`; diff --git a/src/lib/readiness-agent.ts b/src/lib/readiness-agent.ts new file mode 100644 index 0000000..4563173 --- /dev/null +++ b/src/lib/readiness-agent.ts @@ -0,0 +1,499 @@ +import { spawn, spawnSync } from "node:child_process"; +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { loadApiKey } from "@/lib/auth.js"; +import { detectConfiguredTools } from "@/lib/configure.js"; +import { AI_GATEWAY_OPENAI_URL } from "@/lib/const.js"; +import { readinessRuntimeConfig } from "@/lib/readiness-config.js"; +import { + type AgentReadinessOutput, + READINESS_RUBRIC, + READINESS_RUBRIC_VERSION, + readinessJsonSchema, +} from "@/lib/readiness-contract.js"; +import { stripShimDirFromPath } from "@/lib/shims.js"; + +export const READINESS_AGENTS = ["claude", "codex", "opencode"] as const; +export type ReadinessAgent = (typeof READINESS_AGENTS)[number]; + +export function assertReadinessPrerequisites(agent: ReadinessAgent): void { + if (agent === "codex" && isAgentAvailable(agent)) return; + const tool = agent === "claude" ? "claude-code" : agent; + if (detectConfiguredTools().includes(tool)) return; + throw new Error( + `${agent === "claude" ? "Claude Code" : agent === "codex" ? "Codex" : "OpenCode"} is not configured by CoDev. Run \`codev install\`, select this agent, and retry the readiness scan.`, + ); +} + +export interface AgentRunResult { + output: AgentReadinessOutput; + raw: string; + provider: ReadinessAgent; + durationMs: number; + sessionId?: string; +} + +interface ProcessResult { + code: number; + stdout: string; + stderr: string; +} + +export type AgentProgress = (message: string) => void; + +export function activityFromLine(line: string): string | undefined { + try { + const event = JSON.parse(line) as Record; + const part = event.part as Record | undefined; + const item = event.item as Record | undefined; + const message = event.message as Record | undefined; + const content = Array.isArray(message?.content) + ? (message.content as Array>) + : []; + const claudeTool = content.find( + (entry) => entry.type === "tool_use" && typeof entry.name === "string", + )?.name; + const tool = + (typeof part?.tool === "string" && part.tool) || + (typeof claudeTool === "string" && claudeTool) || + (typeof item?.type === "string" && item.type === "command_execution" + ? "command" + : undefined); + if (tool) return `Inspecting repository with ${tool}`; + if (event.type === "turn.started" || event.type === "step_start") + return "Agent evaluation step started"; + if (event.type === "turn.completed" || event.type === "step_finish") + return "Agent evaluation step completed"; + if (event.type === "item.completed") + return "Processing repository evidence"; + } catch {} + return undefined; +} + +export function providerFailureFromLine(line: string): string | undefined { + try { + const event = JSON.parse(line) as Record; + if ( + event.type === "system" && + event.subtype === "api_retry" && + event.error === "authentication_failed" + ) + return "Agent authentication failed. Run `codev login`, then `codev config` to refresh the selected harness credentials."; + } catch {} + return undefined; +} + +export function isAgentAvailable(agent: ReadinessAgent): boolean { + return ( + spawnSync(agent, ["--version"], { + stdio: "ignore", + timeout: 5_000, + env: readinessProcessEnv(), + }).status === 0 + ); +} + +export function readinessProcessEnv( + overrides: NodeJS.ProcessEnv = {}, + base: NodeJS.ProcessEnv = process.env, +): NodeJS.ProcessEnv { + return { + ...base, + PATH: stripShimDirFromPath(base.PATH), + ...overrides, + }; +} + +export function claudeReadinessEnvOverrides(): NodeJS.ProcessEnv { + return { + ANTHROPIC_API_KEY: undefined, + ANTHROPIC_AUTH_TOKEN: undefined, + ANTHROPIC_BASE_URL: undefined, + ANTHROPIC_DEFAULT_HAIKU_MODEL: undefined, + ANTHROPIC_DEFAULT_OPUS_MODEL: undefined, + ANTHROPIC_DEFAULT_SONNET_MODEL: undefined, + ANTHROPIC_MODEL: undefined, + CLAUDE_CODE_USE_BEDROCK: undefined, + CLAUDE_CODE_USE_VERTEX: undefined, + }; +} + +export function buildReadinessPrompt(): string { + return `You are evaluating how ready a software repository is for autonomous coding agents. + +Work read-only. Do not edit, create, delete, format, commit, install dependencies, access the network, or spawn subagents. You may inspect repository files and git history and run safe, non-mutating diagnostic commands. Judge semantic evidence across languages and frameworks; do not rely only on filenames. + +Finish efficiently. First inventory the repository with one batched file-listing command. Prefer manifests, CI configuration, test configuration, documentation, security policy, deployment files, and a small representative source sample. Batch related reads/searches and reuse the same evidence across criteria. Do not inspect every source file or run expensive builds/tests. Aim for no more than 12 repository commands before producing the report. + +Discover independently deployable applications first. Evaluate every rubric criterion. Use "skipped" only when it is genuinely inapplicable or cannot be established from local evidence. A failure means the criterion applies but adequate support is absent. For application-scoped criteria, numerator and denominator count evaluated applications. Repository-scoped criteria use denominator 1. Evidence entries must be existing repository-relative file or directory paths; file paths may optionally be followed by a line number. Use an empty evidence array when the rationale describes something that is absent. Never cite a nonexistent placeholder, URL, or command as evidence; describe command evidence in the rationale instead. + +Return only the JSON object matching the supplied schema. Do not include aggregate scores. + +Rubric version: ${READINESS_RUBRIC_VERSION} +Rubric: +${JSON.stringify(READINESS_RUBRIC, null, 2)}`; +} + +export function openCodeStructuredOutputInstruction(): string { + return `Return exactly one JSON object with this shape and no markdown fence: +{"rubricVersion":"${READINESS_RUBRIC_VERSION}","languages":["string"],"applications":[{"path":".","description":"string","languages":["string"]}],"criteria":{"":{"status":"pass|fail|skipped","numerator":"integer or null","denominator":"positive integer","rationale":"string","evidence":["existing repository-relative path"]}},"warnings":["string"],"recommendations":["2 or 3 strings"],"model":"string or null"} +Use null numerator only for skipped criteria. Include every rubric id exactly once and no additional criterion ids.`; +} + +export function buildAgentCommand( + agent: ReadinessAgent, + prompt: string, + schemaPath: string, + outputPath: string, + sessionId?: string, + modelOverride?: string, +): { command: string; args: string[] } { + if (agent === "claude") { + const args = [ + "-p", + prompt, + "--output-format", + "stream-json", + "--verbose", + "--json-schema", + readFileSync(schemaPath, "utf8"), + "--permission-mode", + "plan", + "--max-turns", + "80", + "--allowedTools", + "Read", + "Glob", + "Grep", + "Bash(git status:*)", + "Bash(git log:*)", + "Bash(git show:*)", + "Bash(git diff:*)", + "Bash(git ls-files:*)", + ]; + if (modelOverride) args.push("--model", modelOverride); + if (sessionId) args.push("--resume", sessionId); + return { command: "claude", args }; + } + if (agent === "codex") { + const config = readinessRuntimeConfig(modelOverride); + const modelArgs = [ + ...(config.model ? ["--model", config.model] : []), + "--config", + `model_reasoning_effort=${JSON.stringify(config.codexReasoningEffort)}`, + ]; + if (sessionId) + return { + command: "codex", + args: [ + "exec", + "resume", + ...modelArgs, + "--json", + "--output-schema", + schemaPath, + "--output-last-message", + outputPath, + sessionId, + prompt, + ], + }; + return { + command: "codex", + args: [ + "exec", + ...modelArgs, + "--sandbox", + "read-only", + "--json", + "--output-schema", + schemaPath, + "--output-last-message", + outputPath, + prompt, + ], + }; + } + const args = [ + "run", + "--format", + "json", + "-m", + readinessRuntimeConfig().opencodeModel, + `${prompt}\n\n${openCodeStructuredOutputInstruction()}`, + ]; + if (sessionId) args.push("--session", sessionId); + return { command: "opencode", args }; +} + +function runProcess( + command: string, + args: string[], + cwd: string, + envOverrides: NodeJS.ProcessEnv = {}, + timeoutMs?: number, + onProgress: AgentProgress = () => {}, +): Promise { + return new Promise((resolve, reject) => { + const config = readinessRuntimeConfig(); + const child = spawn(command, args, { + cwd, + env: readinessProcessEnv(envOverrides), + stdio: ["ignore", "pipe", "pipe"], + }); + let out = ""; + let err = ""; + let size = 0; + let timedOut = false; + let stdoutLines = ""; + let lastProgress = 0; + let providerFailure: string | undefined; + const effectiveTimeoutMs = timeoutMs ?? config.timeoutMs; + const timer = setTimeout(() => { + timedOut = true; + child.kill("SIGTERM"); + setTimeout(() => child.kill("SIGKILL"), 5_000).unref(); + }, effectiveTimeoutMs); + const append = (target: "out" | "err", chunk: Buffer) => { + size += chunk.byteLength; + if (size > config.maxOutputBytes) { + child.kill("SIGTERM"); + return; + } + if (target === "out") { + const text = chunk.toString(); + out += text; + stdoutLines += text; + const lines = stdoutLines.split("\n"); + stdoutLines = lines.pop() ?? ""; + for (const line of lines) { + const failure = providerFailureFromLine(line); + if (failure && !providerFailure) { + providerFailure = failure; + child.kill("SIGTERM"); + } + const activity = activityFromLine(line); + if (activity && Date.now() - lastProgress >= 500) { + lastProgress = Date.now(); + onProgress(activity); + } + } + } else err += chunk.toString(); + }; + child.stdout.on("data", (chunk: Buffer) => append("out", chunk)); + child.stderr.on("data", (chunk: Buffer) => append("err", chunk)); + child.once("error", reject); + child.once("exit", (code) => { + clearTimeout(timer); + if (providerFailure) reject(new Error(providerFailure)); + else if (timedOut) + reject(new Error(`Agent timed out after ${effectiveTimeoutMs} ms.`)); + else if (size > config.maxOutputBytes) + reject( + new Error(`Agent output exceeded ${config.maxOutputBytes} bytes.`), + ); + else resolve({ code: code ?? 1, stdout: out, stderr: err }); + }); + }); +} + +function jsonCandidates(raw: string): unknown[] { + const candidates: unknown[] = []; + const add = (text: string) => { + try { + candidates.push(JSON.parse(text)); + } catch {} + }; + add(raw.trim()); + for (const line of raw.split("\n")) add(line.trim()); + const fenced = raw.match(/```(?:json)?\s*([\s\S]*?)```/i)?.[1]; + if (fenced) add(fenced.trim()); + const start = raw.indexOf("{"); + const end = raw.lastIndexOf("}"); + if (start >= 0 && end > start) add(raw.slice(start, end + 1)); + return candidates; +} + +function findOutput(value: unknown): AgentReadinessOutput | undefined { + if (!value || typeof value !== "object") return undefined; + const object = value as Record; + if (object.rubricVersion && object.criteria) + return object as unknown as AgentReadinessOutput; + for (const key of [ + "structured_output", + "result", + "content", + "text", + "message", + "data", + "part", + ]) { + const nested = object[key]; + if (typeof nested === "string") { + for (const candidate of jsonCandidates(nested)) { + const found = findOutput(candidate); + if (found) return found; + } + } else { + const found = findOutput(nested); + if (found) return found; + } + } + if (Array.isArray(value)) + for (const item of value) { + const found = findOutput(item); + if (found) return found; + } + return undefined; +} + +export function extractAgentOutput( + raw: string, +): AgentReadinessOutput | undefined { + for (const candidate of jsonCandidates(raw)) { + const output = findOutput(candidate); + if (output) return output; + } + return undefined; +} + +function sessionIdFrom(raw: string): string | undefined { + for (const candidate of jsonCandidates(raw)) { + if (candidate && typeof candidate === "object") { + const object = candidate as Record; + for (const key of ["session_id", "sessionID", "thread_id", "threadId"]) + if (typeof object[key] === "string") return object[key]; + } + } + return undefined; +} + +export async function runReadinessAgent( + agent: ReadinessAgent, + root: string, + repair?: { raw: string; errors: string[]; sessionId?: string }, + modelOverride?: string, + onProgress: AgentProgress = () => {}, +): Promise { + const temp = mkdtempSync(join(tmpdir(), "codev-readiness-")); + const schemaPath = join(temp, "schema.json"); + const outputPath = join(temp, "output.json"); + writeFileSync(schemaPath, JSON.stringify(readinessJsonSchema())); + let envOverrides: NodeJS.ProcessEnv = {}; + if (agent === "claude") envOverrides = claudeReadinessEnvOverrides(); + if (agent === "opencode") { + const credentials = loadApiKey(); + if (!credentials?.apiKey) + throw new Error( + "OpenCode readiness requires gateway credentials. Run `codev install` first.", + ); + const configDir = join(temp, "config", "opencode"); + mkdirSync(configDir, { recursive: true }); + const model = readinessRuntimeConfig(modelOverride).opencodeModel.replace( + /^aigateway\//, + "", + ); + writeFileSync( + join(configDir, "opencode.json"), + JSON.stringify({ + $schema: "https://opencode.ai/config.json", + model: `aigateway/${model}`, + provider: { + aigateway: { + npm: "@ai-sdk/openai-compatible", + name: "AI Gateway", + options: { + baseURL: credentials.baseUrl ?? AI_GATEWAY_OPENAI_URL, + apiKey: credentials.apiKey, + }, + models: { [model]: { name: model } }, + }, + }, + }), + { mode: 0o600 }, + ); + envOverrides = { + XDG_CONFIG_HOME: join(temp, "config"), + XDG_DATA_HOME: join(temp, "data"), + }; + } + const prompt = repair + ? `Your previous readiness output was invalid. Correct it and return the complete JSON object only. Do not rescan or change the repository. Remove nonexistent evidence entries and use an empty evidence array when the rationale describes absence. Existing repository-relative directories are valid evidence. Validation errors:\n${repair.errors.map((error) => `- ${error}`).join("\n")}\nPrevious output:\n${repair.raw.slice(-200_000)}` + : buildReadinessPrompt(); + const started = Date.now(); + const deadline = started + readinessRuntimeConfig(modelOverride).timeoutMs; + try { + const command = buildAgentCommand( + agent, + prompt, + schemaPath, + outputPath, + agent === "opencode" ? undefined : repair?.sessionId, + modelOverride, + ); + let result = await runProcess( + command.command, + command.args, + root, + envOverrides, + Math.max(1, deadline - Date.now()), + onProgress, + ); + let sessionId = sessionIdFrom(result.stdout); + let combinedStdout = result.stdout; + for (let turn = 1; agent === "opencode" && turn < 2; turn++) { + if (extractAgentOutput(combinedStdout) || !sessionId) break; + const continuation = buildAgentCommand( + agent, + "Your evaluation response did not match the required readiness contract. Reformat the completed evaluation now without rescanning the repository.", + schemaPath, + outputPath, + sessionId, + modelOverride, + ); + result = await runProcess( + continuation.command, + continuation.args, + root, + envOverrides, + Math.max(1, deadline - Date.now()), + onProgress, + ); + combinedStdout += `\n${result.stdout}`; + sessionId = sessionIdFrom(result.stdout) ?? sessionId; + } + const fileOutput = (() => { + try { + return readFileSync(outputPath, "utf8"); + } catch { + return ""; + } + })(); + const raw = [fileOutput, combinedStdout].filter(Boolean).join("\n"); + if (result.code !== 0) + throw new Error( + `${agent} exited with code ${result.code}: ${[result.stderr, result.stdout].filter(Boolean).join("\n").trim().slice(-4_000)}`, + ); + const output = extractAgentOutput(raw); + if (!output) + throw new Error( + `${agent} did not return a structured readiness object. Output tail:\n${raw.slice(-4_000)}`, + ); + return { + output, + raw, + provider: agent, + durationMs: Date.now() - started, + sessionId, + }; + } finally { + rmSync(temp, { recursive: true, force: true }); + } +} diff --git a/src/lib/readiness-config.ts b/src/lib/readiness-config.ts new file mode 100644 index 0000000..6c68b14 --- /dev/null +++ b/src/lib/readiness-config.ts @@ -0,0 +1,44 @@ +const DEFAULT_MAX_REPAIRS = 2; +const MAX_ALLOWED_REPAIRS = 5; +export const READINESS_OPENCODE_MODEL = "aigateway/MiniMax/MiniMax-M2.7"; + +function boundedInteger( + value: string | undefined, + fallback: number, + min: number, + max: number, +): number { + if (!value) return fallback; + const parsed = Number.parseInt(value, 10); + return Number.isInteger(parsed) && parsed >= min && parsed <= max + ? parsed + : fallback; +} + +export interface ReadinessRuntimeConfig { + opencodeModel: string; + model?: string; + codexReasoningEffort: "low"; + maxRepairs: number; + timeoutMs: number; + maxOutputBytes: number; +} + +export function readinessRuntimeConfig( + modelOverride?: string, +): ReadinessRuntimeConfig { + const model = modelOverride?.trim() || undefined; + return { + model, + opencodeModel: READINESS_OPENCODE_MODEL, + codexReasoningEffort: "low", + maxRepairs: boundedInteger( + undefined, + DEFAULT_MAX_REPAIRS, + 0, + MAX_ALLOWED_REPAIRS, + ), + timeoutMs: 20 * 60 * 1_000, + maxOutputBytes: 20 * 1024 * 1024, + }; +} diff --git a/src/lib/readiness-contract.ts b/src/lib/readiness-contract.ts new file mode 100644 index 0000000..744524c --- /dev/null +++ b/src/lib/readiness-contract.ts @@ -0,0 +1,438 @@ +import { existsSync } from "node:fs"; +import { isAbsolute, relative, resolve } from "node:path"; + +// This is a protocol/schema identifier, not runtime configuration. Change it +// only when READINESS_RUBRIC or the result contract changes, and update the +// proxy/web supported-version constants in the same change. An environment +// override could falsely label one rubric as another and corrupt comparisons. +export const READINESS_RUBRIC_VERSION = "2026-07-14.v1"; + +export type ReadinessStatus = "pass" | "fail" | "skipped"; + +export interface ReadinessCriterionDefinition { + id: string; + category: string; + maturityLevel: number; + description: string; + applicability: string; + evidenceRequired: string; +} + +export interface ReadinessCriterionResult { + status: ReadinessStatus; + numerator: number | null; + denominator: number; + rationale: string; + evidence: string[]; +} + +export interface ReadinessApplication { + path: string; + description: string; + languages: string[]; +} + +export interface AgentReadinessOutput { + rubricVersion: string; + languages: string[]; + applications: ReadinessApplication[]; + criteria: Record; + warnings: string[]; + recommendations: string[]; + model: string | null; +} + +const CATEGORIES: Array<[string, number, string[]]> = [ + [ + "Style & Validation", + 1, + [ + "lint_config", + "type_check", + "formatter", + "pre_commit_hooks", + "strict_typing", + "naming_consistency", + "cyclomatic_complexity", + "large_file_detection", + "dead_code_detection", + "duplicate_code_detection", + "code_modularization", + "tech_debt_tracking", + "n_plus_one_detection", + ], + ], + [ + "Build System", + 2, + [ + "build_cmd_doc", + "deps_pinned", + "vcs_cli_tools", + "automated_pr_review", + "agentic_development", + "fast_ci_feedback", + "build_performance_tracking", + "deployment_frequency", + "single_command_setup", + "feature_flag_infrastructure", + "release_notes_automation", + "progressive_rollout", + "rollback_automation", + "monorepo_tooling", + "heavy_dependency_detection", + "unused_dependencies_detection", + "version_drift_detection", + "release_automation", + "dead_feature_flag_detection", + ], + ], + [ + "Testing", + 2, + [ + "unit_tests_exist", + "integration_tests_exist", + "unit_tests_runnable", + "test_performance_tracking", + "flaky_test_detection", + "test_coverage_thresholds", + "test_naming_conventions", + "test_isolation", + ], + ], + [ + "Documentation", + 2, + [ + "agents_md", + "readme", + "automated_doc_generation", + "skills", + "documentation_freshness", + "api_schema_docs", + "service_flow_documented", + "agents_md_validation", + ], + ], + [ + "Dev Environment", + 2, + [ + "devcontainer", + "env_template", + "local_services_setup", + "database_schema", + "devcontainer_runnable", + ], + ], + [ + "Debugging & Observability", + 3, + [ + "structured_logging", + "distributed_tracing", + "metrics_collection", + "code_quality_metrics", + "error_tracking_contextualized", + "alerting_configured", + "runbooks_documented", + "deployment_observability", + "health_checks", + "circuit_breakers", + "profiling_instrumentation", + ], + ], + [ + "Security", + 3, + [ + "branch_protection", + "secret_scanning", + "codeowners", + "automated_security_review", + "dependency_update_automation", + "gitignore_comprehensive", + "dast_scanning", + "pii_handling", + "privacy_compliance", + "secrets_management", + "log_scrubbing", + "min_release_age", + ], + ], + [ + "Task Discovery", + 4, + [ + "issue_templates", + "issue_labeling_system", + "backlog_health", + "pr_templates", + ], + ], + [ + "Product & Analytics", + 5, + ["product_analytics_instrumentation", "error_to_insight_pipeline"], + ], +]; + +function label(id: string): string { + return id.replaceAll("_", " "); +} + +export const READINESS_RUBRIC: ReadinessCriterionDefinition[] = + CATEGORIES.flatMap(([category, maturityLevel, ids]) => + ids.map((id) => ({ + id, + category, + maturityLevel, + description: `Evaluate whether the repository has effective ${label(id)} support.`, + applicability: + "Evaluate each independently deployable application when applicable; otherwise evaluate once at repository level. Skip only when local repository evidence cannot establish the criterion or the criterion genuinely does not apply.", + evidenceRequired: + "Cite repository-relative files and, where useful, line numbers or the safe local command whose output supports the judgment.", + })), + ); + +export const READINESS_CRITERION_IDS = READINESS_RUBRIC.map(({ id }) => id); +const CRITERION_SET = new Set(READINESS_CRITERION_IDS); + +export interface ReadinessSummary { + criteriaPassed: number; + criteriaTotal: number; + passRate: number; + level: number; +} + +export function summarizeReadiness( + criteria: Record, +): ReadinessSummary { + let passed = 0; + let total = 0; + for (const item of Object.values(criteria)) { + if (item.status === "skipped" || item.numerator === null) continue; + passed += item.numerator / item.denominator; + total++; + } + const passRate = + total === 0 ? 0 : Math.round((passed / total) * 10_000) / 100; + return { + criteriaPassed: Math.round(passed * 100) / 100, + criteriaTotal: total, + passRate, + level: + passRate >= 80 + ? 5 + : passRate >= 60 + ? 4 + : passRate >= 40 + ? 3 + : passRate >= 20 + ? 2 + : 1, + }; +} + +function isInside(root: string, path: string): boolean { + const rel = relative(resolve(root), resolve(root, path)); + return rel === "" || (!rel.startsWith("..") && !isAbsolute(rel)); +} + +function evidencePath(evidence: string): string { + return evidence.split(":", 1)[0]?.trim() ?? ""; +} + +function isValidEvidence(root: string, evidence: string): boolean { + const path = evidencePath(evidence); + return Boolean( + path && isInside(root, path) && existsSync(resolve(root, path)), + ); +} + +export function normalizeReadinessEvidence( + output: AgentReadinessOutput, + root: string, +): AgentReadinessOutput { + return { + ...output, + criteria: Object.fromEntries( + Object.entries(output.criteria).map(([id, item]) => [ + id, + { + ...item, + evidence: Array.isArray(item.evidence) + ? item.evidence.filter((entry) => isValidEvidence(root, entry)) + : item.evidence, + }, + ]), + ), + }; +} + +export function validateReadinessOutput( + value: unknown, + root: string, +): string[] { + const errors: string[] = []; + if (!value || typeof value !== "object" || Array.isArray(value)) + return ["Output must be a JSON object."]; + const output = value as Partial; + if (output.rubricVersion !== READINESS_RUBRIC_VERSION) + errors.push(`rubricVersion must be ${READINESS_RUBRIC_VERSION}.`); + if ( + !Array.isArray(output.languages) || + output.languages.some((v) => typeof v !== "string") + ) + errors.push("languages must be an array of strings."); + if (!Array.isArray(output.applications) || output.applications.length === 0) { + errors.push("applications must contain at least one application."); + } else { + const paths = new Set(); + for (const [index, app] of output.applications.entries()) { + if ( + !app || + typeof app.path !== "string" || + !isInside(root, app.path) || + !existsSync(resolve(root, app.path)) + ) + errors.push( + `applications[${index}].path must exist inside the repository.`, + ); + else if (paths.has(app.path)) + errors.push(`Duplicate application path: ${app.path}.`); + else paths.add(app.path); + if (typeof app?.description !== "string" || app.description.trim() === "") + errors.push(`applications[${index}].description is required.`); + if ( + !Array.isArray(app?.languages) || + app.languages.some((v) => typeof v !== "string") + ) + errors.push(`applications[${index}].languages must be strings.`); + } + } + if ( + !output.criteria || + typeof output.criteria !== "object" || + Array.isArray(output.criteria) + ) { + errors.push("criteria must be an object."); + } else { + for (const id of Object.keys(output.criteria)) + if (!CRITERION_SET.has(id)) errors.push(`Unknown criterion: ${id}.`); + for (const id of READINESS_CRITERION_IDS) { + const item = output.criteria[id]; + if (!item) { + errors.push(`Missing criterion: ${id}.`); + continue; + } + if (!["pass", "fail", "skipped"].includes(item.status)) + errors.push(`${id}.status is invalid.`); + if (!Number.isInteger(item.denominator) || item.denominator <= 0) + errors.push(`${id}.denominator must be a positive integer.`); + if (item.status === "skipped") { + if (item.numerator !== null) + errors.push(`${id}.numerator must be null when skipped.`); + } else if ( + !Number.isInteger(item.numerator) || + (item.numerator ?? -1) < 0 || + (item.numerator ?? 0) > item.denominator + ) + errors.push( + `${id}.numerator must be an integer from zero through denominator.`, + ); + if (typeof item.rationale !== "string" || item.rationale.trim() === "") + errors.push(`${id}.rationale is required.`); + if ( + !Array.isArray(item.evidence) || + item.evidence.some((v) => typeof v !== "string") + ) + errors.push(`${id}.evidence must be an array of strings.`); + for (const evidence of item.evidence ?? []) { + if (!isValidEvidence(root, evidence)) + errors.push(`${id} references missing evidence: ${evidence}.`); + } + } + } + if ( + !Array.isArray(output.warnings) || + output.warnings.some((v) => typeof v !== "string") + ) + errors.push("warnings must be an array of strings."); + if ( + !Array.isArray(output.recommendations) || + output.recommendations.length < 2 || + output.recommendations.length > 3 || + output.recommendations.some((v) => typeof v !== "string") + ) + errors.push("recommendations must contain 2 or 3 strings."); + if (output.model !== null && typeof output.model !== "string") + errors.push("model must be a string or null."); + return errors; +} + +export function readinessJsonSchema(): Record { + const criterionSchema = { + type: "object", + additionalProperties: false, + required: ["status", "numerator", "denominator", "rationale", "evidence"], + properties: { + status: { enum: ["pass", "fail", "skipped"] }, + numerator: { type: ["integer", "null"] }, + denominator: { type: "integer", minimum: 1 }, + rationale: { type: "string", minLength: 1 }, + evidence: { type: "array", items: { type: "string" } }, + }, + }; + return { + type: "object", + additionalProperties: false, + required: [ + "rubricVersion", + "languages", + "applications", + "criteria", + "warnings", + "recommendations", + "model", + ], + properties: { + rubricVersion: { + type: "string", + const: READINESS_RUBRIC_VERSION, + }, + languages: { type: "array", items: { type: "string" } }, + applications: { + type: "array", + minItems: 1, + items: { + type: "object", + additionalProperties: false, + required: ["path", "description", "languages"], + properties: { + path: { type: "string" }, + description: { type: "string" }, + languages: { type: "array", items: { type: "string" } }, + }, + }, + }, + criteria: { + type: "object", + additionalProperties: false, + required: READINESS_CRITERION_IDS, + properties: Object.fromEntries( + READINESS_CRITERION_IDS.map((id) => [id, criterionSchema]), + ), + }, + warnings: { type: "array", items: { type: "string" } }, + recommendations: { + type: "array", + minItems: 2, + maxItems: 3, + items: { type: "string" }, + }, + model: { type: ["string", "null"] }, + }, + }; +} diff --git a/src/lib/readiness.ts b/src/lib/readiness.ts index fd14f13..81a9948 100644 --- a/src/lib/readiness.ts +++ b/src/lib/readiness.ts @@ -1,400 +1,233 @@ import { spawnSync } from "node:child_process"; -import { existsSync, readdirSync, readFileSync } from "node:fs"; -import { join, relative } from "node:path"; -import { login } from "@/lib/auth.js"; +import { type AuthData, login } from "@/lib/auth.js"; import { BACKEND_URL } from "@/lib/const.js"; - -interface Criterion { - numerator: number | null; - denominator: number; - rationale: string; +import { + type AgentRunResult, + assertReadinessPrerequisites, + type ReadinessAgent, + runReadinessAgent, +} from "@/lib/readiness-agent.js"; +import { readinessRuntimeConfig } from "@/lib/readiness-config.js"; +import { + normalizeReadinessEvidence, + READINESS_RUBRIC_VERSION, + summarizeReadiness, + validateReadinessOutput, +} from "@/lib/readiness-contract.js"; +import { ensureFreshGatewayKey } from "@/lib/refresh.js"; + +function git(args: string[], cwd: string): string { + const result = spawnSync("git", args, { cwd, encoding: "utf8" }); + return result.status === 0 ? result.stdout.trim() : ""; } -type Report = Record; - -const APP_CRITERIA = [ - "lint_config", - "type_check", - "formatter", - "pre_commit_hooks", - "strict_typing", - "naming_consistency", - "cyclomatic_complexity", - "dead_code_detection", - "duplicate_code_detection", - "code_modularization", - "n_plus_one_detection", - "heavy_dependency_detection", - "unused_dependencies_detection", - "unit_tests_exist", - "integration_tests_exist", - "unit_tests_runnable", - "test_performance_tracking", - "flaky_test_detection", - "test_coverage_thresholds", - "test_naming_conventions", - "test_isolation", - "api_schema_docs", - "database_schema", - "structured_logging", - "distributed_tracing", - "metrics_collection", - "code_quality_metrics", - "error_tracking_contextualized", - "alerting_configured", - "deployment_observability", - "health_checks", - "circuit_breakers", - "profiling_instrumentation", - "dast_scanning", - "pii_handling", - "log_scrubbing", - "product_analytics_instrumentation", - "error_to_insight_pipeline", -] as const; - -const REPO_CRITERIA = [ - "large_file_detection", - "tech_debt_tracking", - "build_cmd_doc", - "deps_pinned", - "vcs_cli_tools", - "automated_pr_review", - "agentic_development", - "fast_ci_feedback", - "build_performance_tracking", - "deployment_frequency", - "single_command_setup", - "feature_flag_infrastructure", - "release_notes_automation", - "progressive_rollout", - "rollback_automation", - "monorepo_tooling", - "version_drift_detection", - "release_automation", - "dead_feature_flag_detection", - "agents_md", - "readme", - "automated_doc_generation", - "skills", - "documentation_freshness", - "service_flow_documented", - "agents_md_validation", - "devcontainer", - "env_template", - "local_services_setup", - "devcontainer_runnable", - "runbooks_documented", - "branch_protection", - "secret_scanning", - "codeowners", - "automated_security_review", - "dependency_update_automation", - "gitignore_comprehensive", - "privacy_compliance", - "secrets_management", - "min_release_age", - "issue_templates", - "issue_labeling_system", - "backlog_health", - "pr_templates", -] as const; - -const ALL_CRITERIA = [...APP_CRITERIA, ...REPO_CRITERIA]; - -interface AppInfo { - description: string; - path: string; +function repositoryState(root: string): string { + return git(["status", "--porcelain=v1", "--untracked-files=all"], root); } -function walk(dir: string, maxDepth = 3, depth = 0): string[] { - if (depth > maxDepth || !existsSync(dir)) return []; - const out: string[] = []; - for (const entry of readdirSync(dir, { withFileTypes: true })) { - if ( - [ - ".git", - "node_modules", - ".next", - "dist", - "build", - ".venv", - "venv", - ].includes(entry.name) - ) - continue; - const path = join(dir, entry.name); - out.push(path); - if (entry.isDirectory()) out.push(...walk(path, maxDepth, depth + 1)); - } - return out; +function repoUrl(root: string): string { + return git(["config", "--get", "remote.origin.url"], root) || root; } -function hasAny(root: string, patterns: RegExp[]) { - return walk(root).some((path) => - patterns.some((pattern) => pattern.test(relative(root, path))), +function repoName(url: string): string { + return ( + url + .replace(/\.git$/, "") + .split(/[/:]/) + .filter(Boolean) + .at(-1) ?? "repository" ); } -function fileText(path: string) { - try { - return readFileSync(path, "utf8"); - } catch { - return ""; - } +export interface ReadinessRunResult { + exitCode: number; + message: string; } -function detectApps(root: string): Record { - const apps: Record = {}; - for (const path of walk(root, 2)) { - if ( - !existsSync(join(path, "package.json")) && - !existsSync(join(path, "requirements.txt")) && - !existsSync(join(path, "pyproject.toml")) - ) - continue; - const rel = relative(root, path) || "."; - const pkg = fileText(join(path, "package.json")); - const name = pkg ? (JSON.parse(pkg).name ?? rel) : rel; - apps[rel] = { path, description: `${name} application` }; - } - if (Object.keys(apps).length === 0) { - apps["."] = { path: root, description: "Repository root application" }; - } - return apps; +export type ReadinessProgress = (message: string) => void; +export interface ReadinessOptions { + model?: string; } -function scoreApps( - apps: Record, - check: (path: string) => boolean, - label: string, -): Criterion { - const entries = Object.entries(apps); - const passed = entries.filter(([, app]) => check(app.path)); - return { - numerator: passed.length, - denominator: entries.length, - rationale: `${passed.length}/${entries.length} apps pass ${label}.`, - }; +export function gatewayToolForReadiness( + agent: ReadinessAgent, +): "claude-code" | "opencode" | undefined { + if (agent === "claude") return "claude-code"; + if (agent === "opencode") return "opencode"; + return undefined; } -function criterion(numerator: number | null, rationale: string): Criterion { - return { numerator, denominator: 1, rationale }; -} +export async function runReadiness( + agent: ReadinessAgent, + onProgress: ReadinessProgress = () => {}, + options: ReadinessOptions = {}, +): Promise { + const root = process.cwd(); + if (!git(["rev-parse", "--is-inside-work-tree"], root)) { + return { + exitCode: 1, + message: "Readiness must be run inside a git repository.", + }; + } -function git(args: string[], cwd: string) { - const proc = spawnSync("git", args, { cwd, encoding: "utf8" }); - return proc.status === 0 ? proc.stdout.trim() : ""; -} + try { + assertReadinessPrerequisites(agent); + } catch (error) { + return { + exitCode: 1, + message: error instanceof Error ? error.message : String(error), + }; + } -function repoUrl(root: string) { - return git(["config", "--get", "remote.origin.url"], root) || root; -} + const before = repositoryState(root); + onProgress(`Scanning repository with ${agent}, this will take a while`); + let run: AgentRunResult; + try { + const gatewayTool = gatewayToolForReadiness(agent); + if (gatewayTool) await ensureFreshGatewayKey(gatewayTool); + run = await runReadinessAgent( + agent, + root, + undefined, + options.model, + onProgress, + ); + run = { ...run, output: normalizeReadinessEvidence(run.output, root) }; + let totalDurationMs = run.durationMs; + let errors = validateReadinessOutput(run.output, root); + const { maxRepairs } = readinessRuntimeConfig(); + for ( + let attempt = 1; + errors.length > 0 && attempt <= maxRepairs; + attempt++ + ) { + onProgress( + `Repairing invalid report (${attempt}/${maxRepairs}, ${errors.length} validation errors)`, + ); + run = await runReadinessAgent( + agent, + root, + { + raw: run.raw, + errors, + sessionId: run.sessionId, + }, + options.model, + onProgress, + ); + run = { ...run, output: normalizeReadinessEvidence(run.output, root) }; + totalDurationMs += run.durationMs; + errors = validateReadinessOutput(run.output, root); + } + if (errors.length > 0) + throw new Error( + `Report is still invalid after ${maxRepairs} repair attempt${maxRepairs === 1 ? "" : "s"}:\n${errors.map((error) => `- ${error}`).join("\n")}`, + ); + run = { ...run, durationMs: totalDurationMs }; + } catch (error) { + return { + exitCode: 1, + message: error instanceof Error ? error.message : String(error), + }; + } -function buildReport(root: string, apps: Record): Report { + const after = repositoryState(root); + if (after !== before) { + return { + exitCode: 1, + message: + "The agent changed the repository working tree. The report was rejected and not uploaded; review those changes manually.", + }; + } + + const summary = summarizeReadiness(run.output.criteria); + const runtimeConfig = readinessRuntimeConfig(options.model); + const configuredModel = + agent === "opencode" + ? runtimeConfig.opencodeModel + : agent === "codex" + ? runtimeConfig.model + : undefined; + const url = repoUrl(root); const report = Object.fromEntries( - ALL_CRITERIA.map((id) => [ + Object.entries(run.output.criteria).map(([id, item]) => [ id, { - numerator: 0, - denominator: 1, - rationale: "No matching evidence found.", + status: item.status, + numerator: item.numerator, + denominator: item.denominator, + rationale: item.rationale, + evidence: item.evidence, }, ]), - ) as Report; - - report.lint_config = scoreApps( - apps, - (p) => - hasAny(p, [ - /eslint\.config\./, - /\.eslintrc/, - /biome\.json/, - /ruff\.toml/, - /pyproject\.toml/, - ]), - "linter config", - ); - report.type_check = scoreApps( - apps, - (p) => - hasAny(p, [ - /tsconfig\.json/, - /pyrightconfig\.json/, - /mypy\.ini/, - /pyproject\.toml/, - ]), - "type checking", - ); - report.formatter = scoreApps( - apps, - (p) => - hasAny(p, [/prettier/, /biome\.json/, /ruff\.toml/, /pyproject\.toml/]), - "formatter config", - ); - report.pre_commit_hooks = scoreApps( - apps, - (p) => hasAny(p, [/^\.husky\//, /\.pre-commit-config\.yaml/]), - "pre-commit hooks", - ); - report.unit_tests_exist = scoreApps( - apps, - (p) => hasAny(p, [/\.test\./, /\.spec\./, /^tests\//, /test_.*\.py$/]), - "unit tests", ); - report.unit_tests_runnable = scoreApps( - apps, - (p) => - fileText(join(p, "package.json")).includes('"test"') || - hasAny(p, [/pytest\.ini/, /pyproject\.toml/]), - "test command", - ); - report.database_schema = scoreApps( - apps, - (p) => hasAny(p, [/migrations\//, /\.sql$/]), - "database schema files", - ); - report.health_checks = scoreApps( - apps, - (p) => /\/health|healthcheck/i.test(walk(p, 2).map(fileText).join("\n")), - "health checks", - ); - report.structured_logging = scoreApps( - apps, - (p) => - /pino|winston|logrus|logging\.|structlog|logger/i.test( - walk(p, 2).map(fileText).join("\n"), - ), - "structured logging", - ); - - report.agents_md = criterion( - existsSync(join(root, "AGENTS.md")) ? 1 : 0, - existsSync(join(root, "AGENTS.md")) - ? "AGENTS.md exists." - : "AGENTS.md not found.", - ); - report.readme = criterion( - existsSync(join(root, "README.md")) ? 1 : 0, - existsSync(join(root, "README.md")) - ? "README.md exists." - : "README.md not found.", - ); - report.deps_pinned = criterion( - hasAny(root, [ - /package-lock\.json/, - /pnpm-lock\.yaml/, - /bun\.lock/, - /go\.sum/, - /uv\.lock/, - /poetry\.lock/, - ]) - ? 1 - : 0, - "Dependency lockfile scan completed.", - ); - report.env_template = criterion( - hasAny(root, [/\.env\.example$/, /\.env\.template$/]) ? 1 : 0, - "Environment template scan completed.", - ); - report.gitignore_comprehensive = criterion( - fileText(join(root, ".gitignore")).includes(".env") ? 1 : 0, - ".gitignore secret pattern scan completed.", - ); - report.devcontainer = criterion( - hasAny(root, [/^\.devcontainer\/devcontainer\.json$/]) ? 1 : 0, - "Devcontainer scan completed.", - ); - report.issue_templates = criterion( - hasAny(root, [/^\.github\/ISSUE_TEMPLATE\//]) ? 1 : 0, - "Issue template scan completed.", - ); - report.pr_templates = criterion( - hasAny(root, [/pull_request_template/i]) ? 1 : 0, - "PR template scan completed.", - ); - report.codeowners = criterion( - hasAny(root, [/(^|\/)CODEOWNERS$/]) ? 1 : 0, - "CODEOWNERS scan completed.", - ); - report.release_automation = criterion( - hasAny(root, [ - /^\.github\/workflows\/.*release/i, - /^\.github\/workflows\/.*deploy/i, - ]) - ? 1 - : 0, - "Release workflow scan completed.", - ); - report.agentic_development = criterion( - /factory-droid|droid|claude|codex/i.test( - git(["log", "--oneline", "-50"], root), - ) - ? 1 - : 0, - "Recent commit authorship scan completed.", - ); - report.skills = criterion( - hasAny(root, [ - /(^|\/)\.factory\/skills\//, - /(^|\/)\.agents\/skills\//, - /(^|\/)\.claude\/skills\//, - ]) - ? 1 - : 0, - "Agent skill directory scan completed.", - ); - - for (const id of ALL_CRITERIA) { - const item = report[id]; - if ( - item?.rationale === "No matching evidence found." && - APP_CRITERIA.includes(id as (typeof APP_CRITERIA)[number]) - ) { - report[id] = { - numerator: null, - denominator: Object.keys(apps).length, - rationale: "Not evaluated by the deterministic MVP analyzer.", - }; - } - } - return report; -} - -export async function runReadiness() { - const root = process.cwd(); - const apps = detectApps(root); - const report = buildReport(root, apps); const payload = { - repoUrl: repoUrl(root), + repoUrl: url, + repoName: repoName(url), branch: git(["branch", "--show-current"], root), commitHash: git(["rev-parse", "HEAD"], root), + rubricVersion: READINESS_RUBRIC_VERSION, + languages: run.output.languages, apps: Object.fromEntries( - Object.entries(apps).map(([key, app]) => [ - key, - { description: app.description }, + run.output.applications.map((app) => [ + app.path, + { description: app.description, languages: app.languages }, ]), ), - modelUsed: { id: "codev-deterministic-readiness" }, report, + warnings: run.output.warnings, + recommendations: run.output.recommendations, + modelUsed: { + provider: agent, + model: run.output.model ?? configuredModel ?? null, + durationMs: run.durationMs, + }, + // Included for human-readable proxy logs only. The server must recompute these. + summary, }; - const auth = await login(console.error, () => {}); - const res = await fetch(`${BACKEND_URL}/readiness/reports`, { - method: "POST", - headers: { - "content-type": "application/json", - authorization: `Bearer ${auth.access_token}`, - }, - body: JSON.stringify(payload), - }); - if (!res.ok) { - console.error( - `Readiness upload failed (${res.status}): ${await res.text()}`, + onProgress(`Uploading validated Level ${summary.level} report`); + let authError = ""; + let auth: AuthData; + try { + auth = await login( + (message) => { + authError = message; + }, + () => {}, ); - return 1; + } catch (error) { + return { + exitCode: 1, + message: + authError || (error instanceof Error ? error.message : String(error)), + }; + } + let response: Response; + try { + response = await fetch(`${BACKEND_URL}/readiness/reports`, { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Bearer ${auth.access_token}`, + }, + body: JSON.stringify(payload), + }); + } catch (error) { + return { + exitCode: 1, + message: `Readiness upload failed: ${error instanceof Error ? error.message : String(error)}`, + }; + } + if (!response.ok) { + return { + exitCode: 1, + message: `Readiness upload failed (${response.status}): ${await response.text()}`, + }; } - const data = (await res.json()) as { report?: { id?: string } }; - console.log(`Stored readiness report ${data.report?.id ?? ""}`); - return 0; + const data = (await response.json()) as { report?: { id?: string } }; + return { + exitCode: 0, + message: `Stored report ${data.report?.id ?? ""} · Level ${summary.level} · ${summary.passRate}% pass · ${summary.criteriaTotal} criteria evaluated`, + }; } diff --git a/tests/ReadinessApp.test.tsx b/tests/ReadinessApp.test.tsx new file mode 100644 index 0000000..4d68cca --- /dev/null +++ b/tests/ReadinessApp.test.tsx @@ -0,0 +1,40 @@ +import { render } from "ink-testing-library"; +import { describe, expect, it, vi } from "vitest"; +import { ReadinessApp } from "@/ReadinessApp.js"; + +async function settle(ms = 30) { + await new Promise((resolve) => setTimeout(resolve, ms)); +} + +describe("ReadinessApp", () => { + it("uses the shared Ink wizard style and runs the selected agent", async () => { + const run = vi.fn(async (_agent, progress?: (message: string) => void) => { + progress?.("Validating report"); + return { exitCode: 0, message: "Stored report report-1" }; + }); + const { frames, stdin } = render( + , + ); + expect(frames.at(-1)).toContain("AGENT READINESS"); + expect(frames.at(-1)).toContain("OpenCode"); + expect(frames.at(-1)).toContain("(unavailable)"); + + stdin.write("\r"); + await settle(100); + expect(run).toHaveBeenCalledWith("opencode", expect.any(Function), {}); + expect(frames.join("\n")).toContain("Stored report report-1"); + }); + + it("explains when no supported agent is installed", () => { + const { lastFrame } = render( + , + ); + expect(lastFrame()).toContain("No supported coding agent is available"); + }); +}); diff --git a/tests/lib/readiness-contract.test.ts b/tests/lib/readiness-contract.test.ts new file mode 100644 index 0000000..f9575fe --- /dev/null +++ b/tests/lib/readiness-contract.test.ts @@ -0,0 +1,288 @@ +import { homedir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it, vi } from "vitest"; +import * as configure from "@/lib/configure.js"; +import { gatewayToolForReadiness } from "@/lib/readiness.js"; +import { + activityFromLine, + assertReadinessPrerequisites, + buildAgentCommand, + buildReadinessPrompt, + claudeReadinessEnvOverrides, + extractAgentOutput, + openCodeStructuredOutputInstruction, + providerFailureFromLine, + readinessProcessEnv, +} from "@/lib/readiness-agent.js"; +import { readinessRuntimeConfig } from "@/lib/readiness-config.js"; +import { + type AgentReadinessOutput, + normalizeReadinessEvidence, + READINESS_CRITERION_IDS, + READINESS_RUBRIC_VERSION, + readinessJsonSchema, + summarizeReadiness, + validateReadinessOutput, +} from "@/lib/readiness-contract.js"; + +function validOutput(): AgentReadinessOutput { + return { + rubricVersion: READINESS_RUBRIC_VERSION, + languages: ["TypeScript"], + applications: [ + { path: ".", description: "CoDev CLI", languages: ["TypeScript"] }, + ], + criteria: Object.fromEntries( + READINESS_CRITERION_IDS.map((id) => [ + id, + { + status: "skipped", + numerator: null, + denominator: 1, + rationale: "Not established from this fixture.", + evidence: [], + }, + ]), + ), + warnings: [], + recommendations: ["Add a validated check.", "Document the workflow."], + model: null, + }; +} + +describe("readiness contract", () => { + it("accepts a complete versioned report", () => { + expect(validateReadinessOutput(validOutput(), process.cwd())).toEqual([]); + }); + + it("rejects missing criteria and evidence outside the repository", () => { + const output = validOutput(); + delete output.criteria[READINESS_CRITERION_IDS[0] ?? ""]; + const second = READINESS_CRITERION_IDS[1] ?? ""; + output.criteria[second] = { + status: "pass", + numerator: 1, + denominator: 1, + rationale: "Found.", + evidence: ["../outside.txt"], + }; + const errors = validateReadinessOutput(output, process.cwd()); + expect(errors.some((error) => error.includes("Missing criterion"))).toBe( + true, + ); + expect(errors.some((error) => error.includes("missing evidence"))).toBe( + true, + ); + }); + + it("keeps existing directory evidence and removes invalid evidence locally", () => { + const output = validOutput(); + const id = READINESS_CRITERION_IDS[0] ?? ""; + output.criteria[id] = { + status: "fail", + numerator: 0, + denominator: 1, + rationale: "No matching automation was found.", + evidence: ["src/", ".missing-readiness-directory/", "../outside.txt"], + }; + + const normalized = normalizeReadinessEvidence(output, process.cwd()); + + expect(normalized.criteria[id]?.evidence).toEqual(["src/"]); + expect(output.criteria[id]?.evidence).toHaveLength(3); + expect(validateReadinessOutput(normalized, process.cwd())).toEqual([]); + }); + + it("excludes skipped criteria from deterministic scoring", () => { + const criteria = validOutput().criteria; + criteria[READINESS_CRITERION_IDS[0] ?? ""] = { + status: "pass", + numerator: 1, + denominator: 1, + rationale: "Pass", + evidence: ["package.json"], + }; + criteria[READINESS_CRITERION_IDS[1] ?? ""] = { + status: "fail", + numerator: 0, + denominator: 1, + rationale: "Fail", + evidence: [], + }; + expect(summarizeReadiness(criteria)).toEqual({ + criteriaPassed: 1, + criteriaTotal: 2, + passRate: 50, + level: 3, + }); + }); + + it("publishes a closed schema and embeds the rubric in the prompt", () => { + const schema = readinessJsonSchema(); + expect(schema.additionalProperties).toBe(false); + const properties = schema.properties as Record< + string, + Record + >; + expect(properties.rubricVersion?.type).toBe("string"); + expect(buildReadinessPrompt()).toContain(READINESS_RUBRIC_VERSION); + expect(buildReadinessPrompt()).toContain("Do not edit"); + }); + + it("builds read-only headless commands for every supported provider", () => { + const schema = "package.json"; + const output = "/tmp/codev-readiness-test-output.json"; + const claude = buildAgentCommand( + "claude", + "scan", + schema, + output, + undefined, + "test-model", + ); + const codex = buildAgentCommand( + "codex", + "scan", + schema, + output, + undefined, + "test-model", + ); + const openCode = buildAgentCommand( + "opencode", + "scan", + schema, + output, + undefined, + "test-model", + ); + expect(claude.args).toContain("--permission-mode"); + expect(claude.args).toContain("plan"); + expect(codex.args).toContain("read-only"); + expect(codex.args).toContain("--output-schema"); + expect(openCode.args).toContain("-m"); + expect(openCode.args).toContain("aigateway/MiniMax/MiniMax-M2.7"); + expect(claude.args).toContain("test-model"); + expect(codex.args).toContain("test-model"); + expect(openCode.args).toContain("json"); + expect(openCode.args.at(-1)).toContain(READINESS_RUBRIC_VERSION); + expect(openCode.args.at(-1)).toContain('"criteria"'); + expect(claude.args).not.toContain("--max-budget-usd"); + expect( + buildAgentCommand("claude", "scan", schema, output).args, + ).not.toContain("--model"); + expect( + buildAgentCommand("codex", "scan", schema, output).args, + ).not.toContain("--model"); + }); + + it("gives OpenCode the structured contract that its CLI cannot accept as a schema flag", () => { + const instruction = openCodeStructuredOutputInstruction(); + expect(instruction).toContain('"rubricVersion"'); + expect(instruction).toContain('"status":"pass|fail|skipped"'); + expect(instruction).toContain("every rubric id exactly once"); + }); + + it("removes CoDev shims from readiness subprocess PATH resolution", () => { + const env = readinessProcessEnv( + {}, + { + ...process.env, + PATH: [ + join(homedir(), ".codev", "bin"), + "/opt/homebrew/bin", + "/usr/bin", + ].join(":"), + }, + ); + + expect(env.PATH).toBe("/opt/homebrew/bin:/usr/bin"); + }); + + it("isolates Claude readiness from ambient provider credentials", () => { + const env = readinessProcessEnv(claudeReadinessEnvOverrides(), { + ...process.env, + ANTHROPIC_BASE_URL: "http://unrelated-provider.invalid", + ANTHROPIC_AUTH_TOKEN: "unrelated-token", + }); + + expect(env.ANTHROPIC_BASE_URL).toBeUndefined(); + expect(env.ANTHROPIC_AUTH_TOKEN).toBeUndefined(); + }); + + it("requires the selected harness to have been configured by codev install", () => { + const detect = vi + .spyOn(configure, "detectConfiguredTools") + .mockReturnValue(["claude-code"]); + try { + expect(() => assertReadinessPrerequisites("opencode")).toThrow( + /codev install/, + ); + expect(() => assertReadinessPrerequisites("claude")).not.toThrow(); + // Codex may use its own ChatGPT subscription authentication without a + // CoDev-managed internal-gateway configuration. + expect(() => assertReadinessPrerequisites("codex")).not.toThrow(); + } finally { + detect.mockRestore(); + } + }); + + it("refreshes only readiness agents backed by the internal gateway", () => { + expect(gatewayToolForReadiness("claude")).toBe("claude-code"); + expect(gatewayToolForReadiness("opencode")).toBe("opencode"); + expect(gatewayToolForReadiness("codex")).toBeUndefined(); + }); + + it("uses an explicit per-run model without making the rubric configurable", () => { + expect(readinessRuntimeConfig("test-model").opencodeModel).toBe( + "aigateway/MiniMax/MiniMax-M2.7", + ); + expect(readinessRuntimeConfig("test-model").maxRepairs).toBe(2); + }); + + it("turns provider JSON events into safe progress messages", () => { + expect( + activityFromLine( + JSON.stringify({ type: "tool_use", part: { tool: "glob" } }), + ), + ).toBe("Inspecting repository with glob"); + expect( + activityFromLine( + JSON.stringify({ + type: "assistant", + message: { content: [{ type: "tool_use", name: "Read" }] }, + }), + ), + ).toBe("Inspecting repository with Read"); + expect(activityFromLine("not json")).toBeUndefined(); + expect( + providerFailureFromLine( + JSON.stringify({ + type: "system", + subtype: "api_retry", + error: "authentication_failed", + }), + ), + ).toContain("codev login"); + }); + + it("extracts structured results from Claude, Codex, and OpenCode envelopes", () => { + const output = validOutput(); + const encoded = JSON.stringify(output); + expect( + extractAgentOutput( + JSON.stringify({ type: "result", structured_output: output }), + ), + ).toEqual(output); + expect( + extractAgentOutput( + `${JSON.stringify({ type: "thread.started", thread_id: "abc" })}\n${JSON.stringify({ type: "item.completed", text: encoded })}`, + ), + ).toEqual(output); + expect( + extractAgentOutput( + JSON.stringify({ type: "text", part: { text: encoded } }), + ), + ).toEqual(output); + }); +}); From 5352f9038a15ddcedaed22c67dd17699588be1ed Mon Sep 17 00:00:00 2001 From: Bao Tran Date: Wed, 15 Jul 2026 15:07:31 +0700 Subject: [PATCH 03/14] feat: stabilize readiness evaluation --- docs/readiness-evaluation.md | 51 ++ src/lib/readiness-agent.ts | 37 +- src/lib/readiness-contract.ts | 182 ++++++- src/lib/readiness-plan.ts | 721 +++++++++++++++++++++++++++ src/lib/readiness.ts | 26 +- tests/lib/readiness-contract.test.ts | 9 +- tests/lib/readiness-plan.test.ts | 300 +++++++++++ 7 files changed, 1306 insertions(+), 20 deletions(-) create mode 100644 docs/readiness-evaluation.md create mode 100644 src/lib/readiness-plan.ts create mode 100644 tests/lib/readiness-plan.test.ts diff --git a/docs/readiness-evaluation.md b/docs/readiness-evaluation.md new file mode 100644 index 0000000..68467b1 --- /dev/null +++ b/docs/readiness-evaluation.md @@ -0,0 +1,51 @@ +# Readiness evaluation design + +## Goal + +Produce a fresh, evidence-backed readiness report whose applicability and evaluated-criterion count are stable across Claude Code, Codex, and OpenCode. The selected agent should contribute semantic judgment, not decide which parts of the rubric count toward the score. + +## Invariants + +- Every run rebuilds the repository profile from the current working tree. This release does not cache by commit or file hash. +- Every canonical criterion receives exactly one decision: deterministic result, rule-backed not applicable, or semantic evaluation. +- Only CoDev may mark a criterion not applicable. A semantic evaluator must return pass or fail. +- Every evaluated criterion contributes one equally weighted repository-level vote. Provider-specific application discovery cannot change the denominator. +- Deterministic checks accept documented ecosystem variants rather than one preferred filename. +- Absence is a failure only when a criterion is universally expected or applicability is positively established. + +## Evaluation boundary + +Deterministic presence checks are limited to criteria whose requirement is the artifact itself: + +- coding-agent instructions: `AGENTS.md` or `CLAUDE.md` +- root README +- recognized CODEOWNERS locations +- GitHub/GitLab issue and pull/merge-request templates +- dev-container definitions +- repository-local agent skill directories +- meaningful version-controlled pre-commit hooks +- automation that validates agent-instruction files +- documented VCS/hosting CLI commands +- configured dead-code, complexity, clone, and unused-dependency tooling +- repository-owned setup/bootstrap tasks +- version-controlled branch-protection policy-as-code +- ecosystem-aware `.gitignore` coverage +- documentation with explicit architecture or component-flow signals + +Environment templates use both applicability and variant detection. `.env.example`, `.env.local.example`, `.env.sample`, `.env.template`, `.env.dist`, `env.example`, `example.env`, and `sample.env` are recognized. A missing template fails only when source or configuration evidence shows environment-variable consumption; otherwise the criterion is not applicable. + +Database-only and API-only checks are not applicable only when the fresh profile finds no corresponding surface. The signals intentionally combine directory conventions, manifests, dependencies, and representative source/configuration content. Positive detection delegates the actual quality judgment to the semantic evaluator. + +All remaining criteria are semantic. The agent receives the deterministic profile, a bounded list of relevant files, explicit rubric definitions, and a small command budget. CoDev normalizes any semantic skip or omission to a conservative failure and records a warning. + +## Deliberate first-release constraints + +- No persistent or cross-run cache. +- No network or hosted-repository metadata checks; branch protection and backlog health are judged only from local evidence and should fail when they cannot be established. +- Component discovery is manifest-based and deterministic. It is report metadata only and does not alter criterion weighting. +- Deterministic rules favor high-confidence applicability. Ambiguous criteria remain semantic rather than being skipped heuristically. + +## Validation + +- Fixture tests cover environment-template variants, repositories without environment requirements, GitHub/GitLab convention variants, API/database applicability, and denominator parity when an agent skips everything. +- End-to-end parity runs must use the same working tree and rubric version across all three providers. The evaluated count must match exactly; score differences are then semantic disagreements that can be measured and refined. diff --git a/src/lib/readiness-agent.ts b/src/lib/readiness-agent.ts index 4563173..299b2a4 100644 --- a/src/lib/readiness-agent.ts +++ b/src/lib/readiness-agent.ts @@ -18,6 +18,11 @@ import { READINESS_RUBRIC_VERSION, readinessJsonSchema, } from "@/lib/readiness-contract.js"; +import { + type ReadinessEvaluationPlan, + readinessPlanPrompt, + semanticCriterionIds, +} from "@/lib/readiness-plan.js"; import { stripShimDirFromPath } from "@/lib/shims.js"; export const READINESS_AGENTS = ["claude", "codex", "opencode"] as const; @@ -125,26 +130,32 @@ export function claudeReadinessEnvOverrides(): NodeJS.ProcessEnv { }; } -export function buildReadinessPrompt(): string { +export function buildReadinessPrompt(plan?: ReadinessEvaluationPlan): string { + const semanticIds = plan ? new Set(semanticCriterionIds(plan)) : undefined; + const rubric = semanticIds + ? READINESS_RUBRIC.filter(({ id }) => semanticIds.has(id)) + : READINESS_RUBRIC; return `You are evaluating how ready a software repository is for autonomous coding agents. Work read-only. Do not edit, create, delete, format, commit, install dependencies, access the network, or spawn subagents. You may inspect repository files and git history and run safe, non-mutating diagnostic commands. Judge semantic evidence across languages and frameworks; do not rely only on filenames. -Finish efficiently. First inventory the repository with one batched file-listing command. Prefer manifests, CI configuration, test configuration, documentation, security policy, deployment files, and a small representative source sample. Batch related reads/searches and reuse the same evidence across criteria. Do not inspect every source file or run expensive builds/tests. Aim for no more than 12 repository commands before producing the report. +Finish efficiently. CoDev has already performed a fresh deterministic inventory for this run. Start from the supplied profile and relevant-file list instead of rediscovering the repository. Inspect only the files needed to judge the semantic criteria, batch related reads/searches, and reuse evidence. Do not inspect every source file or run expensive builds/tests. Aim for no more than 6 repository commands before producing the report. -Discover independently deployable applications first. Evaluate every rubric criterion. Use "skipped" only when it is genuinely inapplicable or cannot be established from local evidence. A failure means the criterion applies but adequate support is absent. For application-scoped criteria, numerator and denominator count evaluated applications. Repository-scoped criteria use denominator 1. Evidence entries must be existing repository-relative file or directory paths; file paths may optionally be followed by a line number. Use an empty evidence array when the rationale describes something that is absent. Never cite a nonexistent placeholder, URL, or command as evidence; describe command evidence in the rationale instead. +Evaluate every semantic rubric criterion below. Semantic criteria must be pass or fail, never skipped: failure means the repository does not provide enough evidence. Score each criterion once for the repository with denominator 1 and numerator 1 for pass or 0 for fail. Return only semantic criterion IDs in the criteria object; omit fixed decisions because CoDev merges them authoritatively. Evidence entries must be existing repository-relative file or directory paths; file paths may optionally be followed by a line number. Use an empty evidence array when the rationale describes something that is absent. Never cite a nonexistent placeholder, URL, or command as evidence; describe command evidence in the rationale instead. Return only the JSON object matching the supplied schema. Do not include aggregate scores. Rubric version: ${READINESS_RUBRIC_VERSION} -Rubric: -${JSON.stringify(READINESS_RUBRIC, null, 2)}`; +Fresh deterministic plan: +${plan ? readinessPlanPrompt(plan) : "No deterministic plan supplied."} +Semantic rubric: +${JSON.stringify(rubric, null, 2)}`; } export function openCodeStructuredOutputInstruction(): string { return `Return exactly one JSON object with this shape and no markdown fence: -{"rubricVersion":"${READINESS_RUBRIC_VERSION}","languages":["string"],"applications":[{"path":".","description":"string","languages":["string"]}],"criteria":{"":{"status":"pass|fail|skipped","numerator":"integer or null","denominator":"positive integer","rationale":"string","evidence":["existing repository-relative path"]}},"warnings":["string"],"recommendations":["2 or 3 strings"],"model":"string or null"} -Use null numerator only for skipped criteria. Include every rubric id exactly once and no additional criterion ids.`; +{"rubricVersion":"${READINESS_RUBRIC_VERSION}","languages":["string"],"applications":[{"path":".","description":"string","languages":["string"]}],"criteria":{"":{"status":"pass|fail|skipped","numerator":"integer or null","denominator":"positive integer","rationale":"string","evidence":["existing repository-relative path"]}},"warnings":["string"],"recommendations":["2 or 3 strings"],"model":"string or null"} + Use null numerator only for skipped criteria. Include every criterion listed in the Semantic rubric exactly once and omit fixed-decision criteria.`; } export function buildAgentCommand( @@ -167,7 +178,7 @@ export function buildAgentCommand( "--permission-mode", "plan", "--max-turns", - "80", + "20", "--allowedTools", "Read", "Glob", @@ -381,11 +392,17 @@ export async function runReadinessAgent( repair?: { raw: string; errors: string[]; sessionId?: string }, modelOverride?: string, onProgress: AgentProgress = () => {}, + plan?: ReadinessEvaluationPlan, ): Promise { const temp = mkdtempSync(join(tmpdir(), "codev-readiness-")); const schemaPath = join(temp, "schema.json"); const outputPath = join(temp, "output.json"); - writeFileSync(schemaPath, JSON.stringify(readinessJsonSchema())); + writeFileSync( + schemaPath, + JSON.stringify( + readinessJsonSchema(plan ? semanticCriterionIds(plan) : undefined), + ), + ); let envOverrides: NodeJS.ProcessEnv = {}; if (agent === "claude") envOverrides = claudeReadinessEnvOverrides(); if (agent === "opencode") { @@ -426,7 +443,7 @@ export async function runReadinessAgent( } const prompt = repair ? `Your previous readiness output was invalid. Correct it and return the complete JSON object only. Do not rescan or change the repository. Remove nonexistent evidence entries and use an empty evidence array when the rationale describes absence. Existing repository-relative directories are valid evidence. Validation errors:\n${repair.errors.map((error) => `- ${error}`).join("\n")}\nPrevious output:\n${repair.raw.slice(-200_000)}` - : buildReadinessPrompt(); + : buildReadinessPrompt(plan); const started = Date.now(); const deadline = started + readinessRuntimeConfig(modelOverride).timeoutMs; try { diff --git a/src/lib/readiness-contract.ts b/src/lib/readiness-contract.ts index 744524c..42fd4b7 100644 --- a/src/lib/readiness-contract.ts +++ b/src/lib/readiness-contract.ts @@ -5,7 +5,7 @@ import { isAbsolute, relative, resolve } from "node:path"; // only when READINESS_RUBRIC or the result contract changes, and update the // proxy/web supported-version constants in the same change. An environment // override could falsely label one rubric as another and corrupt comparisons. -export const READINESS_RUBRIC_VERSION = "2026-07-14.v1"; +export const READINESS_RUBRIC_VERSION = "2026-07-15.v2"; export type ReadinessStatus = "pass" | "fail" | "skipped"; @@ -182,15 +182,183 @@ function label(id: string): string { return id.replaceAll("_", " "); } +const CRITERION_PASS_CONDITIONS: Record = { + lint_config: + "Pass only when the repository configures a language-appropriate linter and exposes a usable lint command.", + type_check: + "Pass only when typed code has a configured, runnable static type-check command; pass untyped-language repositories only when an equivalent static analyzer is configured.", + formatter: + "Pass only when an automatic formatter and its repository configuration or command are present.", + pre_commit_hooks: + "Pass only when version-controlled pre-commit hooks run meaningful validation before commits.", + strict_typing: + "Pass only when the primary typed languages enable strict type checking or comparably strong settings.", + naming_consistency: + "Pass only when representative source files and enforced conventions show consistent language-idiomatic naming.", + cyclomatic_complexity: + "Pass only when tooling measures or limits code complexity in normal development or CI.", + large_file_detection: + "Pass only when automation detects or blocks oversized source files or generated/binary additions.", + dead_code_detection: + "Pass only when configured tooling detects unused or unreachable code.", + duplicate_code_detection: + "Pass only when configured clone-detection tooling detects repeated code blocks; duplicate object-key lint rules do not count.", + code_modularization: + "Pass only when representative source structure has cohesive modules with clear boundaries and avoids dominant god modules.", + tech_debt_tracking: + "Pass only when the repository has an actionable, maintained mechanism for tracking technical debt.", + n_plus_one_detection: + "Pass only when database access has automated N+1 query detection, query-count assertions, or equivalent safeguards.", + build_cmd_doc: + "Pass only when a new contributor can find an explicit build command in version-controlled documentation.", + deps_pinned: + "Pass only when dependency manifests and lockfiles provide reproducible direct and transitive dependency resolution.", + vcs_cli_tools: + "Pass only when documentation names actual version-control or hosting CLI commands for contributor tasks; CI configuration or generic Git usage alone does not pass.", + automated_pr_review: + "Pass only when pull or merge requests receive automatic code-quality review beyond basic compilation.", + agentic_development: + "Pass only when repository instructions, tools, or workflows explicitly support coding agents with safe, actionable context.", + fast_ci_feedback: + "Pass only when the complete CI workflow set gives early feedback through independently running focused jobs, parallel jobs, or explicit stages; inspect all workflow files before deciding.", + build_performance_tracking: + "Pass only when build duration or regressions are measured over time.", + deployment_frequency: + "Pass only when repository automation records or exposes deployment frequency rather than merely supporting deployment.", + single_command_setup: + "Pass only when a documented single command or task bootstraps the local development environment.", + feature_flag_infrastructure: + "Pass only when application code uses a centralized, testable feature-flag mechanism.", + release_notes_automation: + "Pass only when release notes or changelogs are generated or validated automatically from version-controlled inputs.", + progressive_rollout: + "Pass only when deployment configuration supports staged, canary, percentage, or cohort-based rollout.", + rollback_automation: + "Pass only when a documented automated rollback path exists and identifies the artifact or revision restored.", + monorepo_tooling: + "Pass only when a multi-package repository uses workspace-aware orchestration; pass a single-package repository when no monorepo coordination is needed.", + heavy_dependency_detection: + "Pass only when dependency size, startup cost, or bundle impact is measured or constrained.", + unused_dependencies_detection: + "Pass only when tooling detects unused declared package dependencies; unused imports, variables, or declarations alone do not pass.", + version_drift_detection: + "Pass only when automation detects inconsistent tool or dependency versions across repository components.", + release_automation: + "Pass only when versioning, artifact creation, and publishing are automated through version-controlled workflows.", + dead_feature_flag_detection: + "Pass only when stale feature flags are inventoried, expired, or automatically detected.", + unit_tests_exist: + "Pass only when meaningful unit tests are present for production logic, not merely placeholder tests.", + integration_tests_exist: + "Pass only when tests exercise multiple real internal components together or a real service/database boundary; external third parties may be mocked, but a single isolated unit with all collaborators mocked does not count.", + unit_tests_runnable: + "Pass only when a documented or manifest-defined unit-test command can run without undocumented manual preparation.", + test_performance_tracking: + "Pass only when test duration or slow-test regressions are measured or reported.", + flaky_test_detection: + "Pass only when CI detects, retries with reporting, quarantines, or tracks flaky tests.", + test_coverage_thresholds: + "Pass only when coverage thresholds are configured and enforced rather than coverage being generated without a gate.", + test_naming_conventions: + "Pass only when test locations and names follow a consistent discoverable convention across primary languages.", + test_isolation: + "Pass only when test configuration or fixtures prevent shared mutable state and order dependence.", + agents_md: + "Pass only when AGENTS.md or CLAUDE.md provides repository-specific instructions for coding agents.", + readme: + "Pass only when a repository-root README explains purpose and a usable development entry point.", + automated_doc_generation: + "Pass only when reference documentation is generated or checked automatically from source-controlled definitions.", + skills: + "Pass only when reusable repository-local coding-agent skills are version controlled in a recognized skills directory.", + documentation_freshness: + "Pass only when automation or an explicit maintained process detects stale documentation.", + api_schema_docs: + "Pass only when public APIs have a maintained machine-readable schema or generated reference tied to implementation.", + service_flow_documented: + "Pass only when important service or request flows are documented with current component interactions.", + agents_md_validation: + "Pass only when a script, test, hook, or CI workflow validates coding-agent instruction files; merely having AGENTS.md or CLAUDE.md does not pass.", + devcontainer: + "Pass only when a version-controlled dev-container definition exists.", + env_template: + "Pass only when repositories that require runtime environment variables provide a safe non-secret template using a recognized naming convention.", + local_services_setup: + "Pass only when required local services have a reproducible documented startup mechanism.", + database_schema: + "Pass only when database structure is version controlled through schemas or ordered migrations.", + devcontainer_runnable: + "Pass only when the dev-container is validated by CI or has evidence that its build and initialization commands are maintained.", + structured_logging: + "Pass only when application logging emits structured fields through a centralized logger in representative runtime paths.", + distributed_tracing: + "Pass only when cross-service requests carry trace context and emit spans through configured instrumentation; correlated log trace IDs without spans are insufficient.", + metrics_collection: + "Pass only when application or service metrics are emitted and exposed to a collection backend.", + code_quality_metrics: + "Pass only when code-quality metrics are measured and reported or gated over time.", + error_tracking_contextualized: + "Pass only when an error-tracking or aggregation system receives errors with release, environment, request, or user-safe diagnostic context; local structured logs alone are insufficient.", + alerting_configured: + "Pass only when actionable alert rules or monitors are version controlled.", + runbooks_documented: + "Pass only when operational incidents have discoverable, actionable runbooks.", + deployment_observability: + "Pass only when deployments can be correlated with runtime health, logs, errors, or metrics.", + health_checks: + "Pass only when deployable services expose meaningful liveness/readiness checks and deployment configuration uses them.", + circuit_breakers: + "Pass only when failure-prone remote calls use explicit timeout, retry, and circuit-breaking policies.", + profiling_instrumentation: + "Pass only when production-safe profiling or performance instrumentation can be enabled and interpreted.", + branch_protection: + "Pass only when local policy-as-code or repository documentation establishes protected review and status-check requirements.", + secret_scanning: + "Pass only when committed and incoming changes are automatically scanned for secrets.", + codeowners: "Pass only when a recognized CODEOWNERS file exists.", + automated_security_review: + "Pass only when dependency, static application, or infrastructure security analysis runs automatically.", + dependency_update_automation: + "Pass only when a bot or workflow proposes and validates dependency updates.", + gitignore_comprehensive: + "Pass only when ignore rules cover generated output, local state, credentials, and ecosystem-specific artifacts without hiding source.", + dast_scanning: + "Pass only when a runnable web/API target is tested by automated dynamic security scanning.", + pii_handling: + "Pass only when sensitive personal data has explicit classification, minimization, access, and storage controls in code or policy.", + privacy_compliance: + "Pass only when applicable privacy obligations have version-controlled implementation or operational guidance.", + secrets_management: + "Pass only when runtime secrets come from a managed injection mechanism and are not stored in source-controlled configuration; local .env files alone are not managed secret injection.", + log_scrubbing: + "Pass only when every representative logging path redacts or prevents secrets and personal data; a documented cleartext-secret exception is a failure.", + min_release_age: + "Pass only when dependency intake policy or automation delays or reviews newly published releases.", + issue_templates: + "Pass only when GitHub or GitLab issue templates collect actionable reproduction and context.", + issue_labeling_system: + "Pass only when labels are documented or automated enough to support consistent triage.", + backlog_health: + "Pass only when version-controlled automation or documented practice keeps stale, duplicate, and unprioritized work visible.", + pr_templates: + "Pass only when GitHub pull-request or GitLab merge-request templates provide an actionable review checklist.", + product_analytics_instrumentation: + "Pass only when product behavior is measured through intentional, documented events with ownership or schema.", + error_to_insight_pipeline: + "Pass only when production errors feed a repeatable triage, prioritization, or issue-creation workflow.", +}; + export const READINESS_RUBRIC: ReadinessCriterionDefinition[] = CATEGORIES.flatMap(([category, maturityLevel, ids]) => ids.map((id) => ({ id, category, maturityLevel, - description: `Evaluate whether the repository has effective ${label(id)} support.`, + description: + CRITERION_PASS_CONDITIONS[id] ?? + `Pass only when the repository has effective ${label(id)} support.`, applicability: - "Evaluate each independently deployable application when applicable; otherwise evaluate once at repository level. Skip only when local repository evidence cannot establish the criterion or the criterion genuinely does not apply.", + "CoDev has already determined that this criterion applies. Evaluate it once at repository level and do not skip it.", evidenceRequired: "Cite repository-relative files and, where useful, line numbers or the safe local command whose output supports the judgment.", })), @@ -372,7 +540,9 @@ export function validateReadinessOutput( return errors; } -export function readinessJsonSchema(): Record { +export function readinessJsonSchema( + criterionIds: string[] = READINESS_CRITERION_IDS, +): Record { const criterionSchema = { type: "object", additionalProperties: false, @@ -420,9 +590,9 @@ export function readinessJsonSchema(): Record { criteria: { type: "object", additionalProperties: false, - required: READINESS_CRITERION_IDS, + required: criterionIds, properties: Object.fromEntries( - READINESS_CRITERION_IDS.map((id) => [id, criterionSchema]), + criterionIds.map((id) => [id, criterionSchema]), ), }, warnings: { type: "array", items: { type: "string" } }, diff --git a/src/lib/readiness-plan.ts b/src/lib/readiness-plan.ts new file mode 100644 index 0000000..3c4a92b --- /dev/null +++ b/src/lib/readiness-plan.ts @@ -0,0 +1,721 @@ +import { spawnSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { basename, dirname, extname } from "node:path"; +import { + type AgentReadinessOutput, + READINESS_CRITERION_IDS, + READINESS_RUBRIC_VERSION, + type ReadinessApplication, + type ReadinessCriterionResult, +} from "@/lib/readiness-contract.js"; + +export type ReadinessDecision = + | { mode: "semantic"; reason: string } + | { mode: "not_applicable"; result: ReadinessCriterionResult } + | { mode: "deterministic"; result: ReadinessCriterionResult }; + +export interface ReadinessRepositoryProfile { + files: string[]; + trackedFiles: string[]; + relevantFiles: string[]; + languages: string[]; + applications: ReadinessApplication[]; + hasEnvironmentUsage: boolean; + hasDatabaseSurface: boolean; + hasApiSurface: boolean; + hasLocalServiceRequirement: boolean; +} + +export interface ReadinessEvaluationPlan { + profile: ReadinessRepositoryProfile; + criteria: Record; +} + +const IGNORED_COMPONENT_SEGMENTS = new Set([ + "example", + "examples", + "fixture", + "fixtures", + "test", + "tests", + "vendor", + "third_party", +]); + +const MANIFESTS = new Set([ + "package.json", + "pyproject.toml", + "Cargo.toml", + "go.mod", + "pom.xml", + "build.gradle", + "build.gradle.kts", + "composer.json", + "Gemfile", + "mix.exs", +]); + +const LANGUAGE_BY_EXTENSION: Record = { + ".c": "C", + ".cc": "C++", + ".cpp": "C++", + ".cs": "C#", + ".ex": "Elixir", + ".exs": "Elixir", + ".go": "Go", + ".java": "Java", + ".js": "JavaScript", + ".jsx": "JavaScript", + ".kt": "Kotlin", + ".kts": "Kotlin", + ".php": "PHP", + ".py": "Python", + ".rb": "Ruby", + ".rs": "Rust", + ".swift": "Swift", + ".ts": "TypeScript", + ".tsx": "TypeScript", +}; + +const RELEVANT_FILE = + /(^|\/)(AGENTS\.md|CLAUDE\.md|README(?:\.[^/]+)?|CODEOWNERS|\.gitignore|package\.json|pyproject\.toml|Cargo\.toml|go\.mod|pom\.xml|build\.gradle(?:\.kts)?|docker-compose[^/]*|compose[^/]*|Dockerfile[^/]*|[^/]*(?:config|schema|migration|workflow|pipeline|test|spec)[^/]*)$/i; +const ENV_USAGE = + /\b(process\.env|import\.meta\.env|os\.(?:getenv|environ)|System\.getenv|ENV\[|getenv\s*\(|env\s*\()/; +const DATABASE_SIGNAL = + /\b(postgres(?:ql)?|mysql|mariadb|sqlite|mongodb|redis|supabase|prisma|typeorm|sequelize|drizzle|sqlalchemy|django\.db|alembic|flyway|liquibase|diesel)\b/i; +const API_SIGNAL = + /\b(express|fastify|hono|nestjs|next\/server|flask|fastapi|django|rails|sinatra|spring-web|actix-web|axum|gin-gonic|openapi|swagger|graphql)\b/i; + +function isRelevantFile(file: string): boolean { + return ( + RELEVANT_FILE.test(file) || + /(^|\/)(?:\.github\/workflows|\.gitlab\/ci|\.devcontainer|\.husky|scripts?|tests?)(\/|\.)/i.test( + file, + ) + ); +} + +function listedFiles(root: string, args: string[]): string[] { + const result = spawnSync("git", ["ls-files", ...args, "-z"], { + cwd: root, + encoding: "utf8", + maxBuffer: 20 * 1024 * 1024, + }); + if (result.status !== 0) return []; + return result.stdout + .split("\0") + .map((file) => file.replace(/^\.\//, "")) + .filter(Boolean) + .sort(); +} + +function safeText(root: string, file: string): string { + try { + const content = readFileSync(`${root}/${file}`); + if (content.byteLength > 256_000 || content.includes(0)) return ""; + return content.toString("utf8"); + } catch { + return ""; + } +} + +function anyFile(files: string[], patterns: RegExp[]): string[] { + return files.filter((file) => patterns.some((pattern) => pattern.test(file))); +} + +function result( + status: "pass" | "fail" | "skipped", + rationale: string, + evidence: string[] = [], +): ReadinessCriterionResult { + return { + status, + numerator: status === "skipped" ? null : status === "pass" ? 1 : 0, + denominator: 1, + rationale, + evidence, + }; +} + +function presenceDecision( + files: string[], + patterns: RegExp[], + passRationale: string, + failRationale: string, +): ReadinessDecision { + const matches = anyFile(files, patterns); + return { + mode: "deterministic", + result: matches.length + ? result("pass", passRationale, matches.slice(0, 8)) + : result("fail", failRationale), + }; +} + +function contentDecision( + root: string, + files: string[], + pattern: RegExp, + passRationale: string, + failRationale: string, +): ReadinessDecision { + const matches = files.filter((file) => pattern.test(safeText(root, file))); + return { + mode: "deterministic", + result: matches.length + ? result("pass", passRationale, matches.slice(0, 8)) + : result("fail", failRationale), + }; +} + +function applications( + files: string[], + languages: string[], +): ReadinessApplication[] { + const manifests = files.filter((file) => { + if (!MANIFESTS.has(basename(file)) && !/\.csproj$/i.test(file)) + return false; + return !file + .split("/") + .some((segment) => IGNORED_COMPONENT_SEGMENTS.has(segment.toLowerCase())); + }); + const byDirectory = new Map(); + for (const manifest of manifests) { + const directory = dirname(manifest) === "." ? "." : dirname(manifest); + byDirectory.set(directory, [ + ...(byDirectory.get(directory) ?? []), + manifest, + ]); + } + if (byDirectory.size === 0) { + return [{ path: ".", description: "Repository root", languages }]; + } + return [...byDirectory.entries()] + .sort(([left], [right]) => left.localeCompare(right)) + .map(([path, detected]) => ({ + path, + description: `Component detected from ${detected.map((file) => basename(file)).join(", ")}`, + languages, + })); +} + +export function buildReadinessEvaluationPlan( + root: string, +): ReadinessEvaluationPlan { + const trackedFiles = listedFiles(root, ["--cached"]); + const files = listedFiles(root, [ + "--cached", + "--others", + "--exclude-standard", + ]); + const languages = [ + ...new Set( + files + .map((file) => LANGUAGE_BY_EXTENSION[extname(file).toLowerCase()]) + .filter((language): language is string => Boolean(language)), + ), + ].sort(); + const relevantFiles = files.filter(isRelevantFile).slice(0, 800); + const searchableFiles = files.filter((file) => + /\.(?:[cm]?[jt]sx?|py|rb|rs|go|java|kt|kts|php|exs?|toml|ya?ml|json|md|env)$/i.test( + file, + ), + ); + let hasEnvironmentUsage = false; + let hasDatabaseSurface = files.some((file) => + /(^|\/)(migrations?|schema|prisma)(\/|\.|$)/i.test(file), + ); + let hasApiSurface = files.some((file) => + /(^|\/)(api|routes?|controllers?)(\/|\.|$)/i.test(file), + ); + let hasLocalServiceRequirement = files.some((file) => + /(^|\/)(?:docker-)?compose(?:\.[^/]+)?\.ya?ml$/i.test(file), + ); + for (const file of searchableFiles.slice(0, 2_000)) { + if ( + hasEnvironmentUsage && + hasDatabaseSurface && + hasApiSurface && + hasLocalServiceRequirement + ) + break; + const text = safeText(root, file); + if (!hasEnvironmentUsage && ENV_USAGE.test(text)) + hasEnvironmentUsage = true; + if (!hasDatabaseSurface && DATABASE_SIGNAL.test(text)) + hasDatabaseSurface = true; + if (!hasApiSurface && API_SIGNAL.test(text)) hasApiSurface = true; + if ( + !hasLocalServiceRequirement && + /(?:localhost|127\.0\.0\.1|docker\s+compose|podman\s+compose)/i.test(text) + ) + hasLocalServiceRequirement = true; + } + + const criteria = Object.fromEntries( + READINESS_CRITERION_IDS.map((id) => [ + id, + { mode: "semantic", reason: "Requires contextual repository judgment." }, + ]), + ) as Record; + + criteria.agents_md = presenceDecision( + trackedFiles, + [ + /^AGENTS\.md$/i, + /^CLAUDE\.md$/i, + /(^|\/)AGENTS\.md$/i, + /(^|\/)CLAUDE\.md$/i, + ], + "The repository contains coding-agent instructions.", + "No AGENTS.md or CLAUDE.md coding-agent instructions were found.", + ); + criteria.readme = presenceDecision( + trackedFiles, + [/^README(?:\.[^/]+)?$/i], + "The repository root contains a README.", + "No repository-root README was found.", + ); + criteria.codeowners = presenceDecision( + trackedFiles, + [/^CODEOWNERS$/i, /^\.github\/CODEOWNERS$/i, /^docs\/CODEOWNERS$/i], + "A recognized CODEOWNERS file is present.", + "No recognized CODEOWNERS file was found.", + ); + criteria.issue_templates = presenceDecision( + trackedFiles, + [/^\.github\/ISSUE_TEMPLATE\//i, /^\.gitlab\/issue_templates\//i], + "Issue templates are configured.", + "No GitHub or GitLab issue templates were found.", + ); + criteria.pr_templates = presenceDecision( + trackedFiles, + [ + /^\.github\/(?:PULL_REQUEST_TEMPLATE\/|pull_request_template\.)/i, + /^\.gitlab\/merge_request_templates\//i, + ], + "Pull or merge request templates are configured.", + "No GitHub pull-request or GitLab merge-request templates were found.", + ); + criteria.devcontainer = presenceDecision( + trackedFiles, + [/^\.devcontainer\.json$/i, /^\.devcontainer\/[^/]*\.json$/i], + "A dev-container definition is present.", + "No dev-container definition was found.", + ); + criteria.skills = presenceDecision( + trackedFiles, + [ + /(^|\/)(?:\.claude|\.agents|\.codex)\/skills\//i, + /(^|\/)skills\/[^/]+\/SKILL\.md$/i, + ], + "Repository-local agent skills are present.", + "No recognized repository-local agent skills were found.", + ); + + const envTemplates = anyFile(files, [ + /(^|\/)\.env(?:\.[^/]+)*\.(?:example|sample|template|dist)$/i, + /(^|\/)(?:env\.example|example\.env|sample\.env)$/i, + ]); + criteria.env_template = { + mode: + hasEnvironmentUsage || envTemplates.length + ? "deterministic" + : "not_applicable", + result: envTemplates.length + ? result( + "pass", + "An environment-variable template is present.", + envTemplates.slice(0, 8), + ) + : hasEnvironmentUsage + ? result( + "fail", + "The repository consumes environment variables but no recognized example, sample, template, or dist env file was found.", + ) + : result( + "skipped", + "No repository evidence indicates that runtime environment variables are required.", + ), + }; + + const hookFiles = anyFile(trackedFiles, [ + /^\.husky\/(?!_\/)[^/]+$/i, + /^\.pre-commit-config\.ya?ml$/i, + /^lefthook\.ya?ml$/i, + /^\.lefthook\.ya?ml$/i, + ]); + const meaningfulHooks = hookFiles.filter((file) => + /(?:test|lint|check|typecheck|format|build|audit|scan)/i.test( + safeText(root, file), + ), + ); + criteria.pre_commit_hooks = { + mode: "deterministic", + result: meaningfulHooks.length + ? result( + "pass", + "A version-controlled pre-commit hook runs repository validation.", + meaningfulHooks, + ) + : result( + "fail", + "No version-controlled pre-commit hook running meaningful validation was found.", + ), + }; + + const automationFiles = trackedFiles.filter( + (file) => + /(^|\/)(?:\.github\/workflows|scripts?|ci)(\/|\.)/i.test(file) && + !/(?:readiness|rubric|score)/i.test(file), + ); + const instructionValidation = automationFiles.filter((file) => { + const text = safeText(root, file); + return ( + /(?:AGENTS|CLAUDE)\.md/i.test(text) && + /(?:validate|check|lint|test|require|exist)/i.test(text) + ); + }); + criteria.agents_md_validation = { + mode: "deterministic", + result: instructionValidation.length + ? result( + "pass", + "Automation validates coding-agent instruction files.", + instructionValidation.slice(0, 8), + ) + : result( + "fail", + "No automation validating AGENTS.md or CLAUDE.md was found.", + ), + }; + + const contributorDocs = trackedFiles.filter((file) => + /(^|\/)(?:README(?:\.[^/]+)?|AGENTS\.md|CLAUDE\.md|CONTRIBUTING(?:\.[^/]+)?|docs\/[^/]+)$/i.test( + file, + ), + ); + criteria.vcs_cli_tools = contentDecision( + root, + contributorDocs, + /\b(?:git|gh|glab)\s+(?:clone|issue|pr|checkout|switch|rebase|bisect|blame|commit|push|pull|status|diff|log)\b/i, + "Contributor documentation includes concrete version-control or hosting CLI commands.", + "Contributor documentation contains no concrete version-control or hosting CLI workflow commands.", + ); + const flowDocs = contributorDocs.filter((file) => { + const text = safeText(root, file); + return ( + /(?:architecture|data flow|request flow|command flow|layered|component interactions?)/i.test( + text, + ) && + /(?:src\/|components?\/|lib\/|API|backend|frontend|worker|proxy|-->|→)/i.test( + text, + ) + ); + }); + criteria.service_flow_documented = { + mode: "deterministic", + result: flowDocs.length + ? result( + "pass", + "Repository documentation describes architecture or an important component flow.", + flowDocs.slice(0, 8), + ) + : result( + "fail", + "No architecture, request, data, or command flow documentation with component interactions was found.", + ), + }; + + const gitignoreFiles = trackedFiles.filter((file) => + /(^|\/)\.gitignore$/i.test(file), + ); + const gitignoreText = gitignoreFiles + .map((file) => safeText(root, file)) + .join("\n"); + const ignoreRequirements: Array<[string, RegExp]> = []; + if (trackedFiles.some((file) => basename(file) === "package.json")) { + ignoreRequirements.push( + ["Node dependencies", /(?:^|\/)node_modules\/?/im], + ["JavaScript build output", /(?:^|\/)(?:dist|build|out)\/?/im], + ); + } + if ( + trackedFiles.some((file) => + ["pyproject.toml", "requirements.txt", "setup.py"].includes( + basename(file), + ), + ) + ) { + ignoreRequirements.push( + ["Python bytecode", /__pycache__|\*\.pyc/im], + ["Python virtual environments", /(?:^|\/)(?:\.venv|venv)\/?/im], + ); + } + if (trackedFiles.some((file) => basename(file) === "Cargo.toml")) + ignoreRequirements.push(["Rust build output", /(?:^|\/)target\/?/im]); + if ( + trackedFiles.some((file) => + ["pom.xml", "build.gradle", "build.gradle.kts"].includes(basename(file)), + ) + ) + ignoreRequirements.push([ + "JVM build output", + /(?:^|\/)(?:target|build|\.gradle)\/?/im, + ]); + if (hasEnvironmentUsage) + ignoreRequirements.push([ + "local environment files", + /(?:^|\/)\.env(?:[.*]|$)/im, + ]); + const missingIgnoreGroups = ignoreRequirements + .filter(([, pattern]) => !pattern.test(gitignoreText)) + .map(([name]) => name); + criteria.gitignore_comprehensive = { + mode: "deterministic", + result: + gitignoreFiles.length > 0 && missingIgnoreGroups.length === 0 + ? result( + "pass", + "Git ignore rules cover the repository's detected ecosystems and local configuration.", + gitignoreFiles, + ) + : result( + "fail", + gitignoreFiles.length === 0 + ? "No version-controlled .gitignore was found." + : `Git ignore rules are missing coverage for: ${missingIgnoreGroups.join(", ")}.`, + gitignoreFiles, + ), + }; + + const toolingFiles = trackedFiles.filter((file) => + /(^|\/)(?:package\.json|pyproject\.toml|Cargo\.toml|go\.mod|pom\.xml|build\.gradle(?:\.kts)?|biome\.json|eslint[^/]*|\.eslintrc[^/]*|sonar-project\.properties|Makefile|Taskfile\.ya?ml|justfile|\.github\/workflows\/[^/]+)$/i.test( + file, + ), + ); + criteria.cyclomatic_complexity = contentDecision( + root, + toolingFiles, + /(?:noExcessiveCognitiveComplexity|["']complexity["']\s*:|\bradon\b|\blizard\b|sonar\.(?:cognitive_)?complexity|cyclomatic[-_ ]complexity)/i, + "Configured tooling measures or limits source-code complexity.", + "No configured source-code complexity measurement or limit was found.", + ); + criteria.duplicate_code_detection = contentDecision( + root, + toolingFiles, + /(?:\bjscpd\b|\bsimian\b|\bduplo\b|sonar\.cpd|copy.?paste detector)/i, + "Configured tooling detects duplicated code blocks.", + "No configured clone or duplicated-code detector was found.", + ); + criteria.unused_dependencies_detection = contentDecision( + root, + toolingFiles, + /(?:\bknip\b|\bdepcheck\b|\bunimported\b|cargo\s+udeps|unused[-_ ]dependenc)/i, + "Configured tooling detects unused declared dependencies.", + "No configured unused-dependency detector was found.", + ); + const biomeRecommended = toolingFiles.some( + (file) => + /biome\.jsonc?$/i.test(file) && + /["']recommended["']\s*:\s*true/i.test(safeText(root, file)), + ); + const biomeExecuted = toolingFiles.some( + (file) => + basename(file) === "package.json" && + /\bbiome\s+(?:check|lint)\b/i.test(safeText(root, file)), + ); + const deadCodeTools = toolingFiles.filter((file) => + /(?:\bknip\b|\bts-prune\b|\bvulture\b|\bdeadcode\b|cargo\s+udeps|noUnusedLocals["']?\s*:\s*true|noUnusedVariables)/i.test( + safeText(root, file), + ), + ); + criteria.dead_code_detection = { + mode: "deterministic", + result: + deadCodeTools.length || (biomeRecommended && biomeExecuted) + ? result( + "pass", + "Configured validation detects unused or dead code.", + deadCodeTools.length + ? deadCodeTools.slice(0, 8) + : toolingFiles + .filter((file) => + /(?:biome\.json|package\.json)$/i.test(file), + ) + .slice(0, 8), + ) + : result( + "fail", + "No configured unused-code or dead-code detector was found.", + ), + }; + + const bootstrapFiles = trackedFiles.filter((file) => + /(^|\/)(?:package\.json|Makefile|Taskfile\.ya?ml|justfile|README(?:\.[^/]+)?|CONTRIBUTING(?:\.[^/]+)?)$/i.test( + file, + ), + ); + criteria.single_command_setup = contentDecision( + root, + bootstrapFiles, + /(?:["'](?:setup|bootstrap|init)["']\s*:|^(?:setup|bootstrap|init)\s*:|\b(?:make|just|task|npm|pnpm|yarn|bun)\s+(?:run\s+)?(?:setup|bootstrap|init)\b)/im, + "A documented setup, bootstrap, or init task provides a single repository-owned setup command.", + "No repository-owned single-command setup, bootstrap, or init task was found.", + ); + + const protectionFiles = trackedFiles.filter((file) => + /(?:^\.github\/settings\.ya?ml$|\.tf$|rulesets?\/[^/]+\.json$)/i.test(file), + ); + criteria.branch_protection = contentDecision( + root, + protectionFiles, + /(?:github_branch_protection|github_repository_ruleset|protected_branches?|required_status_checks|required_pull_request_reviews)/i, + "Version-controlled policy-as-code configures branch protection requirements.", + "No version-controlled branch-protection policy-as-code was found.", + ); + + if (hasLocalServiceRequirement) { + criteria.local_services_setup = contentDecision( + root, + [ + ...contributorDocs, + ...trackedFiles.filter((file) => + /(?:compose(?:\.[^/]+)?\.ya?ml$|package\.json$|Makefile$|Taskfile\.ya?ml$|justfile$)/i.test( + file, + ), + ), + ], + /(?:docker\s+compose\s+up|podman\s+compose\s+up|supabase\s+start|\b(?:make|just|task|npm|pnpm|yarn|bun)\s+(?:run\s+)?(?:services?|infra|stack|proxy|backend)\b)/i, + "Required local services have a documented repository-owned startup command.", + "A local service or localhost dependency is required, but no reproducible startup command was found.", + ); + } else { + criteria.local_services_setup = { + mode: "not_applicable", + result: result( + "skipped", + "No required local service or localhost dependency was detected.", + ), + }; + } + + for (const id of ["database_schema", "n_plus_one_detection"]) { + if (!hasDatabaseSurface) { + criteria[id] = { + mode: "not_applicable", + result: result( + "skipped", + "No database or persistence surface was detected in the repository.", + ), + }; + } + } + for (const id of ["api_schema_docs", "dast_scanning"]) { + if (!hasApiSurface) { + criteria[id] = { + mode: "not_applicable", + result: result( + "skipped", + "No HTTP API or web application surface was detected in the repository.", + ), + }; + } + } + + return { + profile: { + files, + trackedFiles, + relevantFiles, + languages, + applications: applications(files, languages), + hasEnvironmentUsage, + hasDatabaseSurface, + hasApiSurface, + hasLocalServiceRequirement, + }, + criteria, + }; +} + +export function semanticCriterionIds(plan: ReadinessEvaluationPlan): string[] { + return READINESS_CRITERION_IDS.filter( + (id) => plan.criteria[id]?.mode === "semantic", + ); +} + +export function finalizeReadinessOutput( + output: AgentReadinessOutput, + plan: ReadinessEvaluationPlan, +): AgentReadinessOutput { + const warnings = [...(Array.isArray(output.warnings) ? output.warnings : [])]; + const criteria = Object.fromEntries( + READINESS_CRITERION_IDS.map((id) => { + const decision = plan.criteria[id]; + if (decision?.mode !== "semantic") return [id, decision?.result]; + const agentResult = output.criteria?.[id]; + if (!agentResult || agentResult.status === "skipped") { + warnings.push( + `${id} was not judged by the agent and was conservatively scored as failing.`, + ); + return [ + id, + result( + "fail", + agentResult?.rationale || + "The semantic evaluator did not establish this criterion.", + agentResult?.evidence ?? [], + ), + ]; + } + return [ + id, + result( + agentResult.status === "pass" ? "pass" : "fail", + agentResult.rationale, + agentResult.evidence, + ), + ]; + }), + ) as Record; + return { + ...output, + rubricVersion: READINESS_RUBRIC_VERSION, + languages: plan.profile.languages, + applications: plan.profile.applications, + criteria, + warnings: [...new Set(warnings)], + recommendations: + Array.isArray(output.recommendations) && + output.recommendations.length >= 2 + ? output.recommendations.slice(0, 3) + : [ + "Address the highest-impact failing readiness criteria.", + "Document and automate the repository's development workflow.", + ], + model: typeof output.model === "string" ? output.model : null, + }; +} + +export function readinessPlanPrompt(plan: ReadinessEvaluationPlan): string { + const semantic = semanticCriterionIds(plan); + const fixed = Object.fromEntries( + READINESS_CRITERION_IDS.filter( + (id) => plan.criteria[id]?.mode !== "semantic", + ).map((id) => [id, plan.criteria[id]]), + ); + return JSON.stringify({ + profile: { + languages: plan.profile.languages, + applications: plan.profile.applications, + hasEnvironmentUsage: plan.profile.hasEnvironmentUsage, + hasDatabaseSurface: plan.profile.hasDatabaseSurface, + hasApiSurface: plan.profile.hasApiSurface, + hasLocalServiceRequirement: plan.profile.hasLocalServiceRequirement, + }, + relevantFiles: plan.profile.relevantFiles, + semanticCriteria: semantic, + fixedDecisions: fixed, + }); +} diff --git a/src/lib/readiness.ts b/src/lib/readiness.ts index 81a9948..85776d0 100644 --- a/src/lib/readiness.ts +++ b/src/lib/readiness.ts @@ -14,6 +14,10 @@ import { summarizeReadiness, validateReadinessOutput, } from "@/lib/readiness-contract.js"; +import { + buildReadinessEvaluationPlan, + finalizeReadinessOutput, +} from "@/lib/readiness-plan.js"; import { ensureFreshGatewayKey } from "@/lib/refresh.js"; function git(args: string[], cwd: string): string { @@ -80,7 +84,9 @@ export async function runReadiness( } const before = repositoryState(root); - onProgress(`Scanning repository with ${agent}, this will take a while`); + onProgress("Building a fresh deterministic repository profile"); + const plan = buildReadinessEvaluationPlan(root); + onProgress(`Evaluating semantic readiness criteria with ${agent}`); let run: AgentRunResult; try { const gatewayTool = gatewayToolForReadiness(agent); @@ -91,8 +97,15 @@ export async function runReadiness( undefined, options.model, onProgress, + plan, ); - run = { ...run, output: normalizeReadinessEvidence(run.output, root) }; + run = { + ...run, + output: normalizeReadinessEvidence( + finalizeReadinessOutput(run.output, plan), + root, + ), + }; let totalDurationMs = run.durationMs; let errors = validateReadinessOutput(run.output, root); const { maxRepairs } = readinessRuntimeConfig(); @@ -114,8 +127,15 @@ export async function runReadiness( }, options.model, onProgress, + plan, ); - run = { ...run, output: normalizeReadinessEvidence(run.output, root) }; + run = { + ...run, + output: normalizeReadinessEvidence( + finalizeReadinessOutput(run.output, plan), + root, + ), + }; totalDurationMs += run.durationMs; errors = validateReadinessOutput(run.output, root); } diff --git a/tests/lib/readiness-contract.test.ts b/tests/lib/readiness-contract.test.ts index f9575fe..f0aa09b 100644 --- a/tests/lib/readiness-contract.test.ts +++ b/tests/lib/readiness-contract.test.ts @@ -127,6 +127,11 @@ describe("readiness contract", () => { expect(properties.rubricVersion?.type).toBe("string"); expect(buildReadinessPrompt()).toContain(READINESS_RUBRIC_VERSION); expect(buildReadinessPrompt()).toContain("Do not edit"); + const subsetSchema = readinessJsonSchema(["lint_config"]); + const subsetCriteria = ( + subsetSchema.properties as Record> + ).criteria as Record; + expect(subsetCriteria.required).toEqual(["lint_config"]); }); it("builds read-only headless commands for every supported provider", () => { @@ -180,7 +185,9 @@ describe("readiness contract", () => { const instruction = openCodeStructuredOutputInstruction(); expect(instruction).toContain('"rubricVersion"'); expect(instruction).toContain('"status":"pass|fail|skipped"'); - expect(instruction).toContain("every rubric id exactly once"); + expect(instruction).toContain( + "every criterion listed in the Semantic rubric", + ); }); it("removes CoDev shims from readiness subprocess PATH resolution", () => { diff --git a/tests/lib/readiness-plan.test.ts b/tests/lib/readiness-plan.test.ts new file mode 100644 index 0000000..f18a71f --- /dev/null +++ b/tests/lib/readiness-plan.test.ts @@ -0,0 +1,300 @@ +import { execFileSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + type AgentReadinessOutput, + READINESS_CRITERION_IDS, + READINESS_RUBRIC_VERSION, + summarizeReadiness, +} from "@/lib/readiness-contract.js"; +import { + buildReadinessEvaluationPlan, + finalizeReadinessOutput, + semanticCriterionIds, +} from "@/lib/readiness-plan.js"; + +const roots: string[] = []; + +function repository(files: Record): string { + const root = mkdtempSync(join(tmpdir(), "codev-readiness-plan-")); + roots.push(root); + execFileSync("git", ["init", "-q"], { cwd: root }); + for (const [file, content] of Object.entries(files)) { + mkdirSync(join(root, file, ".."), { recursive: true }); + writeFileSync(join(root, file), content); + } + execFileSync("git", ["add", "."], { cwd: root }); + return root; +} + +function agentOutput( + status: "pass" | "fail" | "skipped", +): AgentReadinessOutput { + return { + rubricVersion: READINESS_RUBRIC_VERSION, + languages: ["Imaginary"], + applications: [{ path: ".", description: "Agent guess", languages: [] }], + criteria: Object.fromEntries( + READINESS_CRITERION_IDS.map((id) => [ + id, + { + status, + numerator: status === "skipped" ? null : status === "pass" ? 3 : 0, + denominator: status === "skipped" ? 1 : 3, + rationale: `Agent marked ${id} ${status}.`, + evidence: [], + }, + ]), + ), + warnings: [], + recommendations: ["First", "Second"], + model: "fixture", + }; +} + +afterEach(() => { + for (const root of roots.splice(0)) + rmSync(root, { recursive: true, force: true }); +}); + +describe("readiness evaluation plan", () => { + it("recognizes env template naming variants only when env configuration is relevant", () => { + const root = repository({ + "package.json": '{"scripts":{"start":"node src/index.js"}}', + "src/index.js": "console.log(process.env.PORT);", + ".env.local.example": "PORT=3000\n", + }); + + const plan = buildReadinessEvaluationPlan(root); + + expect(plan.profile.hasEnvironmentUsage).toBe(true); + expect(plan.criteria.env_template).toMatchObject({ + mode: "deterministic", + result: { status: "pass", evidence: [".env.local.example"] }, + }); + }); + + it("does not penalize a repository that has no environment-variable requirement", () => { + const root = repository({ + "Cargo.toml": '[package]\nname = "pure-library"\n', + "src/lib.rs": "pub fn add(a: i32, b: i32) -> i32 { a + b }", + }); + + const plan = buildReadinessEvaluationPlan(root); + + expect(plan.criteria.env_template).toMatchObject({ + mode: "not_applicable", + result: { status: "skipped", numerator: null }, + }); + }); + + it("uses exact cross-platform repository conventions for presence checks", () => { + const root = repository({ + "CLAUDE.md": "# Agent instructions", + "README.md": "# Fixture", + ".github/CODEOWNERS": "* @team", + ".github/ISSUE_TEMPLATE/bug.yml": "name: Bug", + ".gitlab/merge_request_templates/default.md": "Checklist", + ".devcontainer/devcontainer.json": "{}", + ".agents/skills/review/SKILL.md": "---\nname: review\n---", + }); + + const plan = buildReadinessEvaluationPlan(root); + + for (const id of [ + "agents_md", + "readme", + "codeowners", + "issue_templates", + "pr_templates", + "devcontainer", + "skills", + ]) { + expect(plan.criteria[id]).toMatchObject({ + mode: "deterministic", + result: { status: "pass" }, + }); + } + }); + + it("scores tracked validation hooks and instruction validation deterministically", () => { + const root = repository({ + "AGENTS.md": "# Agent instructions", + ".husky/pre-commit": "pnpm lint && pnpm test", + ".github/workflows/agents.yml": + "steps:\n - run: node scripts/validate-agents.js AGENTS.md\n", + }); + + const plan = buildReadinessEvaluationPlan(root); + + expect(plan.criteria.pre_commit_hooks).toMatchObject({ + mode: "deterministic", + result: { status: "pass", evidence: [".husky/pre-commit"] }, + }); + expect(plan.criteria.agents_md_validation).toMatchObject({ + mode: "deterministic", + result: { status: "pass" }, + }); + expect(plan.profile.relevantFiles).toContain( + ".github/workflows/agents.yml", + ); + }); + + it("does not confuse general linting with specialized repository tooling", () => { + const root = repository({ + "package.json": + '{"scripts":{"check":"biome check","setup":"pnpm install"},"devDependencies":{"@biomejs/biome":"1"}}', + "biome.json": '{"linter":{"enabled":true,"rules":{"recommended":true}}}', + "README.md": "Run `pnpm setup` to bootstrap the repository.", + }); + + const plan = buildReadinessEvaluationPlan(root); + + expect(plan.criteria.single_command_setup).toMatchObject({ + mode: "deterministic", + result: { status: "pass" }, + }); + expect(plan.criteria.dead_code_detection).toMatchObject({ + mode: "deterministic", + result: { status: "pass" }, + }); + for (const id of [ + "cyclomatic_complexity", + "duplicate_code_detection", + "unused_dependencies_detection", + ]) { + expect(plan.criteria[id]).toMatchObject({ + mode: "deterministic", + result: { status: "fail" }, + }); + } + }); + + it("requires real VCS commands and branch policy rather than CI mentions", () => { + const root = repository({ + "README.md": "GitHub Actions runs on every pull request.", + ".github/workflows/test.yml": + "jobs: { test: { runs-on: ubuntu-latest } }", + }); + const configured = repository({ + "CONTRIBUTING.md": "Use `gh pr create` after `git push`.", + "infra/github.tf": + 'resource "github_branch_protection" "main" { required_status_checks { strict = true } }', + }); + + expect( + buildReadinessEvaluationPlan(root).criteria.vcs_cli_tools, + ).toMatchObject({ result: { status: "fail" } }); + expect( + buildReadinessEvaluationPlan(root).criteria.branch_protection, + ).toMatchObject({ result: { status: "fail" } }); + expect( + buildReadinessEvaluationPlan(configured).criteria.vcs_cli_tools, + ).toMatchObject({ result: { status: "pass" } }); + expect( + buildReadinessEvaluationPlan(configured).criteria.branch_protection, + ).toMatchObject({ result: { status: "pass" } }); + }); + + it("evaluates flow documentation and ecosystem-specific ignore coverage", () => { + const complete = buildReadinessEvaluationPlan( + repository({ + "package.json": '{"scripts":{"build":"tsc"}}', + "src/index.ts": "console.log(process.env.PORT);", + "AGENTS.md": + "## Layered architecture\n`src/index.ts` dispatches to `components/`, then `lib/` owns backend API calls.", + ".gitignore": "node_modules/\ndist/\n.env*\n!.env.example\n", + }), + ); + const incomplete = buildReadinessEvaluationPlan( + repository({ + "pyproject.toml": "[project]\nname = 'fixture'\n", + "README.md": "# Fixture\nA small project.", + ".gitignore": "__pycache__/\n", + }), + ); + + expect(complete.criteria.service_flow_documented).toMatchObject({ + mode: "deterministic", + result: { status: "pass" }, + }); + expect(complete.criteria.gitignore_comprehensive).toMatchObject({ + mode: "deterministic", + result: { status: "pass" }, + }); + expect(incomplete.criteria.service_flow_documented).toMatchObject({ + result: { status: "fail" }, + }); + expect(incomplete.criteria.gitignore_comprehensive).toMatchObject({ + result: { status: "fail" }, + }); + }); + + it("makes evaluated-count independent of agent skip behavior", () => { + const root = repository({ + "README.md": "# Fixture", + "package.json": '{"scripts":{"test":"vitest"}}', + "src/index.ts": "export const value = 1;", + }); + const plan = buildReadinessEvaluationPlan(root); + const semanticCount = semanticCriterionIds(plan).length; + + const skipped = finalizeReadinessOutput(agentOutput("skipped"), plan); + const passed = finalizeReadinessOutput(agentOutput("pass"), plan); + + expect(summarizeReadiness(skipped.criteria).criteriaTotal).toBe( + summarizeReadiness(passed.criteria).criteriaTotal, + ); + expect( + Object.values(skipped.criteria).filter((item) => item.status === "fail") + .length, + ).toBeGreaterThanOrEqual(semanticCount); + expect(skipped.languages).toEqual(["TypeScript"]); + expect(skipped.applications).toEqual(passed.applications); + }); + + it("only skips database and API checks after high-confidence absence", () => { + const plain = buildReadinessEvaluationPlan( + repository({ "README.md": "# Notes" }), + ); + const service = buildReadinessEvaluationPlan( + repository({ + "package.json": '{"dependencies":{"express":"1","prisma":"1"}}', + "prisma/schema.prisma": "model User { id Int @id }", + "src/routes/users.ts": "export const route = true;", + }), + ); + + expect(plain.criteria.database_schema?.mode).toBe("not_applicable"); + expect(plain.criteria.api_schema_docs?.mode).toBe("not_applicable"); + expect(plain.criteria.local_services_setup?.mode).toBe("not_applicable"); + expect(service.criteria.database_schema?.mode).toBe("semantic"); + expect(service.criteria.api_schema_docs?.mode).toBe("semantic"); + }); + + it("requires a startup command only when a local service is detected", () => { + const missingSetup = buildReadinessEvaluationPlan( + repository({ + "src/config.ts": 'export const url = "http://localhost:8787";', + }), + ); + const documentedSetup = buildReadinessEvaluationPlan( + repository({ + "src/config.ts": 'export const url = "http://localhost:5432";', + "README.md": "Run `docker compose up` to start local services.", + "compose.yml": "services: { db: { image: postgres } }", + }), + ); + + expect(missingSetup.criteria.local_services_setup).toMatchObject({ + mode: "deterministic", + result: { status: "fail" }, + }); + expect(documentedSetup.criteria.local_services_setup).toMatchObject({ + mode: "deterministic", + result: { status: "pass" }, + }); + }); +}); From d2f7fda6aebdccc73b8773d565860ef4ac207de1 Mon Sep 17 00:00:00 2001 From: Bao Tran Date: Wed, 15 Jul 2026 16:11:18 +0700 Subject: [PATCH 04/14] clean doc --- docs/readiness-evaluation.md | 51 ------------------------------------ 1 file changed, 51 deletions(-) delete mode 100644 docs/readiness-evaluation.md diff --git a/docs/readiness-evaluation.md b/docs/readiness-evaluation.md deleted file mode 100644 index 68467b1..0000000 --- a/docs/readiness-evaluation.md +++ /dev/null @@ -1,51 +0,0 @@ -# Readiness evaluation design - -## Goal - -Produce a fresh, evidence-backed readiness report whose applicability and evaluated-criterion count are stable across Claude Code, Codex, and OpenCode. The selected agent should contribute semantic judgment, not decide which parts of the rubric count toward the score. - -## Invariants - -- Every run rebuilds the repository profile from the current working tree. This release does not cache by commit or file hash. -- Every canonical criterion receives exactly one decision: deterministic result, rule-backed not applicable, or semantic evaluation. -- Only CoDev may mark a criterion not applicable. A semantic evaluator must return pass or fail. -- Every evaluated criterion contributes one equally weighted repository-level vote. Provider-specific application discovery cannot change the denominator. -- Deterministic checks accept documented ecosystem variants rather than one preferred filename. -- Absence is a failure only when a criterion is universally expected or applicability is positively established. - -## Evaluation boundary - -Deterministic presence checks are limited to criteria whose requirement is the artifact itself: - -- coding-agent instructions: `AGENTS.md` or `CLAUDE.md` -- root README -- recognized CODEOWNERS locations -- GitHub/GitLab issue and pull/merge-request templates -- dev-container definitions -- repository-local agent skill directories -- meaningful version-controlled pre-commit hooks -- automation that validates agent-instruction files -- documented VCS/hosting CLI commands -- configured dead-code, complexity, clone, and unused-dependency tooling -- repository-owned setup/bootstrap tasks -- version-controlled branch-protection policy-as-code -- ecosystem-aware `.gitignore` coverage -- documentation with explicit architecture or component-flow signals - -Environment templates use both applicability and variant detection. `.env.example`, `.env.local.example`, `.env.sample`, `.env.template`, `.env.dist`, `env.example`, `example.env`, and `sample.env` are recognized. A missing template fails only when source or configuration evidence shows environment-variable consumption; otherwise the criterion is not applicable. - -Database-only and API-only checks are not applicable only when the fresh profile finds no corresponding surface. The signals intentionally combine directory conventions, manifests, dependencies, and representative source/configuration content. Positive detection delegates the actual quality judgment to the semantic evaluator. - -All remaining criteria are semantic. The agent receives the deterministic profile, a bounded list of relevant files, explicit rubric definitions, and a small command budget. CoDev normalizes any semantic skip or omission to a conservative failure and records a warning. - -## Deliberate first-release constraints - -- No persistent or cross-run cache. -- No network or hosted-repository metadata checks; branch protection and backlog health are judged only from local evidence and should fail when they cannot be established. -- Component discovery is manifest-based and deterministic. It is report metadata only and does not alter criterion weighting. -- Deterministic rules favor high-confidence applicability. Ambiguous criteria remain semantic rather than being skipped heuristically. - -## Validation - -- Fixture tests cover environment-template variants, repositories without environment requirements, GitHub/GitLab convention variants, API/database applicability, and denominator parity when an agent skips everything. -- End-to-end parity runs must use the same working tree and rubric version across all three providers. The evaluated count must match exactly; score differences are then semantic disagreements that can be measured and refined. From d261388e628c985273748ce2d8757d2526e3ac72 Mon Sep 17 00:00:00 2001 From: Bao Tran Date: Wed, 15 Jul 2026 16:19:04 +0700 Subject: [PATCH 05/14] test: make readiness codex + PATH-shim checks CI-deterministic - Mock spawnSync so the Codex availability probe is deterministic on CI runners that don't install a codex binary (was throwing 'not configured'). - Use the OS path delimiter in the shim-strip PATH test so it passes on Windows (was hardcoding ':' and POSIX paths). --- tests/lib/readiness-contract.test.ts | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/tests/lib/readiness-contract.test.ts b/tests/lib/readiness-contract.test.ts index 54a95f9..5d4de70 100644 --- a/tests/lib/readiness-contract.test.ts +++ b/tests/lib/readiness-contract.test.ts @@ -1,5 +1,5 @@ import { homedir } from "node:os"; -import { join } from "node:path"; +import { delimiter, join } from "node:path"; import { describe, expect, it, vi } from "vitest"; import * as configure from "@/lib/configure.js"; import { gatewayToolForReadiness } from "@/lib/readiness.js"; @@ -25,6 +25,25 @@ import { validateReadinessOutput, } from "@/lib/readiness-contract.js"; +// assertReadinessPrerequisites() treats Codex as usable when a real `codex` +// binary is on PATH, probed via spawnSync("codex", ["--version"]). CI runners +// don't install codex, so mock the probe to keep that check deterministic: +// `codex` reports available (status 0); anything else reports unavailable. +vi.mock("node:child_process", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + spawnSync: vi.fn((command: string) => ({ + status: command === "codex" ? 0 : 1, + signal: null, + output: [], + pid: 0, + stdout: Buffer.from(""), + stderr: Buffer.from(""), + })), + }; +}); + function validOutput(): AgentReadinessOutput { return { rubricVersion: READINESS_RUBRIC_VERSION, @@ -199,11 +218,11 @@ describe("readiness contract", () => { join(homedir(), ".codev-hub", "bin"), "/opt/homebrew/bin", "/usr/bin", - ].join(":"), + ].join(delimiter), }, ); - expect(env.PATH).toBe("/opt/homebrew/bin:/usr/bin"); + expect(env.PATH).toBe(["/opt/homebrew/bin", "/usr/bin"].join(delimiter)); }); it("isolates Claude readiness from ambient provider credentials", () => { From 680884a797577ed5fb21b563f2be576d88a1370f Mon Sep 17 00:00:00 2001 From: Bao Tran Date: Wed, 15 Jul 2026 16:53:20 +0700 Subject: [PATCH 06/14] fix: strip legacy ~/.codev/bin shims from PATH so readiness detects all agents - stripShimDirFromPath only stripped ~/.codev-hub/bin (current shim dir), not ~/.codev/bin (pre-0.4 shim dir). Old shims there exec the bare `codev` command which no longer exists, so isAgentAvailable resolved the stale shim instead of the real binary for codex/opencode, marking them "(unavailable)" in the readiness selector. With only one available agent arrow-key navigation cycled back to the same row, appearing unresponsive. - Add regression tests for legacy dir stripping and arrow-key navigation. --- src/lib/const.ts | 1 - src/lib/shims.ts | 12 +++++- tests/ReadinessNav.test.tsx | 77 +++++++++++++++++++++++++++++++++++++ tests/lib/shims.test.ts | 21 ++++++++++ 4 files changed, 108 insertions(+), 3 deletions(-) create mode 100644 tests/ReadinessNav.test.tsx diff --git a/src/lib/const.ts b/src/lib/const.ts index db13462..f353bf2 100644 --- a/src/lib/const.ts +++ b/src/lib/const.ts @@ -4,7 +4,6 @@ import { join } from "node:path"; import pkg from "../../package.json" with { type: "json" }; const BASE_URL = atob("aHR0cHM6Ly9uZXRtaW5kLnZpZXR0ZWwudm4="); -// Local dev override: export const BACKEND_URL = `http://localhost:8787`; export const BACKEND_URL = `${BASE_URL}/codev-backend`; export const SSO_URL = `${BASE_URL}/sso-wrapper`; export const LOGIN_SUCCESS_URL = `${BASE_URL}/codev-landing-page/oauth/success`; diff --git a/src/lib/shims.ts b/src/lib/shims.ts index 87251f9..07eb023 100644 --- a/src/lib/shims.ts +++ b/src/lib/shims.ts @@ -64,6 +64,14 @@ export function shimDir(): string { return join(homedir(), ".codev-hub", "bin"); } +// Pre-0.4 hub versions wrote shims to ~/.codev/bin (now ~/.codev-hub/bin). +// Those shims exec the bare `codev` command which no longer exists, so leaving +// them on PATH makes isAgentAvailable and runAgent resolve the stale shim +// instead of the real binary. +function legacyShimDirs(): string[] { + return [join(homedir(), ".codev", "bin")]; +} + // run.ts uses this to strip our shim dir from the child's PATH so that // `codevhub claude` -> spawn("claude") resolves the real npm-installed binary // instead of recursing through the shim. @@ -72,10 +80,10 @@ export function stripShimDirFromPath( sep = delimiter, ): string { if (!path) return ""; - const dir = shimDir(); + const excluded = new Set([shimDir(), ...legacyShimDirs()]); return path .split(sep) - .filter((p) => p !== dir) + .filter((p) => !excluded.has(p)) .join(sep); } diff --git a/tests/ReadinessNav.test.tsx b/tests/ReadinessNav.test.tsx new file mode 100644 index 0000000..765a0f6 --- /dev/null +++ b/tests/ReadinessNav.test.tsx @@ -0,0 +1,77 @@ +import { render } from "ink-testing-library"; +import { describe, expect, it, vi } from "vitest"; +import { ReadinessApp } from "@/ReadinessApp.js"; + +async function settle(ms = 30) { + await new Promise((resolve) => setTimeout(resolve, ms)); +} + +describe("ReadinessApp arrow-key navigation", () => { + it("moves cursor down with down-arrow and selects the highlighted agent on Enter", async () => { + const run = vi.fn(async () => ({ + exitCode: 0, + message: "ok", + })); + const { stdin, frames } = render( + , + ); + + // Cursor starts on Claude (index 0). Press down-arrow to move to Codex. + stdin.write("\x1b[B"); + await settle(); + expect(frames.at(-1)).toContain("Codex"); + // The cursor highlight (bold) should now be on Codex, not Claude. + // ink-testing-library strips ANSI, so we check via the selection callback. + stdin.write("\r"); + await settle(100); + expect(run).toHaveBeenCalledWith("codex", expect.any(Function), {}); + }); + + it("moves cursor up with up-arrow from the middle item", async () => { + const run = vi.fn(async () => ({ + exitCode: 0, + message: "ok", + })); + const { stdin } = render( + , + ); + + // Move down twice (claude -> codex -> opencode), then up once (opencode -> codex). + stdin.write("\x1b[B"); + await settle(); + stdin.write("\x1b[B"); + await settle(); + stdin.write("\x1b[A"); + await settle(); + stdin.write("\r"); + await settle(100); + expect(run).toHaveBeenCalledWith("codex", expect.any(Function), {}); + }); + + it("skips unavailable agents when navigating", async () => { + const run = vi.fn(async () => ({ + exitCode: 0, + message: "ok", + })); + const { stdin } = render( + , + ); + + // Cursor starts on Claude (index 0). Down-arrow should skip Codex (unavailable) + // and land on OpenCode (index 2). + stdin.write("\x1b[B"); + await settle(); + stdin.write("\r"); + await settle(100); + expect(run).toHaveBeenCalledWith("opencode", expect.any(Function), {}); + }); +}); diff --git a/tests/lib/shims.test.ts b/tests/lib/shims.test.ts index 0da7cee..2d336f1 100644 --- a/tests/lib/shims.test.ts +++ b/tests/lib/shims.test.ts @@ -164,6 +164,27 @@ describe("stripShimDirFromPath", () => { ["/usr/local/bin", "/usr/bin"].join(delimiter), ); }); + + test.skipIf(process.platform === "win32")( + "also strips the legacy ~/.codev/bin shim dir from pre-0.4 installs", + async () => { + const { stripShimDirFromPath } = await import("@/lib/shims.js"); + const legacy = join(tempDir, ".codev", "bin"); + const path = ["/usr/local/bin", legacy, "/usr/bin"].join(":"); + expect(stripShimDirFromPath(path, ":")).toBe("/usr/local/bin:/usr/bin"); + }, + ); + + test.skipIf(process.platform === "win32")( + "strips both current and legacy shim dirs simultaneously", + async () => { + const { shimDir, stripShimDirFromPath } = await import("@/lib/shims.js"); + const dir = shimDir(); + const legacy = join(tempDir, ".codev", "bin"); + const path = [dir, "/usr/bin", legacy, "/usr/local/bin"].join(":"); + expect(stripShimDirFromPath(path, ":")).toBe("/usr/bin:/usr/local/bin"); + }, + ); }); describe.skipIf(process.platform === "win32")("installShims (Unix)", () => { From 4e7b9e0949e5f5bc00052f4439f23aaf03c784e8 Mon Sep 17 00:00:00 2001 From: Bao Tran Date: Thu, 16 Jul 2026 13:24:42 +0700 Subject: [PATCH 07/14] feat: add versioned readiness profile selection --- src/ReadinessApp.tsx | 158 +++++++-- src/components/ReadinessProfileSelect.tsx | 65 ++++ src/index.tsx | 25 +- src/lib/help.ts | 2 + src/lib/readiness-agent.ts | 31 +- src/lib/readiness-cli.ts | 35 ++ src/lib/readiness-contract.ts | 31 +- src/lib/readiness-plan.ts | 86 +++-- src/lib/readiness-profile.ts | 284 +++++++++++++++++ src/lib/readiness-rules.ts | 371 ++++++++++++++++++++++ src/lib/readiness.ts | 106 +++++-- tests/ReadinessApp.test.tsx | 30 +- tests/ReadinessNav.test.tsx | 46 ++- tests/lib/readiness-cli.test.ts | 27 ++ tests/lib/readiness-profile.test.ts | 42 +++ tests/lib/readiness-rules.test.ts | 89 ++++++ tests/lib/readiness-upload.test.ts | 117 +++++++ 17 files changed, 1444 insertions(+), 101 deletions(-) create mode 100644 src/components/ReadinessProfileSelect.tsx create mode 100644 src/lib/readiness-cli.ts create mode 100644 src/lib/readiness-profile.ts create mode 100644 src/lib/readiness-rules.ts create mode 100644 tests/lib/readiness-cli.test.ts create mode 100644 tests/lib/readiness-profile.test.ts create mode 100644 tests/lib/readiness-rules.test.ts create mode 100644 tests/lib/readiness-upload.test.ts diff --git a/src/ReadinessApp.tsx b/src/ReadinessApp.tsx index 2806e99..de25a24 100644 --- a/src/ReadinessApp.tsx +++ b/src/ReadinessApp.tsx @@ -1,12 +1,16 @@ import { Box, Text, useApp } from "ink"; import Spinner from "ink-spinner"; -import { useCallback, useEffect, useMemo, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { Banner } from "@/components/Banner.js"; import { Frame } from "@/components/Frame.js"; import { ReadinessAgentSelect, readinessAgentSelectTitle, } from "@/components/ReadinessAgentSelect.js"; +import { + ReadinessProfileSelect, + readinessProfileSelectTitle, +} from "@/components/ReadinessProfileSelect.js"; import { Step } from "@/components/Step.js"; import { type ReadinessOptions, @@ -18,19 +22,38 @@ import { READINESS_AGENTS, type ReadinessAgent, } from "@/lib/readiness-agent.js"; +import { + fetchReadinessProfiles, + type ReadinessProfile, + type ReadinessProfileSession, + selectReadinessProfile, +} from "@/lib/readiness-profile.js"; -type Phase = "select" | "running" | "done" | "failed"; +type Phase = + | "loading-profiles" + | "select-profile" + | "select-agent" + | "running" + | "done" + | "failed"; +const EMPTY_READINESS_OPTIONS: ReadinessOptions = {}; interface ReadinessAppProps { available?: Record; run?: typeof runReadiness; options?: ReadinessOptions; + profileSelector?: string; + requestedAgent?: ReadinessAgent; + loadProfiles?: typeof fetchReadinessProfiles; } export function ReadinessApp({ available, run = runReadiness, - options = {}, + options = EMPTY_READINESS_OPTIONS, + profileSelector, + requestedAgent, + loadProfiles = fetchReadinessProfiles, }: ReadinessAppProps) { const { exit } = useApp(); const detected = useMemo( @@ -41,19 +64,39 @@ export function ReadinessApp({ ) as Record), [available], ); - const [phase, setPhase] = useState("select"); + const [phase, setPhase] = useState("loading-profiles"); const [agent, setAgent] = useState(null); - const [progress, setProgress] = useState("Preparing readiness scan"); + const [progress, setProgress] = useState("Loading readiness profiles"); const [result, setResult] = useState(null); + const [session, setSession] = useState(null); + const [profile, setProfile] = useState(null); + const profileFetchMs = useRef(0); const hasAvailableAgent = READINESS_AGENTS.some( (candidate) => detected[candidate], ); - const selectAgent = useCallback( - (choice: ReadinessAgent) => { + const startRun = useCallback( + ( + choice: ReadinessAgent, + chosen: ReadinessProfile, + loaded: ReadinessProfileSession, + ) => { + if (!detected[choice]) { + setResult({ + exitCode: 1, + message: `${choice} is not available on PATH.`, + }); + setPhase("failed"); + return; + } setAgent(choice); setPhase("running"); - run(choice, setProgress, options) + run(choice, setProgress, { + ...options, + profile: chosen, + auth: loaded.auth, + profileFetchMs: profileFetchMs.current, + }) .then((next) => { setResult(next); setPhase(next.exitCode === 0 ? "done" : "failed"); @@ -66,9 +109,52 @@ export function ReadinessApp({ setPhase("failed"); }); }, - [run, options], + [detected, run, options], + ); + const chooseProfile = useCallback( + (chosen: ReadinessProfile, loaded = session) => { + if (!loaded) return; + setProfile(chosen); + if (requestedAgent) startRun(requestedAgent, chosen, loaded); + else setPhase("select-agent"); + }, + [requestedAgent, session, startRun], + ); + const selectAgent = useCallback( + (choice: ReadinessAgent) => { + if (profile && session) startRun(choice, profile, session); + }, + [profile, session, startRun], ); + useEffect(() => { + let active = true; + const started = Date.now(); + loadProfiles(setProgress) + .then((loaded) => { + if (!active) return; + profileFetchMs.current = Date.now() - started; + setSession(loaded); + const chosen = selectReadinessProfile(loaded.profiles, profileSelector); + if (chosen) { + setProfile(chosen); + if (requestedAgent) startRun(requestedAgent, chosen, loaded); + else setPhase("select-agent"); + } else setPhase("select-profile"); + }) + .catch((error) => { + if (!active) return; + setResult({ + exitCode: 1, + message: error instanceof Error ? error.message : String(error), + }); + setPhase("failed"); + }); + return () => { + active = false; + }; + }, [loadProfiles, profileSelector, requestedAgent, startRun]); + useEffect(() => { if (phase !== "done") return; const timer = setTimeout(() => exit(), 50); @@ -79,24 +165,52 @@ export function ReadinessApp({ - {!hasAvailableAgent && ( + {phase === "loading-profiles" && ( + + + + + {` ${progress}...`} + + )} + {phase !== "loading-profiles" && !hasAvailableAgent && ( No supported coding agent is available. Run `codevhub install` first. )} - - - - {phase !== "select" && ( + {session && profile === null && phase === "select-profile" && ( + + + + )} + {profile && ( + + {}} + /> + + )} + {phase !== "loading-profiles" && phase !== "select-profile" && ( + + + + )} + {["running", "done", "failed"].includes(phase) && profile && ( Evaluate repository} diff --git a/src/components/ReadinessProfileSelect.tsx b/src/components/ReadinessProfileSelect.tsx new file mode 100644 index 0000000..0d2ae96 --- /dev/null +++ b/src/components/ReadinessProfileSelect.tsx @@ -0,0 +1,65 @@ +import { Box, Text, useInput } from "ink"; +import { useState } from "react"; +import type { ReadinessProfile } from "@/lib/readiness-profile.js"; + +interface ReadinessProfileSelectProps { + profiles: ReadinessProfile[]; + selected?: ReadinessProfile | null; + readOnly?: boolean; + onSelect: (profile: ReadinessProfile) => void; +} + +export function ReadinessProfileSelect({ + profiles, + selected = null, + readOnly = false, + onSelect, +}: ReadinessProfileSelectProps) { + const [cursor, setCursor] = useState(0); + useInput( + (_input, key) => { + if (key.upArrow || key.downArrow) { + const direction = key.upArrow ? -1 : 1; + setCursor( + (current) => + (current + direction + profiles.length) % profiles.length, + ); + } else if (key.return) { + const choice = profiles[cursor]; + if (choice) onSelect(choice); + } + }, + { isActive: !readOnly && profiles.length > 0 }, + ); + return ( + + {profiles.map((profile, index) => { + const chosen = selected?.activeVersion.id === profile.activeVersion.id; + return ( + + + {chosen ? "●" : "○"} + + + + {profile.name} r{profile.activeVersion.revision} + {profile.isDefault ? " (default)" : ""} + + + ); + })} + + ); +} + +export function readinessProfileSelectTitle(readOnly = false) { + return ( + + {"Choose readiness profile "} + {!readOnly && (↑/↓ to move, Enter to confirm)} + + ); +} diff --git a/src/index.tsx b/src/index.tsx index e153cbe..626d7ff 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -11,6 +11,10 @@ import { forwardToCodegraph } from "@/lib/codegraph.js"; import { printHelp, printVersion } from "@/lib/help.js"; import { initLogging } from "@/lib/log.js"; import { runLogs } from "@/lib/logs.js"; +import { + parseReadinessArgs, + type ReadinessCliOptions, +} from "@/lib/readiness-cli.js"; import { ensureNodeSqliteOrReexec } from "@/lib/reexec.js"; import { ensureFreshGatewayKey } from "@/lib/refresh.js"; import { @@ -237,13 +241,24 @@ switch (command) { break; } case "readiness": { - const modelIndex = args.indexOf("--model"); - const model = modelIndex >= 0 ? args[modelIndex + 1] : undefined; - if ((modelIndex >= 0 && !model) || args.length !== (model ? 2 : 0)) { - console.error("Usage: codev readiness [--model ]"); + let readiness: ReadinessCliOptions; + try { + readiness = parseReadinessArgs(args); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + console.error( + "Usage: codevhub readiness [--profile ] [--agent ] [--model ]", + ); process.exit(1); + break; } - const { waitUntilExit } = render(); + const { waitUntilExit } = render( + , + ); try { await waitUntilExit(); process.exit(0); diff --git a/src/lib/help.ts b/src/lib/help.ts index 174da0e..dfeaecb 100644 --- a/src/lib/help.ts +++ b/src/lib/help.ts @@ -23,6 +23,8 @@ Hub commands: (--force, -f re-uploads every conversation) model Switch the default model readiness Analyze and upload readiness for the current repository + (--profile , --agent , + --model for Claude Code or Codex) restore [agent] Restore an agent's pre-CoDev config (no arg processes every agent) logs Show the last run from the diagnostic log diff --git a/src/lib/readiness-agent.ts b/src/lib/readiness-agent.ts index 2e75167..1c467bd 100644 --- a/src/lib/readiness-agent.ts +++ b/src/lib/readiness-agent.ts @@ -133,7 +133,18 @@ export function claudeReadinessEnvOverrides(): NodeJS.ProcessEnv { export function buildReadinessPrompt(plan?: ReadinessEvaluationPlan): string { const semanticIds = plan ? new Set(semanticCriterionIds(plan)) : undefined; const rubric = semanticIds - ? READINESS_RUBRIC.filter(({ id }) => semanticIds.has(id)) + ? (plan?.definitions + .filter(({ key }) => semanticIds.has(key)) + .map((criterion) => ({ + id: criterion.key, + name: criterion.name, + category: criterion.category, + maturityLevel: criterion.maturityLevel, + description: criterion.description, + passCondition: criterion.passCondition, + evidenceRequired: criterion.evidenceRequirement, + semanticDecision: criterion.decision, + })) ?? []) : READINESS_RUBRIC; return `You are evaluating how ready a software repository is for autonomous coding agents. @@ -145,16 +156,18 @@ Evaluate every semantic rubric criterion below. Semantic criteria must be pass o Return only the JSON object matching the supplied schema. Do not include aggregate scores. -Rubric version: ${READINESS_RUBRIC_VERSION} +Rubric version: ${plan?.analyzerVersion ?? READINESS_RUBRIC_VERSION} Fresh deterministic plan: ${plan ? readinessPlanPrompt(plan) : "No deterministic plan supplied."} Semantic rubric: ${JSON.stringify(rubric, null, 2)}`; } -export function openCodeStructuredOutputInstruction(): string { +export function openCodeStructuredOutputInstruction( + rubricVersion = READINESS_RUBRIC_VERSION, +): string { return `Return exactly one JSON object with this shape and no markdown fence: -{"rubricVersion":"${READINESS_RUBRIC_VERSION}","languages":["string"],"applications":[{"path":".","description":"string","languages":["string"]}],"criteria":{"":{"status":"pass|fail|skipped","numerator":"integer or null","denominator":"positive integer","rationale":"string","evidence":["existing repository-relative path"]}},"warnings":["string"],"recommendations":["2 or 3 strings"],"model":"string or null"} +{"rubricVersion":"${rubricVersion}","languages":["string"],"applications":[{"path":".","description":"string","languages":["string"]}],"criteria":{"":{"status":"pass|fail|skipped","numerator":"integer or null","denominator":"positive integer","rationale":"string","evidence":["existing repository-relative path"]}},"warnings":["string"],"recommendations":["2 or 3 strings"],"model":"string or null"} Use null numerator only for skipped criteria. Include every criterion listed in the Semantic rubric exactly once and omit fixed-decision criteria.`; } @@ -165,6 +178,7 @@ export function buildAgentCommand( outputPath: string, sessionId?: string, modelOverride?: string, + rubricVersion = READINESS_RUBRIC_VERSION, ): { command: string; args: string[] } { if (agent === "claude") { const args = [ @@ -238,7 +252,7 @@ export function buildAgentCommand( "json", "-m", readinessRuntimeConfig().opencodeModel, - `${prompt}\n\n${openCodeStructuredOutputInstruction()}`, + `${prompt}\n\n${openCodeStructuredOutputInstruction(rubricVersion)}`, ]; if (sessionId) args.push("--session", sessionId); return { command: "opencode", args }; @@ -400,7 +414,10 @@ export async function runReadinessAgent( writeFileSync( schemaPath, JSON.stringify( - readinessJsonSchema(plan ? semanticCriterionIds(plan) : undefined), + readinessJsonSchema( + plan ? semanticCriterionIds(plan) : undefined, + plan?.analyzerVersion, + ), ), ); let envOverrides: NodeJS.ProcessEnv = {}; @@ -454,6 +471,7 @@ export async function runReadinessAgent( outputPath, agent === "opencode" ? undefined : repair?.sessionId, modelOverride, + plan?.analyzerVersion, ); let result = await runProcess( command.command, @@ -474,6 +492,7 @@ export async function runReadinessAgent( outputPath, sessionId, modelOverride, + plan?.analyzerVersion, ); result = await runProcess( continuation.command, diff --git a/src/lib/readiness-cli.ts b/src/lib/readiness-cli.ts new file mode 100644 index 0000000..4681da2 --- /dev/null +++ b/src/lib/readiness-cli.ts @@ -0,0 +1,35 @@ +import { + READINESS_AGENTS, + type ReadinessAgent, +} from "@/lib/readiness-agent.js"; + +export interface ReadinessCliOptions { + model?: string; + profile?: string; + agent?: ReadinessAgent; +} + +export function parseReadinessArgs(args: string[]): ReadinessCliOptions { + const parsed: ReadinessCliOptions = {}; + for (let index = 0; index < args.length; index++) { + const raw = args[index] ?? ""; + const equal = raw.indexOf("="); + const flag = equal >= 0 ? raw.slice(0, equal) : raw; + const inline = equal >= 0 ? raw.slice(equal + 1) : undefined; + if (!["--model", "--profile", "--agent"].includes(flag)) + throw new Error(`Unknown readiness option: ${raw}.`); + const value = inline ?? args[++index]; + if (!value || value.startsWith("--")) + throw new Error(`Missing value for ${flag}.`); + const field = flag.slice(2) as keyof ReadinessCliOptions; + if (parsed[field]) throw new Error(`Duplicate readiness option: ${flag}.`); + if (field === "agent") { + if (!(READINESS_AGENTS as readonly string[]).includes(value)) + throw new Error(`Unknown readiness agent: ${value}.`); + parsed.agent = value as ReadinessAgent; + } else parsed[field] = value; + } + if (parsed.agent === "opencode" && parsed.model) + throw new Error("--model cannot be used with OpenCode readiness runs."); + return parsed; +} diff --git a/src/lib/readiness-contract.ts b/src/lib/readiness-contract.ts index 42fd4b7..ea6b4c3 100644 --- a/src/lib/readiness-contract.ts +++ b/src/lib/readiness-contract.ts @@ -1,4 +1,4 @@ -import { existsSync } from "node:fs"; +import { existsSync, lstatSync, realpathSync } from "node:fs"; import { isAbsolute, relative, resolve } from "node:path"; // This is a protocol/schema identifier, not runtime configuration. Change it @@ -365,7 +365,6 @@ export const READINESS_RUBRIC: ReadinessCriterionDefinition[] = ); export const READINESS_CRITERION_IDS = READINESS_RUBRIC.map(({ id }) => id); -const CRITERION_SET = new Set(READINESS_CRITERION_IDS); export interface ReadinessSummary { criteriaPassed: number; @@ -414,9 +413,17 @@ function evidencePath(evidence: string): string { function isValidEvidence(root: string, evidence: string): boolean { const path = evidencePath(evidence); - return Boolean( - path && isInside(root, path) && existsSync(resolve(root, path)), - ); + if (!path || !isInside(root, path) || !existsSync(resolve(root, path))) + return false; + try { + const target = resolve(root, path); + return ( + !lstatSync(target).isSymbolicLink() && + isInside(realpathSync(root), realpathSync(target)) + ); + } catch { + return false; + } } export function normalizeReadinessEvidence( @@ -442,13 +449,15 @@ export function normalizeReadinessEvidence( export function validateReadinessOutput( value: unknown, root: string, + criterionIds: string[] = READINESS_CRITERION_IDS, + rubricVersion = READINESS_RUBRIC_VERSION, ): string[] { const errors: string[] = []; if (!value || typeof value !== "object" || Array.isArray(value)) return ["Output must be a JSON object."]; const output = value as Partial; - if (output.rubricVersion !== READINESS_RUBRIC_VERSION) - errors.push(`rubricVersion must be ${READINESS_RUBRIC_VERSION}.`); + if (output.rubricVersion !== rubricVersion) + errors.push(`rubricVersion must be ${rubricVersion}.`); if ( !Array.isArray(output.languages) || output.languages.some((v) => typeof v !== "string") @@ -487,9 +496,10 @@ export function validateReadinessOutput( ) { errors.push("criteria must be an object."); } else { + const criterionSet = new Set(criterionIds); for (const id of Object.keys(output.criteria)) - if (!CRITERION_SET.has(id)) errors.push(`Unknown criterion: ${id}.`); - for (const id of READINESS_CRITERION_IDS) { + if (!criterionSet.has(id)) errors.push(`Unknown criterion: ${id}.`); + for (const id of criterionIds) { const item = output.criteria[id]; if (!item) { errors.push(`Missing criterion: ${id}.`); @@ -542,6 +552,7 @@ export function validateReadinessOutput( export function readinessJsonSchema( criterionIds: string[] = READINESS_CRITERION_IDS, + rubricVersion = READINESS_RUBRIC_VERSION, ): Record { const criterionSchema = { type: "object", @@ -570,7 +581,7 @@ export function readinessJsonSchema( properties: { rubricVersion: { type: "string", - const: READINESS_RUBRIC_VERSION, + const: rubricVersion, }, languages: { type: "array", items: { type: "string" } }, applications: { diff --git a/src/lib/readiness-plan.ts b/src/lib/readiness-plan.ts index 3c4a92b..8c767f6 100644 --- a/src/lib/readiness-plan.ts +++ b/src/lib/readiness-plan.ts @@ -4,13 +4,20 @@ import { basename, dirname, extname } from "node:path"; import { type AgentReadinessOutput, READINESS_CRITERION_IDS, - READINESS_RUBRIC_VERSION, type ReadinessApplication, type ReadinessCriterionResult, } from "@/lib/readiness-contract.js"; +import { + bundledStandardProfile, + enabledProfileCriteria, + isStandardProfile, + type ReadinessCriterionConfig, + type ReadinessProfile, +} from "@/lib/readiness-profile.js"; +import { evaluateConfiguredCriterion } from "@/lib/readiness-rules.js"; export type ReadinessDecision = - | { mode: "semantic"; reason: string } + | { mode: "semantic"; reason: string; evidence?: string[] } | { mode: "not_applicable"; result: ReadinessCriterionResult } | { mode: "deterministic"; result: ReadinessCriterionResult }; @@ -29,6 +36,9 @@ export interface ReadinessRepositoryProfile { export interface ReadinessEvaluationPlan { profile: ReadinessRepositoryProfile; criteria: Record; + criteriaOrder: string[]; + definitions: ReadinessCriterionConfig[]; + analyzerVersion: string; } const IGNORED_COMPONENT_SEGMENTS = new Set([ @@ -201,6 +211,7 @@ function applications( export function buildReadinessEvaluationPlan( root: string, + selectedProfile: ReadinessProfile = bundledStandardProfile(), ): ReadinessEvaluationPlan { const trackedFiles = listedFiles(root, ["--cached"]); const files = listedFiles(root, [ @@ -623,24 +634,63 @@ export function buildReadinessEvaluationPlan( } } + const repositoryProfile = { + files, + trackedFiles, + relevantFiles, + languages, + applications: applications(files, languages), + hasEnvironmentUsage, + hasDatabaseSurface, + hasApiSurface, + hasLocalServiceRequirement, + }; + const definitions = enabledProfileCriteria(selectedProfile); + if (definitions.length === 0) + throw new Error("The selected readiness profile has no enabled criteria."); + const selectedCriteria: Record = {}; + if (isStandardProfile(selectedProfile)) { + for (const definition of definitions) { + const decision = criteria[definition.key]; + if (!decision) + throw new Error( + `Standard profile references unsupported criterion: ${definition.key}.`, + ); + selectedCriteria[definition.key] = decision; + } + } else { + for (const definition of definitions) + selectedCriteria[definition.key] = evaluateConfiguredCriterion( + definition, + { + root, + files, + trackedFiles, + }, + ); + } + const discoveredEvidence = Object.values(selectedCriteria).flatMap( + (decision) => + decision.mode === "semantic" + ? (decision.evidence ?? []) + : decision.result.evidence, + ); return { profile: { - files, - trackedFiles, - relevantFiles, - languages, - applications: applications(files, languages), - hasEnvironmentUsage, - hasDatabaseSurface, - hasApiSurface, - hasLocalServiceRequirement, + ...repositoryProfile, + relevantFiles: [ + ...new Set([...repositoryProfile.relevantFiles, ...discoveredEvidence]), + ].slice(0, 800), }, - criteria, + criteria: selectedCriteria, + criteriaOrder: definitions.map((criterion) => criterion.key), + definitions, + analyzerVersion: selectedProfile.activeVersion.analyzerVersion, }; } export function semanticCriterionIds(plan: ReadinessEvaluationPlan): string[] { - return READINESS_CRITERION_IDS.filter( + return plan.criteriaOrder.filter( (id) => plan.criteria[id]?.mode === "semantic", ); } @@ -651,7 +701,7 @@ export function finalizeReadinessOutput( ): AgentReadinessOutput { const warnings = [...(Array.isArray(output.warnings) ? output.warnings : [])]; const criteria = Object.fromEntries( - READINESS_CRITERION_IDS.map((id) => { + plan.criteriaOrder.map((id) => { const decision = plan.criteria[id]; if (decision?.mode !== "semantic") return [id, decision?.result]; const agentResult = output.criteria?.[id]; @@ -681,7 +731,7 @@ export function finalizeReadinessOutput( ) as Record; return { ...output, - rubricVersion: READINESS_RUBRIC_VERSION, + rubricVersion: plan.analyzerVersion, languages: plan.profile.languages, applications: plan.profile.applications, criteria, @@ -701,9 +751,9 @@ export function finalizeReadinessOutput( export function readinessPlanPrompt(plan: ReadinessEvaluationPlan): string { const semantic = semanticCriterionIds(plan); const fixed = Object.fromEntries( - READINESS_CRITERION_IDS.filter( - (id) => plan.criteria[id]?.mode !== "semantic", - ).map((id) => [id, plan.criteria[id]]), + plan.criteriaOrder + .filter((id) => plan.criteria[id]?.mode !== "semantic") + .map((id) => [id, plan.criteria[id]]), ); return JSON.stringify({ profile: { diff --git a/src/lib/readiness-profile.ts b/src/lib/readiness-profile.ts new file mode 100644 index 0000000..026fe72 --- /dev/null +++ b/src/lib/readiness-profile.ts @@ -0,0 +1,284 @@ +import type { AuthData } from "@/lib/auth.js"; +import { login } from "@/lib/auth.js"; +import { BACKEND_URL } from "@/lib/const.js"; +import { loggedFetch } from "@/lib/log.js"; +import { + READINESS_RUBRIC, + READINESS_RUBRIC_VERSION, +} from "@/lib/readiness-contract.js"; +import { validateProfileRules } from "@/lib/readiness-rules.js"; + +const PROFILE_TIMEOUT_MS = 10_000; +const MAX_CRITERIA = 200; +const KEY = /^[a-z][a-z0-9_]{0,63}$/; + +export interface ReadinessCriterionConfig { + key: string; + name: string; + category: string; + description: string; + maturityLevel: number; + repositoryScope: string; + enabled: boolean; + order: number; + passCondition: string; + evidenceRequirement: string; + applicability: unknown; + evidenceLocators: unknown[]; + decision: unknown; + recommendationTemplate: string; + priority: number; +} + +export interface ReadinessProfileDefinition { + criteria: ReadinessCriterionConfig[]; +} + +export interface ReadinessProfileVersion { + id: string; + revision: number; + contentHash: string; + schemaVersion: string; + analyzerVersion: string; + definition: ReadinessProfileDefinition; +} + +export interface ReadinessProfile { + id: string; + name: string; + slug: string; + description: string; + scope: string; + isDefault: boolean; + status: string; + activeVersion: ReadinessProfileVersion; +} + +export interface ReadinessProfileSession { + auth: AuthData; + profiles: ReadinessProfile[]; +} + +function record(value: unknown): Record | null { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : null; +} + +function text(value: unknown, field: string, max = 4_000): string { + if (typeof value !== "string" || !value.trim() || value.length > max) + throw new Error(`Readiness profile ${field} is invalid.`); + return value; +} + +function integer(value: unknown, field: string, min = 0): number { + if (!Number.isSafeInteger(value) || (value as number) < min) + throw new Error(`Readiness profile ${field} is invalid.`); + return value as number; +} + +function decodeCriterion(value: unknown): ReadinessCriterionConfig { + const item = record(value); + if (!item) throw new Error("Readiness profile criterion is invalid."); + const key = text(item.key, "criterion key", 64); + if (!KEY.test(key)) + throw new Error(`Readiness profile criterion key is unsafe: ${key}.`); + if (typeof item.enabled !== "boolean") + throw new Error(`Readiness profile ${key}.enabled is invalid.`); + if ( + !Array.isArray(item.evidenceLocators) || + item.evidenceLocators.length > 20 + ) + throw new Error(`Readiness profile ${key}.evidenceLocators is invalid.`); + if (!record(item.decision)) + throw new Error(`Readiness profile ${key}.decision is invalid.`); + return { + key, + name: text(item.name, `${key}.name`, 200), + category: text(item.category, `${key}.category`, 200), + description: text(item.description, `${key}.description`), + maturityLevel: integer(item.maturityLevel, `${key}.maturityLevel`, 1), + repositoryScope: text(item.repositoryScope, `${key}.repositoryScope`, 100), + enabled: item.enabled, + order: integer(item.order, `${key}.order`), + passCondition: text(item.passCondition, `${key}.passCondition`), + evidenceRequirement: text( + item.evidenceRequirement, + `${key}.evidenceRequirement`, + ), + applicability: item.applicability, + evidenceLocators: item.evidenceLocators, + decision: item.decision, + recommendationTemplate: text( + item.recommendationTemplate, + `${key}.recommendationTemplate`, + ), + priority: integer(item.priority, `${key}.priority`), + }; +} + +export function decodeReadinessProfile(value: unknown): ReadinessProfile { + const profile = record(value); + const version = record(profile?.activeVersion); + const definition = record(version?.definition); + if ( + !profile || + !version || + !definition || + !Array.isArray(definition.criteria) + ) + throw new Error("Readiness profile response is invalid."); + if ( + definition.criteria.length === 0 || + definition.criteria.length > MAX_CRITERIA + ) + throw new Error("Readiness profile criterion inventory is invalid."); + const criteria = definition.criteria.map(decodeCriterion); + const keys = new Set(); + for (const criterion of criteria) { + if (keys.has(criterion.key)) + throw new Error(`Duplicate readiness criterion: ${criterion.key}.`); + keys.add(criterion.key); + } + const decoded: ReadinessProfile = { + id: text(profile.id, "id", 200), + name: text(profile.name, "name", 200), + slug: text(profile.slug, "slug", 200), + description: + typeof profile.description === "string" ? profile.description : "", + scope: text(profile.scope, "scope", 100), + isDefault: profile.isDefault === true, + status: text(profile.status, "status", 100), + activeVersion: { + id: text(version.id, "version id", 200), + revision: integer(version.revision, "revision", 1), + contentHash: text(version.contentHash, "content hash", 256), + schemaVersion: text(version.schemaVersion, "schema version", 100), + analyzerVersion: text(version.analyzerVersion, "analyzer version", 100), + definition: { criteria }, + }, + }; + if (decoded.activeVersion.schemaVersion !== "1") + throw new Error( + `Unsupported readiness profile schema: ${decoded.activeVersion.schemaVersion}.`, + ); + if (decoded.activeVersion.analyzerVersion !== READINESS_RUBRIC_VERSION) + throw new Error( + `Unsupported readiness analyzer version: ${decoded.activeVersion.analyzerVersion}.`, + ); + if (!isStandardProfile(decoded)) validateProfileRules(decoded); + return decoded; +} + +export async function fetchReadinessProfiles( + onStatus: (message: string) => void = () => {}, +): Promise { + let authError = ""; + const auth = await login( + (message) => { + authError = message; + onStatus(message); + }, + () => {}, + ).catch((error) => { + throw new Error( + authError || (error instanceof Error ? error.message : String(error)), + ); + }); + onStatus("Loading accessible readiness profiles"); + const response = await loggedFetch( + "readiness.profiles", + `${BACKEND_URL}/readiness/profiles`, + { + headers: { authorization: `Bearer ${auth.access_token}` }, + signal: AbortSignal.timeout(PROFILE_TIMEOUT_MS), + }, + ); + if (!response.ok) + throw new Error( + `Readiness profile fetch failed (${response.status}): ${await response.text()}`, + ); + const body = (await response.json()) as unknown; + const values = Array.isArray(body) + ? body + : Array.isArray(record(body)?.profiles) + ? (record(body)?.profiles as unknown[]) + : null; + if (!values) throw new Error("Readiness profile response is invalid."); + const profiles = values.map(decodeReadinessProfile); + if (profiles.length === 0) + throw new Error( + "No published readiness profile is available to this user.", + ); + return { auth, profiles }; +} + +export function selectReadinessProfile( + profiles: ReadinessProfile[], + selector?: string, +): ReadinessProfile | undefined { + if (selector) { + const matches = profiles.filter( + (profile) => profile.id === selector || profile.slug === selector, + ); + if (matches.length !== 1) + throw new Error(`Readiness profile not found or ambiguous: ${selector}.`); + return matches[0]; + } + if (profiles.length === 1) return profiles[0]; + const defaults = profiles.filter((profile) => profile.isDefault); + return defaults.length === 1 ? defaults[0] : undefined; +} + +export function enabledProfileCriteria( + profile: ReadinessProfile, +): ReadinessCriterionConfig[] { + return profile.activeVersion.definition.criteria + .filter((criterion) => criterion.enabled) + .sort( + (left, right) => + left.order - right.order || left.key.localeCompare(right.key), + ); +} + +export function isStandardProfile(profile: ReadinessProfile): boolean { + return profile.slug === "standard" && profile.scope === "system"; +} + +export function bundledStandardProfile(): ReadinessProfile { + return { + id: "builtin:standard", + name: "Standard", + slug: "standard", + description: "Built-in CoDev Standard readiness profile.", + scope: "system", + isDefault: true, + status: "published", + activeVersion: { + id: `builtin:standard@${READINESS_RUBRIC_VERSION}`, + revision: 1, + contentHash: READINESS_RUBRIC_VERSION, + schemaVersion: "1", + analyzerVersion: READINESS_RUBRIC_VERSION, + definition: { + criteria: READINESS_RUBRIC.map((criterion, order) => ({ + key: criterion.id, + name: criterion.id.replaceAll("_", " "), + category: criterion.category, + description: criterion.description, + maturityLevel: criterion.maturityLevel, + repositoryScope: "repository", + enabled: true, + order, + passCondition: criterion.description, + evidenceRequirement: criterion.evidenceRequired, + applicability: { kind: "always" }, + evidenceLocators: [], + decision: { engine: "builtin" }, + recommendationTemplate: `Address ${criterion.id.replaceAll("_", " ")}.`, + priority: criterion.maturityLevel, + })), + }, + }, + }; +} diff --git a/src/lib/readiness-rules.ts b/src/lib/readiness-rules.ts new file mode 100644 index 0000000..2c61db5 --- /dev/null +++ b/src/lib/readiness-rules.ts @@ -0,0 +1,371 @@ +import { lstatSync, readFileSync, realpathSync } from "node:fs"; +import { isAbsolute, relative, resolve } from "node:path"; +import type { ReadinessCriterionResult } from "@/lib/readiness-contract.js"; +import type { + ReadinessCriterionConfig, + ReadinessProfile, +} from "@/lib/readiness-profile.js"; + +const MAX_RULE_DEPTH = 8; +const MAX_RULE_NODES = 100; +const MAX_PATTERN_LENGTH = 300; +const MAX_FILES_PER_LOCATOR = 200; +const MAX_EVIDENCE = 8; +const MAX_TEXT_BYTES = 256_000; +const MAX_TOTAL_TEXT_BYTES = 2_000_000; + +type Json = Record; + +export interface DeclarativeInventory { + root: string; + files: string[]; + trackedFiles: string[]; +} + +export type ConfiguredDecision = + | { mode: "semantic"; reason: string; evidence: string[] } + | { mode: "not_applicable"; result: ReadinessCriterionResult } + | { mode: "deterministic"; result: ReadinessCriterionResult }; + +function record(value: unknown): Json | null { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? (value as Json) + : null; +} + +function kind(value: Json): string | undefined { + return typeof value.kind === "string" ? value.kind : undefined; +} + +function boundedString(value: unknown, field: string): string { + if (typeof value !== "string" || !value || value.length > MAX_PATTERN_LENGTH) + throw new Error(`Readiness rule ${field} is invalid.`); + return value; +} + +function strings(value: Json, singular: string, plural: string): string[] { + const raw = Array.isArray(value[plural]) + ? value[plural] + : value[singular] === undefined + ? [] + : [value[singular]]; + if ( + raw.length === 0 || + raw.length > 20 || + raw.some((entry) => typeof entry !== "string") + ) + throw new Error(`Readiness rule ${plural} is invalid.`); + return (raw as string[]).map((entry) => boundedString(entry, plural)); +} + +function globRegex(glob: string): RegExp { + let source = "^"; + for (let index = 0; index < glob.length; index++) { + const char = glob[index] ?? ""; + if (char === "*") { + if (glob[index + 1] === "*") { + index++; + source += ".*"; + } else source += "[^/]*"; + } else if (char === "?") source += "[^/]"; + else source += char.replace(/[\\^$.*+?()[\]{}|]/g, "\\$&"); + } + return new RegExp(`${source}$`); +} + +function matchesGlobs(files: string[], patterns: string[]): string[] { + const compiled = patterns.map(globRegex); + return files + .filter((file) => compiled.some((pattern) => pattern.test(file))) + .slice(0, MAX_FILES_PER_LOCATOR); +} + +function inside(root: string, target: string): boolean { + const rel = relative(resolve(root), resolve(target)); + return rel === "" || (!rel.startsWith("..") && !isAbsolute(rel)); +} + +function safeText( + inventory: DeclarativeInventory, + file: string, + budget: { bytes: number }, +): string { + try { + const target = resolve(inventory.root, file); + if (!inside(inventory.root, target) || lstatSync(target).isSymbolicLink()) + return ""; + if (!inside(realpathSync(inventory.root), realpathSync(target))) return ""; + const content = readFileSync(target); + if ( + content.byteLength > MAX_TEXT_BYTES || + budget.bytes + content.byteLength > MAX_TOTAL_TEXT_BYTES || + content.includes(0) + ) + return ""; + budget.bytes += content.byteLength; + return content.toString("utf8"); + } catch { + return ""; + } +} + +function safeRegex(pattern: string, flags: string): RegExp { + boundedString(pattern, "pattern"); + if (!/^[im]*$/.test(flags)) + throw new Error("Readiness rule flags are invalid."); + // Reject constructs that are unnecessary for configuration matching and are + // common sources of catastrophic backtracking or surprising cross-file logic. + if (/\\[1-9]|\(\?[=!<]|\([^)]*[+*][^)]*\)[+*{]/.test(pattern)) + throw new Error("Readiness rule regex uses unsupported constructs."); + return new RegExp(pattern, flags); +} + +function nested(value: unknown, path: string): unknown { + let current = value; + for (const part of path.split(".")) { + const item = record(current); + if (!item || !(part in item)) return undefined; + current = item[part]; + } + return current; +} + +function predicateMatches( + value: unknown, + inventory: DeclarativeInventory, + budget: { bytes: number }, +): string[] { + const predicate = record(value); + if (!predicate || typeof predicate.type !== "string") + throw new Error("Readiness predicate is invalid."); + switch (predicate.type) { + case "tracked_path_exists": + return matchesGlobs( + inventory.trackedFiles, + predicate.path === undefined + ? strings(predicate, "pattern", "patterns") + : [boundedString(predicate.path, "path")], + ); + case "text_matches": { + const candidates = matchesGlobs( + inventory.trackedFiles, + predicate.path === undefined + ? strings(predicate, "filePattern", "filePatterns") + : [boundedString(predicate.path, "path")], + ); + const regex = safeRegex( + boundedString(predicate.pattern, "pattern"), + typeof predicate.flags === "string" ? predicate.flags : "", + ); + return candidates.filter((file) => + regex.test(safeText(inventory, file, budget)), + ); + } + case "manifest_script": + case "manifest_dependency": { + const name = boundedString(predicate.name, "name"); + const manifests = + predicate.manifest === undefined + ? inventory.trackedFiles.filter((file) => + /(^|\/)package\.json$/.test(file), + ) + : matchesGlobs(inventory.trackedFiles, [ + boundedString(predicate.manifest, "manifest"), + ]); + return manifests + .filter((file) => { + try { + const manifest = JSON.parse( + safeText(inventory, file, budget), + ) as Json; + if (predicate.type === "manifest_script") + return typeof record(manifest.scripts)?.[name] === "string"; + return [ + "dependencies", + "devDependencies", + "peerDependencies", + "optionalDependencies", + ].some((group) => name in (record(manifest[group]) ?? {})); + } catch { + return false; + } + }) + .slice(0, MAX_FILES_PER_LOCATOR); + } + case "manifest_setting": { + const manifest = boundedString( + predicate.manifest ?? predicate.filePattern, + "manifest", + ); + const setting = boundedString( + predicate.setting ?? predicate.path, + "path", + ); + return matchesGlobs(inventory.trackedFiles, [manifest]) + .filter((file) => { + try { + const value = nested( + JSON.parse(safeText(inventory, file, budget)), + setting, + ); + const hasExpected = "value" in predicate || "equals" in predicate; + const expected = + "value" in predicate ? predicate.value : predicate.equals; + return hasExpected + ? JSON.stringify(value) === JSON.stringify(expected) + : value !== undefined; + } catch { + return false; + } + }) + .slice(0, MAX_FILES_PER_LOCATOR); + } + default: + throw new Error(`Unsupported readiness predicate: ${predicate.type}.`); + } +} + +function evaluateRule( + value: unknown, + inventory: DeclarativeInventory, + budget: { nodes: number; bytes: number }, + depth = 0, +): boolean { + if (++budget.nodes > MAX_RULE_NODES || depth > MAX_RULE_DEPTH) + throw new Error("Readiness applicability rule exceeds safe bounds."); + const rule = record(value); + if (!rule) throw new Error("Readiness applicability rule is invalid."); + switch (kind(rule)) { + case "always": + return true; + case "predicate": + return ( + predicateMatches(rule.predicate ?? rule, inventory, budget).length > 0 + ); + case "all": + case "any": { + if ( + !Array.isArray(rule.rules) || + rule.rules.length === 0 || + rule.rules.length > 20 + ) + throw new Error("Readiness applicability composition is invalid."); + const values = rule.rules.map((child) => + evaluateRule(child, inventory, budget, depth + 1), + ); + return kind(rule) === "all" + ? values.every(Boolean) + : values.some(Boolean); + } + case "not": + return !evaluateRule(rule.rule, inventory, budget, depth + 1); + case "minimum_match": { + const minimum = rule.minimum ?? rule.count; + if (!Array.isArray(rule.rules) || !Number.isSafeInteger(minimum)) + throw new Error("Readiness minimum-match rule is invalid."); + const count = rule.rules.filter((child) => + evaluateRule(child, inventory, budget, depth + 1), + ).length; + return count >= (minimum as number); + } + default: + throw new Error( + `Unsupported readiness applicability kind: ${String(kind(rule))}.`, + ); + } +} + +function result( + status: "pass" | "fail" | "skipped", + rationale: string, + evidence: string[] = [], +): ReadinessCriterionResult { + return { + status, + numerator: status === "skipped" ? null : status === "pass" ? 1 : 0, + denominator: 1, + rationale, + evidence: evidence.slice(0, MAX_EVIDENCE), + }; +} + +export function evaluateConfiguredCriterion( + criterion: ReadinessCriterionConfig, + inventory: DeclarativeInventory, +): ConfiguredDecision { + const budget = { nodes: 0, bytes: 0 }; + if (!evaluateRule(criterion.applicability, inventory, budget)) + return { + mode: "not_applicable", + result: result( + "skipped", + `The applicability rule for ${criterion.name} did not match.`, + ), + }; + const evidence = [ + ...new Set( + criterion.evidenceLocators.flatMap((locator) => + predicateMatches(locator, inventory, budget), + ), + ), + ].slice(0, MAX_EVIDENCE); + const decision = record(criterion.decision); + const engine = + typeof decision?.engine === "string" ? decision.engine : undefined; + if (engine === "semantic") + return { + mode: "semantic", + reason: + typeof decision?.instructions === "string" + ? decision.instructions + : criterion.passCondition, + evidence, + }; + if (engine !== "deterministic") + throw new Error( + `Unsupported readiness decision engine for ${criterion.key}.`, + ); + const expected = decision?.expected === "absent" ? "absent" : "present"; + const passed = + expected === "present" ? evidence.length > 0 : evidence.length === 0; + const rationale = passed + ? typeof decision?.passRationale === "string" + ? decision.passRationale + : criterion.passCondition + : typeof decision?.failRationale === "string" + ? decision.failRationale + : `Required evidence was not established for ${criterion.name}.`; + return { + mode: "deterministic", + result: result(passed ? "pass" : "fail", rationale, evidence), + }; +} + +function validateConfiguredCriterion( + criterion: ReadinessCriterionConfig, + inventory: DeclarativeInventory, +): void { + const budget = { nodes: 0, bytes: 0 }; + evaluateRule(criterion.applicability, inventory, budget); + for (const locator of criterion.evidenceLocators) + predicateMatches(locator, inventory, budget); + const engine = record(criterion.decision)?.engine; + if (engine !== "deterministic" && engine !== "semantic") + throw new Error( + `Unsupported readiness decision engine for ${criterion.key}.`, + ); +} + +export function validateProfileRules(profile: ReadinessProfile): void { + const inventory: DeclarativeInventory = { + root: process.cwd(), + files: [], + trackedFiles: [], + }; + for (const criterion of profile.activeVersion.definition.criteria) { + if (!criterion.enabled) continue; + // Validation happens with an empty inventory and still walks every rule and + // locator, compiling all patterns before repository evaluation begins. + validateConfiguredCriterion(criterion, inventory); + } +} diff --git a/src/lib/readiness.ts b/src/lib/readiness.ts index 85776d0..6ab1e2f 100644 --- a/src/lib/readiness.ts +++ b/src/lib/readiness.ts @@ -10,14 +10,18 @@ import { import { readinessRuntimeConfig } from "@/lib/readiness-config.js"; import { normalizeReadinessEvidence, - READINESS_RUBRIC_VERSION, summarizeReadiness, validateReadinessOutput, } from "@/lib/readiness-contract.js"; import { buildReadinessEvaluationPlan, finalizeReadinessOutput, + semanticCriterionIds, } from "@/lib/readiness-plan.js"; +import { + bundledStandardProfile, + type ReadinessProfile, +} from "@/lib/readiness-profile.js"; import { ensureFreshGatewayKey } from "@/lib/refresh.js"; function git(args: string[], cwd: string): string { @@ -51,6 +55,9 @@ export interface ReadinessRunResult { export type ReadinessProgress = (message: string) => void; export interface ReadinessOptions { model?: string; + profile?: ReadinessProfile; + auth?: AuthData; + profileFetchMs?: number; } export function gatewayToolForReadiness( @@ -67,6 +74,8 @@ export async function runReadiness( options: ReadinessOptions = {}, ): Promise { const root = process.cwd(); + const totalStarted = Date.now(); + const selectedProfile = options.profile ?? bundledStandardProfile(); if (!git(["rev-parse", "--is-inside-work-tree"], root)) { return { exitCode: 1, @@ -85,20 +94,48 @@ export async function runReadiness( const before = repositoryState(root); onProgress("Building a fresh deterministic repository profile"); - const plan = buildReadinessEvaluationPlan(root); + const deterministicStarted = Date.now(); + let plan: ReturnType; + try { + plan = buildReadinessEvaluationPlan(root, selectedProfile); + } catch (error) { + return { + exitCode: 1, + message: error instanceof Error ? error.message : String(error), + }; + } + const deterministicMs = Date.now() - deterministicStarted; onProgress(`Evaluating semantic readiness criteria with ${agent}`); let run: AgentRunResult; + const semanticStarted = Date.now(); try { - const gatewayTool = gatewayToolForReadiness(agent); - if (gatewayTool) await ensureFreshGatewayKey(gatewayTool); - run = await runReadinessAgent( - agent, - root, - undefined, - options.model, - onProgress, - plan, - ); + if (semanticCriterionIds(plan).length === 0) { + run = { + provider: agent, + durationMs: 0, + raw: "", + output: { + rubricVersion: plan.analyzerVersion, + languages: plan.profile.languages, + applications: plan.profile.applications, + criteria: {}, + warnings: [], + recommendations: [], + model: null, + }, + }; + } else { + const gatewayTool = gatewayToolForReadiness(agent); + if (gatewayTool) await ensureFreshGatewayKey(gatewayTool); + run = await runReadinessAgent( + agent, + root, + undefined, + options.model, + onProgress, + plan, + ); + } run = { ...run, output: normalizeReadinessEvidence( @@ -107,7 +144,12 @@ export async function runReadiness( ), }; let totalDurationMs = run.durationMs; - let errors = validateReadinessOutput(run.output, root); + let errors = validateReadinessOutput( + run.output, + root, + plan.criteriaOrder, + plan.analyzerVersion, + ); const { maxRepairs } = readinessRuntimeConfig(); for ( let attempt = 1; @@ -137,7 +179,12 @@ export async function runReadiness( ), }; totalDurationMs += run.durationMs; - errors = validateReadinessOutput(run.output, root); + errors = validateReadinessOutput( + run.output, + root, + plan.criteriaOrder, + plan.analyzerVersion, + ); } if (errors.length > 0) throw new Error( @@ -150,6 +197,7 @@ export async function runReadiness( message: error instanceof Error ? error.message : String(error), }; } + const semanticMs = Date.now() - semanticStarted; const after = repositoryState(root); if (after !== before) { @@ -186,7 +234,13 @@ export async function runReadiness( repoName: repoName(url), branch: git(["branch", "--show-current"], root), commitHash: git(["rev-parse", "HEAD"], root), - rubricVersion: READINESS_RUBRIC_VERSION, + rubricVersion: plan.analyzerVersion, + readinessProfileId: selectedProfile.id, + readinessProfileVersionId: selectedProfile.activeVersion.id, + profileRevision: selectedProfile.activeVersion.revision, + profileContentHash: selectedProfile.activeVersion.contentHash, + analyzerVersion: selectedProfile.activeVersion.analyzerVersion, + profileSnapshot: selectedProfile, languages: run.output.languages, apps: Object.fromEntries( run.output.applications.map((app) => [ @@ -202,6 +256,12 @@ export async function runReadiness( model: run.output.model ?? configuredModel ?? null, durationMs: run.durationMs, }, + timings: { + profileFetchMs: options.profileFetchMs ?? 0, + deterministicMs, + semanticMs, + totalMs: Date.now() - totalStarted, + }, // Included for human-readable proxy logs only. The server must recompute these. summary, }; @@ -210,12 +270,14 @@ export async function runReadiness( let authError = ""; let auth: AuthData; try { - auth = await login( - (message) => { - authError = message; - }, - () => {}, - ); + auth = + options.auth ?? + (await login( + (message) => { + authError = message; + }, + () => {}, + )); } catch (error) { return { exitCode: 1, @@ -248,6 +310,6 @@ export async function runReadiness( const data = (await response.json()) as { report?: { id?: string } }; return { exitCode: 0, - message: `Stored report ${data.report?.id ?? ""} · Level ${summary.level} · ${summary.passRate}% pass · ${summary.criteriaTotal} criteria evaluated`, + message: `Stored report ${data.report?.id ?? ""} · ${selectedProfile.name} r${selectedProfile.activeVersion.revision} · Level ${summary.level} · ${summary.passRate}% pass · ${summary.criteriaTotal} evaluated · ${plan.criteriaOrder.length - summary.criteriaTotal} N/A`, }; } diff --git a/tests/ReadinessApp.test.tsx b/tests/ReadinessApp.test.tsx index 4d68cca..ce97a95 100644 --- a/tests/ReadinessApp.test.tsx +++ b/tests/ReadinessApp.test.tsx @@ -1,5 +1,6 @@ import { render } from "ink-testing-library"; import { describe, expect, it, vi } from "vitest"; +import { bundledStandardProfile } from "@/lib/readiness-profile.js"; import { ReadinessApp } from "@/ReadinessApp.js"; async function settle(ms = 30) { @@ -7,34 +8,49 @@ async function settle(ms = 30) { } describe("ReadinessApp", () => { + const loadProfiles = vi.fn(async () => ({ + auth: { access_token: "test" } as never, + profiles: [bundledStandardProfile()], + })); + it("uses the shared Ink wizard style and runs the selected agent", async () => { const run = vi.fn(async (_agent, progress?: (message: string) => void) => { progress?.("Validating report"); return { exitCode: 0, message: "Stored report report-1" }; }); - const { frames, stdin } = render( + const { frames } = render( , ); - expect(frames.at(-1)).toContain("AGENT READINESS"); - expect(frames.at(-1)).toContain("OpenCode"); - expect(frames.at(-1)).toContain("(unavailable)"); + await settle(200); + expect(frames.join("\n")).toContain("AGENT READINESS"); + expect(frames.join("\n")).toContain("OpenCode"); + expect(frames.join("\n")).toContain("(unavailable)"); - stdin.write("\r"); await settle(100); - expect(run).toHaveBeenCalledWith("opencode", expect.any(Function), {}); + expect(run).toHaveBeenCalledWith( + "opencode", + expect.any(Function), + expect.objectContaining({ + profile: expect.objectContaining({ slug: "standard" }), + }), + ); expect(frames.join("\n")).toContain("Stored report report-1"); }); - it("explains when no supported agent is installed", () => { + it("explains when no supported agent is installed", async () => { const { lastFrame } = render( , ); + await settle(200); expect(lastFrame()).toContain("No supported coding agent is available"); }); }); diff --git a/tests/ReadinessNav.test.tsx b/tests/ReadinessNav.test.tsx index 765a0f6..547fb73 100644 --- a/tests/ReadinessNav.test.tsx +++ b/tests/ReadinessNav.test.tsx @@ -1,5 +1,6 @@ import { render } from "ink-testing-library"; import { describe, expect, it, vi } from "vitest"; +import { bundledStandardProfile } from "@/lib/readiness-profile.js"; import { ReadinessApp } from "@/ReadinessApp.js"; async function settle(ms = 30) { @@ -7,6 +8,11 @@ async function settle(ms = 30) { } describe("ReadinessApp arrow-key navigation", () => { + const loadProfiles = vi.fn(async () => ({ + auth: { access_token: "test" } as never, + profiles: [bundledStandardProfile()], + })); + it("moves cursor down with down-arrow and selects the highlighted agent on Enter", async () => { const run = vi.fn(async () => ({ exitCode: 0, @@ -16,18 +22,24 @@ describe("ReadinessApp arrow-key navigation", () => { , ); + await settle(250); // Cursor starts on Claude (index 0). Press down-arrow to move to Codex. stdin.write("\x1b[B"); - await settle(); + await settle(250); expect(frames.at(-1)).toContain("Codex"); // The cursor highlight (bold) should now be on Codex, not Claude. // ink-testing-library strips ANSI, so we check via the selection callback. stdin.write("\r"); - await settle(100); - expect(run).toHaveBeenCalledWith("codex", expect.any(Function), {}); + await settle(250); + expect(run).toHaveBeenCalledWith( + "codex", + expect.any(Function), + expect.objectContaining({ profile: expect.any(Object) }), + ); }); it("moves cursor up with up-arrow from the middle item", async () => { @@ -39,19 +51,25 @@ describe("ReadinessApp arrow-key navigation", () => { , ); + await settle(150); // Move down twice (claude -> codex -> opencode), then up once (opencode -> codex). stdin.write("\x1b[B"); - await settle(); + await settle(150); stdin.write("\x1b[B"); - await settle(); + await settle(150); stdin.write("\x1b[A"); - await settle(); + await settle(150); stdin.write("\r"); - await settle(100); - expect(run).toHaveBeenCalledWith("codex", expect.any(Function), {}); + await settle(250); + expect(run).toHaveBeenCalledWith( + "codex", + expect.any(Function), + expect.objectContaining({ profile: expect.any(Object) }), + ); }); it("skips unavailable agents when navigating", async () => { @@ -63,15 +81,21 @@ describe("ReadinessApp arrow-key navigation", () => { , ); + await settle(250); // Cursor starts on Claude (index 0). Down-arrow should skip Codex (unavailable) // and land on OpenCode (index 2). stdin.write("\x1b[B"); - await settle(); + await settle(150); stdin.write("\r"); - await settle(100); - expect(run).toHaveBeenCalledWith("opencode", expect.any(Function), {}); + await settle(250); + expect(run).toHaveBeenCalledWith( + "opencode", + expect.any(Function), + expect.objectContaining({ profile: expect.any(Object) }), + ); }); }); diff --git a/tests/lib/readiness-cli.test.ts b/tests/lib/readiness-cli.test.ts new file mode 100644 index 0000000..7cb92fe --- /dev/null +++ b/tests/lib/readiness-cli.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from "vitest"; +import { parseReadinessArgs } from "@/lib/readiness-cli.js"; + +describe("readiness CLI arguments", () => { + it("accepts interactive and non-interactive selectors in either flag form", () => { + expect( + parseReadinessArgs([ + "--profile=team", + "--agent", + "codex", + "--model", + "gpt-test", + ]), + ).toEqual({ profile: "team", agent: "codex", model: "gpt-test" }); + }); + + it("rejects unknown flags, missing values, duplicates, and OpenCode model overrides", () => { + expect(() => parseReadinessArgs(["--wat"])).toThrow(/Unknown/); + expect(() => parseReadinessArgs(["--profile"])).toThrow(/Missing/); + expect(() => + parseReadinessArgs(["--profile", "one", "--profile", "two"]), + ).toThrow(/Duplicate/); + expect(() => + parseReadinessArgs(["--agent", "opencode", "--model", "ignored"]), + ).toThrow(/cannot be used/); + }); +}); diff --git a/tests/lib/readiness-profile.test.ts b/tests/lib/readiness-profile.test.ts new file mode 100644 index 0000000..d42841c --- /dev/null +++ b/tests/lib/readiness-profile.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; +import { + bundledStandardProfile, + decodeReadinessProfile, + selectReadinessProfile, +} from "@/lib/readiness-profile.js"; + +describe("readiness profiles", () => { + it("decodes and freezes the full active version identity", () => { + const source = structuredClone(bundledStandardProfile()); + const decoded = decodeReadinessProfile(source); + expect(decoded.activeVersion.id).toBe(source.activeVersion.id); + expect(decoded.activeVersion.definition.criteria).toHaveLength(82); + }); + + it("selects by id/slug and auto-selects only a sole or unique default", () => { + const standard = bundledStandardProfile(); + const custom = { + ...structuredClone(standard), + id: "custom-id", + slug: "custom", + name: "Custom", + isDefault: false, + activeVersion: { ...standard.activeVersion, id: "custom-version" }, + }; + expect(selectReadinessProfile([standard, custom])?.slug).toBe("standard"); + expect(selectReadinessProfile([standard, custom], "custom-id")?.slug).toBe( + "custom", + ); + expect(() => selectReadinessProfile([standard], "missing")).toThrow( + /not found/, + ); + }); + + it("rejects unsafe or duplicate dynamic criterion keys", () => { + const profile = structuredClone(bundledStandardProfile()); + const first = profile.activeVersion.definition.criteria[0]; + if (!first) throw new Error("Standard profile fixture is empty."); + first.key = "__proto__"; + expect(() => decodeReadinessProfile(profile)).toThrow(/unsafe/); + }); +}); diff --git a/tests/lib/readiness-rules.test.ts b/tests/lib/readiness-rules.test.ts new file mode 100644 index 0000000..3b35b9e --- /dev/null +++ b/tests/lib/readiness-rules.test.ts @@ -0,0 +1,89 @@ +import { execFileSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import type { ReadinessCriterionConfig } from "@/lib/readiness-profile.js"; +import { evaluateConfiguredCriterion } from "@/lib/readiness-rules.js"; + +const roots: string[] = []; +function repository(files: Record) { + const root = mkdtempSync(join(tmpdir(), "codev-readiness-rules-")); + roots.push(root); + execFileSync("git", ["init", "-q"], { cwd: root }); + for (const [file, content] of Object.entries(files)) { + mkdirSync(join(root, file, ".."), { recursive: true }); + writeFileSync(join(root, file), content); + } + execFileSync("git", ["add", "."], { cwd: root }); + const trackedFiles = Object.keys(files); + return { root, files: trackedFiles, trackedFiles }; +} + +function criterion( + overrides: Partial = {}, +): ReadinessCriterionConfig { + return { + key: "tests_configured", + name: "Tests configured", + category: "Testing", + description: "Tests should be configured.", + maturityLevel: 1, + repositoryScope: "repository", + enabled: true, + order: 0, + passCondition: "A test script exists.", + evidenceRequirement: "package.json", + applicability: { kind: "always" }, + evidenceLocators: [ + { type: "manifest_script", manifest: "package.json", name: "test" }, + ], + decision: { engine: "deterministic", expected: "present" }, + recommendationTemplate: "Add tests.", + priority: 1, + ...overrides, + }; +} + +afterEach(() => { + for (const root of roots.splice(0)) + rmSync(root, { recursive: true, force: true }); +}); + +describe("declarative readiness rules", () => { + it("evaluates bounded manifest predicates deterministically", () => { + const inventory = repository({ + "package.json": JSON.stringify({ scripts: { test: "vitest" } }), + }); + expect(evaluateConfiguredCriterion(criterion(), inventory)).toMatchObject({ + mode: "deterministic", + result: { status: "pass", evidence: ["package.json"] }, + }); + }); + + it("marks N/A only from a failed applicability rule", () => { + const inventory = repository({ README: "notes" }); + const configured = criterion({ + applicability: { + kind: "predicate", + predicate: { type: "tracked_path_exists", path: "src/**" }, + }, + }); + expect(evaluateConfiguredCriterion(configured, inventory)).toMatchObject({ + mode: "not_applicable", + result: { status: "skipped", numerator: null }, + }); + }); + + it("rejects unsafe regex constructs", () => { + const inventory = repository({ "src/a.ts": "aaaa" }); + const configured = criterion({ + evidenceLocators: [ + { type: "text_matches", path: "src/**", pattern: "(a+)+$" }, + ], + }); + expect(() => evaluateConfiguredCriterion(configured, inventory)).toThrow( + /unsupported constructs/, + ); + }); +}); diff --git a/tests/lib/readiness-upload.test.ts b/tests/lib/readiness-upload.test.ts new file mode 100644 index 0000000..efc5a72 --- /dev/null +++ b/tests/lib/readiness-upload.test.ts @@ -0,0 +1,117 @@ +import { execFileSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import * as configure from "@/lib/configure.js"; +import { runReadiness } from "@/lib/readiness.js"; +import type { ReadinessProfile } from "@/lib/readiness-profile.js"; + +const originalCwd = process.cwd(); +const roots: string[] = []; + +function deterministicProfile(): ReadinessProfile { + return { + id: "profile-1", + name: "Focused", + slug: "focused", + description: "Focused profile", + scope: "personal", + isDefault: false, + status: "published", + activeVersion: { + id: "version-2", + revision: 2, + contentHash: "sha256:test", + schemaVersion: "1", + analyzerVersion: "2026-07-15.v2", + definition: { + criteria: [ + { + key: "tests_configured", + name: "Tests configured", + category: "Testing", + description: "A test script is configured.", + maturityLevel: 1, + repositoryScope: "repository", + enabled: true, + order: 0, + passCondition: "A test script exists.", + evidenceRequirement: "package.json", + applicability: { kind: "always" }, + evidenceLocators: [ + { + type: "manifest_script", + manifest: "package.json", + name: "test", + }, + ], + decision: { engine: "deterministic", expected: "present" }, + recommendationTemplate: "Add a test script.", + priority: 1, + }, + ], + }, + }, + }; +} + +afterEach(() => { + process.chdir(originalCwd); + vi.restoreAllMocks(); + for (const root of roots.splice(0)) + rmSync(root, { recursive: true, force: true }); +}); + +describe("profile-aware readiness upload", () => { + it("uploads the frozen profile identity, snapshot, inventory, and timings", async () => { + const root = mkdtempSync(join(tmpdir(), "codev-readiness-upload-")); + roots.push(root); + execFileSync("git", ["init", "-q"], { cwd: root }); + mkdirSync(join(root, "src")); + writeFileSync( + join(root, "package.json"), + JSON.stringify({ scripts: { test: "vitest" } }), + ); + execFileSync("git", ["add", "."], { cwd: root }); + process.chdir(root); + vi.spyOn(configure, "detectConfiguredTools").mockReturnValue(["codex"]); + const fetch = vi.fn( + async (_input: string | URL | Request, _init?: RequestInit) => + new Response(JSON.stringify({ report: { id: "report-1" } }), { + status: 200, + headers: { "content-type": "application/json" }, + }), + ); + vi.spyOn(globalThis, "fetch").mockImplementation(fetch); + + const result = await runReadiness("codex", () => {}, { + profile: deterministicProfile(), + auth: { access_token: "test-token" } as never, + profileFetchMs: 12, + }); + + expect(result.exitCode).toBe(0); + const request = fetch.mock.calls[0]?.[1]; + if (!request) throw new Error("Readiness upload request was not captured."); + const payload = JSON.parse(String(request.body)) as Record; + expect(payload).toMatchObject({ + readinessProfileId: "profile-1", + readinessProfileVersionId: "version-2", + profileRevision: 2, + profileContentHash: "sha256:test", + analyzerVersion: "2026-07-15.v2", + timings: { + profileFetchMs: 12, + deterministicMs: expect.any(Number), + semanticMs: expect.any(Number), + totalMs: expect.any(Number), + }, + }); + expect(payload.profileSnapshot).toMatchObject({ + id: "profile-1", + activeVersion: { id: "version-2" }, + }); + expect(Object.keys(payload.report as object)).toEqual(["tests_configured"]); + }); +}); From aecfba62b5319880f0de0ddbdee4cd1d2faa62fb Mon Sep 17 00:00:00 2001 From: Bao Tran Date: Wed, 22 Jul 2026 11:28:59 +0700 Subject: [PATCH 08/14] feat: support configurable readiness profile rules - Add first-class built-in scanner decisions while keeping static and AI-assisted evaluation paths explicit. - Support any, all, none, and minimum evidence-locator aggregation for deterministic rules. - Validate profile definitions before execution and preserve immutable profile identity during report upload. - Fix the isolated OpenCode readiness configuration to resolve a concrete gateway URL. - Add focused coverage for built-in routing, aggregation behavior, validation, and upload metadata. Decisions and assumptions: - Built-in scanners remain tested CoDev implementations selected by stable rule keys; users do not author arbitrary scanner code. - Deterministic custom rules operate only on bounded repository evidence locators. - Existing published profile revisions remain immutable; new contract shapes apply to successor revisions. --- src/lib/readiness-agent.ts | 39 +++++++----- src/lib/readiness-plan.ts | 32 ++++++++-- src/lib/readiness-profile.ts | 32 +++++++++- src/lib/readiness-rules.ts | 60 +++++++++++++++--- tests/lib/readiness-contract.test.ts | 13 ++++ tests/lib/readiness-plan.test.ts | 33 ++++++++++ tests/lib/readiness-profile.test.ts | 15 +++++ tests/lib/readiness-rules.test.ts | 91 +++++++++++++++++++++++++++- tests/lib/readiness-upload.test.ts | 3 +- 9 files changed, 283 insertions(+), 35 deletions(-) diff --git a/src/lib/readiness-agent.ts b/src/lib/readiness-agent.ts index 1c467bd..0bbcfc9 100644 --- a/src/lib/readiness-agent.ts +++ b/src/lib/readiness-agent.ts @@ -8,7 +8,7 @@ import { } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { loadApiKey } from "@/lib/auth.js"; +import { type ApiKeyCreds, loadApiKey } from "@/lib/auth.js"; import { detectConfiguredTools } from "@/lib/configure.js"; import { AI_GATEWAY_OPENAI_URL } from "@/lib/const.js"; import { readinessRuntimeConfig } from "@/lib/readiness-config.js"; @@ -116,6 +116,27 @@ export function readinessProcessEnv( }; } +export function openCodeReadinessConfig( + credentials: ApiKeyCreds, + model: string, +) { + return { + $schema: "https://opencode.ai/config.json", + model: `aigateway/${model}`, + provider: { + aigateway: { + npm: "@ai-sdk/openai-compatible", + name: "AI Gateway", + options: { + baseURL: credentials.baseUrl ?? AI_GATEWAY_OPENAI_URL(), + apiKey: credentials.apiKey, + }, + models: { [model]: { name: model } }, + }, + }, + }; +} + export function claudeReadinessEnvOverrides(): NodeJS.ProcessEnv { return { ANTHROPIC_API_KEY: undefined, @@ -436,21 +457,7 @@ export async function runReadinessAgent( ); writeFileSync( join(configDir, "opencode.json"), - JSON.stringify({ - $schema: "https://opencode.ai/config.json", - model: `aigateway/${model}`, - provider: { - aigateway: { - npm: "@ai-sdk/openai-compatible", - name: "AI Gateway", - options: { - baseURL: credentials.baseUrl ?? AI_GATEWAY_OPENAI_URL, - apiKey: credentials.apiKey, - }, - models: { [model]: { name: model } }, - }, - }, - }), + JSON.stringify(openCodeReadinessConfig(credentials, model)), { mode: 0o600 }, ); envOverrides = { diff --git a/src/lib/readiness-plan.ts b/src/lib/readiness-plan.ts index 8c767f6..b748705 100644 --- a/src/lib/readiness-plan.ts +++ b/src/lib/readiness-plan.ts @@ -8,6 +8,7 @@ import { type ReadinessCriterionResult, } from "@/lib/readiness-contract.js"; import { + builtInReadinessRuleKey, bundledStandardProfile, enabledProfileCriteria, isStandardProfile, @@ -659,7 +660,17 @@ export function buildReadinessEvaluationPlan( selectedCriteria[definition.key] = decision; } } else { - for (const definition of definitions) + for (const definition of definitions) { + const builtInKey = builtInReadinessRuleKey(definition); + if (builtInKey) { + const decision = criteria[builtInKey]; + if (!decision) + throw new Error( + `Readiness profile references unsupported built-in criterion: ${builtInKey}.`, + ); + selectedCriteria[definition.key] = decision; + continue; + } selectedCriteria[definition.key] = evaluateConfiguredCriterion( definition, { @@ -668,6 +679,7 @@ export function buildReadinessEvaluationPlan( trackedFiles, }, ); + } } const discoveredEvidence = Object.values(selectedCriteria).flatMap( (decision) => @@ -729,6 +741,16 @@ export function finalizeReadinessOutput( ]; }), ) as Record; + const configuredRecommendations = plan.definitions + .filter((definition) => criteria[definition.key]?.status === "fail") + .toSorted((left, right) => left.priority - right.priority) + .map((definition) => definition.recommendationTemplate); + const agentRecommendations = Array.isArray(output.recommendations) + ? output.recommendations + : []; + const recommendations = [ + ...new Set([...configuredRecommendations, ...agentRecommendations]), + ].slice(0, 3); return { ...output, rubricVersion: plan.analyzerVersion, @@ -737,13 +759,13 @@ export function finalizeReadinessOutput( criteria, warnings: [...new Set(warnings)], recommendations: - Array.isArray(output.recommendations) && - output.recommendations.length >= 2 - ? output.recommendations.slice(0, 3) + recommendations.length >= 2 + ? recommendations : [ + ...recommendations, "Address the highest-impact failing readiness criteria.", "Document and automate the repository's development workflow.", - ], + ].slice(0, 3), model: typeof output.model === "string" ? output.model : null, }; } diff --git a/src/lib/readiness-profile.ts b/src/lib/readiness-profile.ts index 026fe72..0f8b461 100644 --- a/src/lib/readiness-profile.ts +++ b/src/lib/readiness-profile.ts @@ -45,6 +45,7 @@ export interface ReadinessProfileVersion { export interface ReadinessProfile { id: string; + ownerProfileId: string | null; name: string; slug: string; description: string; @@ -142,6 +143,10 @@ export function decodeReadinessProfile(value: unknown): ReadinessProfile { } const decoded: ReadinessProfile = { id: text(profile.id, "id", 200), + ownerProfileId: + typeof profile.ownerProfileId === "string" + ? profile.ownerProfileId + : null, name: text(profile.name, "name", 200), slug: text(profile.slug, "slug", 200), description: @@ -226,8 +231,14 @@ export function selectReadinessProfile( return matches[0]; } if (profiles.length === 1) return profiles[0]; - const defaults = profiles.filter((profile) => profile.isDefault); - return defaults.length === 1 ? defaults[0] : undefined; + const personalDefault = profiles.find( + (profile) => profile.isDefault && profile.scope === "personal", + ); + if (personalDefault) return personalDefault; + const systemDefaults = profiles.filter( + (profile) => profile.isDefault && profile.scope === "system", + ); + return systemDefaults.length === 1 ? systemDefaults[0] : undefined; } export function enabledProfileCriteria( @@ -241,6 +252,20 @@ export function enabledProfileCriteria( ); } +export function builtInReadinessRuleKey( + criterion: ReadinessCriterionConfig, +): string | undefined { + const decision = record(criterion.decision); + if (decision?.engine !== "builtin") return undefined; + const ruleKey = + typeof decision.ruleKey === "string" ? decision.ruleKey : criterion.key; + if (ruleKey !== criterion.key) + throw new Error( + `Readiness built-in rule key for ${criterion.key} does not match its criterion key.`, + ); + return ruleKey; +} + export function isStandardProfile(profile: ReadinessProfile): boolean { return profile.slug === "standard" && profile.scope === "system"; } @@ -248,6 +273,7 @@ export function isStandardProfile(profile: ReadinessProfile): boolean { export function bundledStandardProfile(): ReadinessProfile { return { id: "builtin:standard", + ownerProfileId: null, name: "Standard", slug: "standard", description: "Built-in CoDev Standard readiness profile.", @@ -274,7 +300,7 @@ export function bundledStandardProfile(): ReadinessProfile { evidenceRequirement: criterion.evidenceRequired, applicability: { kind: "always" }, evidenceLocators: [], - decision: { engine: "builtin" }, + decision: { engine: "builtin", ruleKey: criterion.id }, recommendationTemplate: `Address ${criterion.id.replaceAll("_", " ")}.`, priority: criterion.maturityLevel, })), diff --git a/src/lib/readiness-rules.ts b/src/lib/readiness-rules.ts index 2c61db5..d9dbfbd 100644 --- a/src/lib/readiness-rules.ts +++ b/src/lib/readiness-rules.ts @@ -302,13 +302,10 @@ export function evaluateConfiguredCriterion( `The applicability rule for ${criterion.name} did not match.`, ), }; - const evidence = [ - ...new Set( - criterion.evidenceLocators.flatMap((locator) => - predicateMatches(locator, inventory, budget), - ), - ), - ].slice(0, MAX_EVIDENCE); + const locatorEvidence = criterion.evidenceLocators.map((locator) => + predicateMatches(locator, inventory, budget), + ); + const evidence = [...new Set(locatorEvidence.flat())].slice(0, MAX_EVIDENCE); const decision = record(criterion.decision); const engine = typeof decision?.engine === "string" ? decision.engine : undefined; @@ -325,9 +322,19 @@ export function evaluateConfiguredCriterion( throw new Error( `Unsupported readiness decision engine for ${criterion.key}.`, ); - const expected = decision?.expected === "absent" ? "absent" : "present"; + const match = decision?.match; + const matched = locatorEvidence.filter( + (entries) => entries.length > 0, + ).length; + const minimum = decision?.minimum; const passed = - expected === "present" ? evidence.length > 0 : evidence.length === 0; + match === "none" + ? matched === 0 + : match === "all" + ? locatorEvidence.length > 0 && matched === locatorEvidence.length + : match === "minimum" + ? matched >= (minimum as number) + : matched > 0; const rationale = passed ? typeof decision?.passRationale === "string" ? decision.passRationale @@ -350,10 +357,45 @@ function validateConfiguredCriterion( for (const locator of criterion.evidenceLocators) predicateMatches(locator, inventory, budget); const engine = record(criterion.decision)?.engine; + if (engine === "builtin") { + const ruleKey = record(criterion.decision)?.ruleKey; + if (ruleKey !== criterion.key) + throw new Error( + `Readiness built-in rule key for ${criterion.key} is invalid.`, + ); + return; + } if (engine !== "deterministic" && engine !== "semantic") throw new Error( `Unsupported readiness decision engine for ${criterion.key}.`, ); + if (engine === "deterministic") { + const decision = record(criterion.decision); + const match = decision?.match; + if ( + match !== "any" && + match !== "all" && + match !== "none" && + match !== "minimum" + ) + throw new Error( + `Readiness deterministic match mode for ${criterion.key} is invalid.`, + ); + if (criterion.evidenceLocators.length === 0) + throw new Error( + `Readiness deterministic locators for ${criterion.key} are required.`, + ); + const minimum = decision?.minimum; + if ( + match === "minimum" && + (!Number.isSafeInteger(minimum) || + (minimum as number) < 1 || + (minimum as number) > criterion.evidenceLocators.length) + ) + throw new Error( + `Readiness deterministic minimum for ${criterion.key} is invalid.`, + ); + } } export function validateProfileRules(profile: ReadinessProfile): void { diff --git a/tests/lib/readiness-contract.test.ts b/tests/lib/readiness-contract.test.ts index 5d4de70..fd46bd4 100644 --- a/tests/lib/readiness-contract.test.ts +++ b/tests/lib/readiness-contract.test.ts @@ -10,6 +10,7 @@ import { buildReadinessPrompt, claudeReadinessEnvOverrides, extractAgentOutput, + openCodeReadinessConfig, openCodeStructuredOutputInstruction, providerFailureFromLine, readinessProcessEnv, @@ -209,6 +210,18 @@ describe("readiness contract", () => { ); }); + it("writes a concrete gateway URL into the isolated OpenCode config", () => { + const config = openCodeReadinessConfig( + { apiKey: "sk-test" }, + "MiniMax/MiniMax-M2.7", + ); + + expect(config.provider.aigateway.options.baseURL).toMatch( + /^https?:\/\/.+\/v1$/, + ); + expect(JSON.stringify(config)).not.toContain("undefined/chat/completions"); + }); + it("removes CoDev shims from readiness subprocess PATH resolution", () => { const env = readinessProcessEnv( {}, diff --git a/tests/lib/readiness-plan.test.ts b/tests/lib/readiness-plan.test.ts index f18a71f..274e800 100644 --- a/tests/lib/readiness-plan.test.ts +++ b/tests/lib/readiness-plan.test.ts @@ -14,6 +14,7 @@ import { finalizeReadinessOutput, semanticCriterionIds, } from "@/lib/readiness-plan.js"; +import { bundledStandardProfile } from "@/lib/readiness-profile.js"; const roots: string[] = []; @@ -60,6 +61,28 @@ afterEach(() => { }); describe("readiness evaluation plan", () => { + it("preserves built-in deterministic checks when Standard is cloned", () => { + const root = repository({ + "package.json": '{"scripts":{"test":"vitest"}}', + ".pre-commit-config.yaml": "repos: []", + }); + const standard = buildReadinessEvaluationPlan(root); + const clonedProfile = { + ...structuredClone(bundledStandardProfile()), + id: "personal-clone", + ownerProfileId: "user-1", + slug: "personal-clone", + scope: "personal", + isDefault: false, + }; + const cloned = buildReadinessEvaluationPlan(root, clonedProfile); + + expect(cloned.criteria.pre_commit_hooks).toEqual( + standard.criteria.pre_commit_hooks, + ); + expect(cloned.criteria.agents_md).toEqual(standard.criteria.agents_md); + }); + it("recognizes env template naming variants only when env configuration is relevant", () => { const root = repository({ "package.json": '{"scripts":{"start":"node src/index.js"}}', @@ -253,6 +276,16 @@ describe("readiness evaluation plan", () => { ).toBeGreaterThanOrEqual(semanticCount); expect(skipped.languages).toEqual(["TypeScript"]); expect(skipped.applications).toEqual(passed.applications); + const highestPriorityFailure = plan.definitions + .filter( + (definition) => skipped.criteria[definition.key]?.status === "fail", + ) + .toSorted((left, right) => left.priority - right.priority)[0]; + if (!highestPriorityFailure) + throw new Error("Expected a failing criterion."); + expect(skipped.recommendations).toContain( + highestPriorityFailure.recommendationTemplate, + ); }); it("only skips database and API checks after high-confidence absence", () => { diff --git a/tests/lib/readiness-profile.test.ts b/tests/lib/readiness-profile.test.ts index d42841c..50261a3 100644 --- a/tests/lib/readiness-profile.test.ts +++ b/tests/lib/readiness-profile.test.ts @@ -32,6 +32,21 @@ describe("readiness profiles", () => { ); }); + it("prefers a personal default over the shared system default", () => { + const standard = bundledStandardProfile(); + const personal = { + ...structuredClone(standard), + id: "personal-id", + ownerProfileId: "user-1", + slug: "personal", + scope: "personal", + activeVersion: { ...standard.activeVersion, id: "personal-version" }, + }; + expect(selectReadinessProfile([standard, personal])?.id).toBe( + "personal-id", + ); + }); + it("rejects unsafe or duplicate dynamic criterion keys", () => { const profile = structuredClone(bundledStandardProfile()); const first = profile.activeVersion.definition.criteria[0]; diff --git a/tests/lib/readiness-rules.test.ts b/tests/lib/readiness-rules.test.ts index 3b35b9e..28b8540 100644 --- a/tests/lib/readiness-rules.test.ts +++ b/tests/lib/readiness-rules.test.ts @@ -38,7 +38,7 @@ function criterion( evidenceLocators: [ { type: "manifest_script", manifest: "package.json", name: "test" }, ], - decision: { engine: "deterministic", expected: "present" }, + decision: { engine: "deterministic", match: "any" }, recommendationTemplate: "Add tests.", priority: 1, ...overrides, @@ -86,4 +86,93 @@ describe("declarative readiness rules", () => { /unsupported constructs/, ); }); + + it("passes an absence check when prohibited evidence is missing", () => { + const inventory = repository({ README: "notes" }); + const configured = criterion({ + evidenceLocators: [{ type: "tracked_path_exists", path: ".env" }], + decision: { engine: "deterministic", match: "none" }, + }); + expect(evaluateConfiguredCriterion(configured, inventory)).toMatchObject({ + mode: "deterministic", + result: { status: "pass", evidence: [] }, + }); + }); + + it("combines locator results with any, all, and minimum policies", () => { + const inventory = repository({ + "package.json": JSON.stringify({ scripts: { test: "vitest" } }), + }); + const evidenceLocators = [ + { + type: "manifest_script" as const, + manifest: "package.json", + name: "test", + }, + { + type: "manifest_script" as const, + manifest: "package.json", + name: "lint", + }, + ]; + expect( + evaluateConfiguredCriterion( + criterion({ + evidenceLocators, + decision: { engine: "deterministic", match: "any" }, + }), + inventory, + ), + ).toMatchObject({ result: { status: "pass" } }); + expect( + evaluateConfiguredCriterion( + criterion({ + evidenceLocators, + decision: { engine: "deterministic", match: "all" }, + }), + inventory, + ), + ).toMatchObject({ result: { status: "fail" } }); + expect( + evaluateConfiguredCriterion( + criterion({ + evidenceLocators, + decision: { engine: "deterministic", match: "minimum", minimum: 1 }, + }), + inventory, + ), + ).toMatchObject({ result: { status: "pass" } }); + }); + + it("compares typed manifest setting values without string coercion", () => { + const inventory = repository({ + "package.json": JSON.stringify({ private: true, engines: { node: 22 } }), + }); + const booleanSetting = criterion({ + evidenceLocators: [ + { + type: "manifest_setting", + manifest: "package.json", + path: "private", + value: true, + }, + ], + }); + const numberSetting = criterion({ + evidenceLocators: [ + { + type: "manifest_setting", + manifest: "package.json", + path: "engines.node", + value: 22, + }, + ], + }); + expect( + evaluateConfiguredCriterion(booleanSetting, inventory), + ).toMatchObject({ result: { status: "pass" } }); + expect(evaluateConfiguredCriterion(numberSetting, inventory)).toMatchObject( + { result: { status: "pass" } }, + ); + }); }); diff --git a/tests/lib/readiness-upload.test.ts b/tests/lib/readiness-upload.test.ts index efc5a72..7b592cc 100644 --- a/tests/lib/readiness-upload.test.ts +++ b/tests/lib/readiness-upload.test.ts @@ -13,6 +13,7 @@ const roots: string[] = []; function deterministicProfile(): ReadinessProfile { return { id: "profile-1", + ownerProfileId: "user-1", name: "Focused", slug: "focused", description: "Focused profile", @@ -46,7 +47,7 @@ function deterministicProfile(): ReadinessProfile { name: "test", }, ], - decision: { engine: "deterministic", expected: "present" }, + decision: { engine: "deterministic", match: "any" }, recommendationTemplate: "Add a test script.", priority: 1, }, From f8267ebe40fa8dc838cc077b418db48ec993fe0b Mon Sep 17 00:00:00 2001 From: Bao Tran Date: Wed, 22 Jul 2026 17:02:02 +0700 Subject: [PATCH 09/14] fix: preserve readiness TUI after authentication --- src/ReadinessApp.tsx | 139 ++++++++++++++++++++++++++--------- src/lib/auth.ts | 24 ++++++ src/lib/readiness-agent.ts | 10 ++- src/lib/readiness-profile.ts | 24 +++--- src/lib/readiness.ts | 14 +++- src/lib/upload.ts | 20 ++--- tests/ReadinessApp.test.tsx | 20 +++++ tests/lib/upload.test.ts | 15 +--- 8 files changed, 192 insertions(+), 74 deletions(-) diff --git a/src/ReadinessApp.tsx b/src/ReadinessApp.tsx index de25a24..6af7ef4 100644 --- a/src/ReadinessApp.tsx +++ b/src/ReadinessApp.tsx @@ -1,8 +1,9 @@ -import { Box, Text, useApp } from "ink"; +import { Box, Text, useApp, useInput } from "ink"; import Spinner from "ink-spinner"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { Banner } from "@/components/Banner.js"; import { Frame } from "@/components/Frame.js"; +import { PasteBackPrompt, usePasteBack } from "@/components/PasteBack.js"; import { ReadinessAgentSelect, readinessAgentSelectTitle, @@ -18,7 +19,7 @@ import { runReadiness, } from "@/lib/readiness.js"; import { - isAgentAvailable, + isReadinessAgentAvailable, READINESS_AGENTS, type ReadinessAgent, } from "@/lib/readiness-agent.js"; @@ -60,17 +61,30 @@ export function ReadinessApp({ () => available ?? (Object.fromEntries( - READINESS_AGENTS.map((agent) => [agent, isAgentAvailable(agent)]), + READINESS_AGENTS.map((agent) => [ + agent, + isReadinessAgentAvailable(agent), + ]), ) as Record), [available], ); const [phase, setPhase] = useState("loading-profiles"); const [agent, setAgent] = useState(null); const [progress, setProgress] = useState("Loading readiness profiles"); + const [loginUrl, setLoginUrl] = useState(null); const [result, setResult] = useState(null); const [session, setSession] = useState(null); const [profile, setProfile] = useState(null); const profileFetchMs = useRef(0); + // Keep stdin referenced across the async SSO-to-selector transition. Without + // continuous input ownership, Ink can emit `beforeExit` after the callback + // server closes and unmount just as the agent selector becomes interactive. + useInput(() => undefined, { + isActive: phase !== "done" && phase !== "failed", + }); + const paste = usePasteBack( + loginUrl !== null && phase !== "failed" && phase !== "done", + ); const hasAvailableAgent = READINESS_AGENTS.some( (candidate) => detected[candidate], ); @@ -90,12 +104,18 @@ export function ReadinessApp({ return; } setAgent(choice); + setLoginUrl(null); setPhase("running"); run(choice, setProgress, { ...options, profile: chosen, auth: loaded.auth, profileFetchMs: profileFetchMs.current, + onLoginUrl: setLoginUrl, + onManualSubmit: (submit) => { + paste.submitRef.current = submit; + }, + onLoginDone: () => setLoginUrl(null), }) .then((next) => { setResult(next); @@ -109,7 +129,7 @@ export function ReadinessApp({ setPhase("failed"); }); }, - [detected, run, options], + [detected, run, options, paste.submitRef], ); const chooseProfile = useCallback( (chosen: ReadinessProfile, loaded = session) => { @@ -130,10 +150,16 @@ export function ReadinessApp({ useEffect(() => { let active = true; const started = Date.now(); - loadProfiles(setProgress) + loadProfiles(setProgress, { + onLoginUrl: setLoginUrl, + onManualSubmit: (submit) => { + paste.submitRef.current = submit; + }, + }) .then((loaded) => { if (!active) return; profileFetchMs.current = Date.now() - started; + setLoginUrl(null); setSession(loaded); const chosen = selectReadinessProfile(loaded.profiles, profileSelector); if (chosen) { @@ -153,24 +179,49 @@ export function ReadinessApp({ return () => { active = false; }; - }, [loadProfiles, profileSelector, requestedAgent, startRun]); + }, [ + loadProfiles, + profileSelector, + requestedAgent, + startRun, + paste.submitRef, + ]); useEffect(() => { - if (phase !== "done") return; - const timer = setTimeout(() => exit(), 50); + if (phase !== "done" && phase !== "failed") return; + const timer = setTimeout(() => { + if (phase === "failed") + exit(new Error(result?.message ?? "Readiness failed.")); + else exit(); + }, 50); return () => clearTimeout(timer); - }, [phase, exit]); + }, [phase, result, exit]); return ( {phase === "loading-profiles" && ( - - - - - {` ${progress}...`} + + + + + + {` ${progress}...`} + + {loginUrl && !paste.submitting && ( + + + {"If the browser didn't open, visit this URL manually:"} + + {loginUrl} + + + )} )} {phase !== "loading-profiles" && !hasAvailableAgent && ( @@ -197,30 +248,47 @@ export function ReadinessApp({ /> )} - {phase !== "loading-profiles" && phase !== "select-profile" && ( - - - - )} + {profile && + phase !== "loading-profiles" && + phase !== "select-profile" && ( + + + + )} {["running", "done", "failed"].includes(phase) && profile && ( Evaluate repository} > {phase === "running" ? ( - - - - - {` ${progress}...`} + + + + + + {` ${progress}...`} + + {loginUrl && !paste.submitting && ( + + + {"If the browser didn't open, visit this URL manually:"} + + {loginUrl} + + + )} ) : ( @@ -231,7 +299,12 @@ export function ReadinessApp({ )} {phase === "failed" && ( - Fix the issue above and rerun `codev readiness`. + + ✗ {result?.message ?? "Readiness failed."} + + Fix the issue above and rerun `codevhub readiness`. + + )} diff --git a/src/lib/auth.ts b/src/lib/auth.ts index 9e2d9a9..3d90b31 100644 --- a/src/lib/auth.ts +++ b/src/lib/auth.ts @@ -484,6 +484,30 @@ async function refreshToAuthData(refreshToken: string): Promise { }; } +export interface InteractiveAuthCallbacks { + onLoginUrl?: (url: string) => void; + onManualSubmit?: (submit: (pasted: string) => string | null) => void; + onLoginDone?: () => void; +} + +export async function ensureInteractiveAuth( + onStatus: (message: string) => void = () => {}, + callbacks: InteractiveAuthCallbacks = {}, +): Promise { + const auth = loadAuth(); + if (auth) return auth; + + const fresh = await login(onStatus, (openBrowser, url, submitManualCode) => { + if (callbacks.onLoginUrl) callbacks.onLoginUrl(url); + else + onStatus(`If your browser didn't open, visit this URL manually: ${url}`); + callbacks.onManualSubmit?.(submitManualCode); + openBrowser(); + }); + callbacks.onLoginDone?.(); + return fresh; +} + // Non-interactive SSO: return a usable session WITHOUT ever prompting — reuse a // non-expired cached session, else silently refresh via the stored // refresh_token. Returns null when neither works (no browser/paste fallback), so diff --git a/src/lib/readiness-agent.ts b/src/lib/readiness-agent.ts index 0bbcfc9..00bbeb8 100644 --- a/src/lib/readiness-agent.ts +++ b/src/lib/readiness-agent.ts @@ -28,10 +28,14 @@ import { stripShimDirFromPath } from "@/lib/shims.js"; export const READINESS_AGENTS = ["claude", "codex", "opencode"] as const; export type ReadinessAgent = (typeof READINESS_AGENTS)[number]; -export function assertReadinessPrerequisites(agent: ReadinessAgent): void { - if (agent === "codex" && isAgentAvailable(agent)) return; +export function isReadinessAgentAvailable(agent: ReadinessAgent): boolean { + if (agent === "codex" && isAgentAvailable(agent)) return true; const tool = agent === "claude" ? "claude-code" : agent; - if (detectConfiguredTools().includes(tool)) return; + return detectConfiguredTools().includes(tool); +} + +export function assertReadinessPrerequisites(agent: ReadinessAgent): void { + if (isReadinessAgentAvailable(agent)) return; throw new Error( `${agent === "claude" ? "Claude Code" : agent === "codex" ? "Codex" : "OpenCode"} is not configured by CoDev. Run \`codevhub install\`, select this agent, and retry the readiness scan.`, ); diff --git a/src/lib/readiness-profile.ts b/src/lib/readiness-profile.ts index 0f8b461..10b3c08 100644 --- a/src/lib/readiness-profile.ts +++ b/src/lib/readiness-profile.ts @@ -1,5 +1,8 @@ -import type { AuthData } from "@/lib/auth.js"; -import { login } from "@/lib/auth.js"; +import { + type AuthData, + ensureInteractiveAuth, + type InteractiveAuthCallbacks, +} from "@/lib/auth.js"; import { BACKEND_URL } from "@/lib/const.js"; import { loggedFetch } from "@/lib/log.js"; import { @@ -175,17 +178,20 @@ export function decodeReadinessProfile(value: unknown): ReadinessProfile { return decoded; } +export type ReadinessLoginCallbacks = Pick< + InteractiveAuthCallbacks, + "onLoginUrl" | "onManualSubmit" +>; + export async function fetchReadinessProfiles( onStatus: (message: string) => void = () => {}, + callbacks: ReadinessLoginCallbacks = {}, ): Promise { let authError = ""; - const auth = await login( - (message) => { - authError = message; - onStatus(message); - }, - () => {}, - ).catch((error) => { + const auth = await ensureInteractiveAuth((message) => { + authError = message; + onStatus(message); + }, callbacks).catch((error) => { throw new Error( authError || (error instanceof Error ? error.message : String(error)), ); diff --git a/src/lib/readiness.ts b/src/lib/readiness.ts index 6ab1e2f..8e63d91 100644 --- a/src/lib/readiness.ts +++ b/src/lib/readiness.ts @@ -1,5 +1,5 @@ import { spawnSync } from "node:child_process"; -import { type AuthData, login } from "@/lib/auth.js"; +import { type AuthData, ensureInteractiveAuth } from "@/lib/auth.js"; import { BACKEND_URL } from "@/lib/const.js"; import { type AgentRunResult, @@ -58,6 +58,9 @@ export interface ReadinessOptions { profile?: ReadinessProfile; auth?: AuthData; profileFetchMs?: number; + onLoginUrl?: (url: string) => void; + onManualSubmit?: (submit: (pasted: string) => string | null) => void; + onLoginDone?: () => void; } export function gatewayToolForReadiness( @@ -272,11 +275,16 @@ export async function runReadiness( try { auth = options.auth ?? - (await login( + (await ensureInteractiveAuth( (message) => { authError = message; + onProgress(message); + }, + { + onLoginUrl: options.onLoginUrl, + onManualSubmit: options.onManualSubmit, + onLoginDone: options.onLoginDone, }, - () => {}, )); } catch (error) { return { diff --git a/src/lib/upload.ts b/src/lib/upload.ts index 72e1a9c..fc4f8c2 100644 --- a/src/lib/upload.ts +++ b/src/lib/upload.ts @@ -18,8 +18,8 @@ import { basename, dirname, join } from "node:path"; import { gzipSync } from "node:zlib"; import { type AuthData, + ensureInteractiveAuth, loadAuth, - login, refreshCodevConfig, } from "@/lib/auth.js"; import { fetchSupabaseSession } from "@/lib/backend.js"; @@ -294,24 +294,14 @@ async function ensureAuth( ) { const auth = loadAuth(); if (auth) return auth; - const fresh = await login(onStatus, (openBrowser, url, submitManualCode) => { - // Hand the URL to the dedicated channel when the caller provides one - // (UploadApp pins it under the spinner). Fall back to onStatus so the - // daemon log — or any future caller that doesn't wire onLoginUrl — - // still has a paste fallback. - if (onLoginUrl) onLoginUrl(url); - else - onStatus(`If your browser didn't open, visit this URL manually: ${url}`); - // Expose the paste-back submitter to an interactive caller (UploadApp) so - // a no-browser user can finish login without leaving `codevhub upload`. The - // daemon doesn't wire this — it has no TTY and bails when logged out. - onManualSubmit?.(submitManualCode); - openBrowser(); + const fresh = await ensureInteractiveAuth(onStatus, { + onLoginUrl, + onManualSubmit, + onLoginDone, }); // Login finished (loopback browser callback or manual paste). Signal the // caller to dismiss the login URL + paste-back prompt before the upload // continues, so they don't linger on screen. - onLoginDone?.(); // login() no longer refreshes CoDev config on its own — every caller does // it explicitly so the timing fits each flow. On a fresh // login we don't have a cache yet, so populating it here avoids burning diff --git a/tests/ReadinessApp.test.tsx b/tests/ReadinessApp.test.tsx index ce97a95..0b14750 100644 --- a/tests/ReadinessApp.test.tsx +++ b/tests/ReadinessApp.test.tsx @@ -53,4 +53,24 @@ describe("ReadinessApp", () => { await settle(200); expect(lastFrame()).toContain("No supported coding agent is available"); }); + + it("shows a profile-load failure instead of a read-only agent picker", async () => { + const loadError = new Error( + "Readiness profile fetch failed (503): unavailable", + ); + const { frames } = render( + { + throw loadError; + })} + />, + ); + + await settle(100); + const output = frames.join("\n"); + expect(output).toContain(loadError.message); + expect(output).not.toContain("Choose coding agent"); + }); }); diff --git a/tests/lib/upload.test.ts b/tests/lib/upload.test.ts index 4f8dd33..98d3520 100644 --- a/tests/lib/upload.test.ts +++ b/tests/lib/upload.test.ts @@ -9,7 +9,6 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import type { MockInstance } from "vitest"; import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; -import * as auth from "@/lib/auth.js"; import { fileSha256, filterNewFiles, @@ -137,9 +136,11 @@ describe("upload helpers", () => { describe("runUpload", () => { test("signals onLoginDone after a fresh login completes", async () => { // auth.json has Supabase coords but no SSO session, so loadAuth() returns - // null and ensureAuth() must log in. Mock login() to resolve immediately - // (no browser), then assert onLoginDone fired so the caller can dismiss + // null and ensureAuth() must log in. Use the development bypass so the + // shared interactive-auth helper resolves immediately without a browser, + // then assert onLoginDone fired so the caller can dismiss // the login prompt before the upload proceeds. + vi.stubEnv("CODEV_BYPASS_LOGIN", "1"); mkdirSync(join(tempHome, ".codev-hub"), { recursive: true }); writeFileSync( join(tempHome, ".codev-hub", "auth.json"), @@ -150,12 +151,6 @@ describe("runUpload", () => { ); writeLog("fresh.md", "hello"); - const loginSpy = vi.spyOn(auth, "login").mockResolvedValue({ - access_token: "token", - id_token: "token", - expires_at: Date.now() + 3600000, - user: { sub: "u", email: "u@example.com", displayName: "User" }, - }); const onLoginDone = vi.fn(); const fetchSpy = vi.spyOn(globalThis, "fetch").mockImplementation((async ( input: string | URL | Request, @@ -209,12 +204,10 @@ describe("runUpload", () => { try { const summary = await runUpload({ onLoginDone }); - expect(loginSpy).toHaveBeenCalledTimes(1); expect(onLoginDone).toHaveBeenCalledTimes(1); expect(summary.uploaded).toBe(1); } finally { fetchSpy.mockRestore(); - loginSpy.mockRestore(); } }); From 2171a52ef55df858a967f31f0b25a7654013495f Mon Sep 17 00:00:00 2001 From: Bao Tran Date: Wed, 22 Jul 2026 17:44:16 +0700 Subject: [PATCH 10/14] fix: streamline readiness profile validation --- src/ReadinessApp.tsx | 5 ++++- src/lib/readiness-plan.ts | 3 ++- src/lib/readiness-profile.ts | 17 ++++++++++++----- src/lib/readiness-rules.ts | 16 ++++++++++------ tests/lib/readiness-profile.test.ts | 21 +++++++++++++++++++++ tests/lib/readiness-rules.test.ts | 22 ++++++++++++++++++++++ 6 files changed, 71 insertions(+), 13 deletions(-) diff --git a/src/ReadinessApp.tsx b/src/ReadinessApp.tsx index 6af7ef4..d6bb54f 100644 --- a/src/ReadinessApp.tsx +++ b/src/ReadinessApp.tsx @@ -161,7 +161,10 @@ export function ReadinessApp({ profileFetchMs.current = Date.now() - started; setLoginUrl(null); setSession(loaded); - const chosen = selectReadinessProfile(loaded.profiles, profileSelector); + const chosen = + profileSelector || loaded.profiles.length === 1 + ? selectReadinessProfile(loaded.profiles, profileSelector) + : undefined; if (chosen) { setProfile(chosen); if (requestedAgent) startRun(requestedAgent, chosen, loaded); diff --git a/src/lib/readiness-plan.ts b/src/lib/readiness-plan.ts index b748705..ebefced 100644 --- a/src/lib/readiness-plan.ts +++ b/src/lib/readiness-plan.ts @@ -744,7 +744,8 @@ export function finalizeReadinessOutput( const configuredRecommendations = plan.definitions .filter((definition) => criteria[definition.key]?.status === "fail") .toSorted((left, right) => left.priority - right.priority) - .map((definition) => definition.recommendationTemplate); + .map((definition) => definition.recommendationTemplate.trim()) + .filter(Boolean); const agentRecommendations = Array.isArray(output.recommendations) ? output.recommendations : []; diff --git a/src/lib/readiness-profile.ts b/src/lib/readiness-profile.ts index 10b3c08..7bc708b 100644 --- a/src/lib/readiness-profile.ts +++ b/src/lib/readiness-profile.ts @@ -75,6 +75,13 @@ function text(value: unknown, field: string, max = 4_000): string { return value; } +function optionalText(value: unknown, field: string, max = 4_000): string { + if (value === undefined || value === null) return ""; + if (typeof value !== "string" || value.length > max) + throw new Error(`Readiness profile ${field} is invalid.`); + return value; +} + function integer(value: unknown, field: string, min = 0): number { if (!Number.isSafeInteger(value) || (value as number) < min) throw new Error(`Readiness profile ${field} is invalid.`); @@ -99,21 +106,21 @@ function decodeCriterion(value: unknown): ReadinessCriterionConfig { return { key, name: text(item.name, `${key}.name`, 200), - category: text(item.category, `${key}.category`, 200), - description: text(item.description, `${key}.description`), + category: optionalText(item.category, `${key}.category`, 200), + description: optionalText(item.description, `${key}.description`), maturityLevel: integer(item.maturityLevel, `${key}.maturityLevel`, 1), repositoryScope: text(item.repositoryScope, `${key}.repositoryScope`, 100), enabled: item.enabled, order: integer(item.order, `${key}.order`), - passCondition: text(item.passCondition, `${key}.passCondition`), - evidenceRequirement: text( + passCondition: optionalText(item.passCondition, `${key}.passCondition`), + evidenceRequirement: optionalText( item.evidenceRequirement, `${key}.evidenceRequirement`, ), applicability: item.applicability, evidenceLocators: item.evidenceLocators, decision: item.decision, - recommendationTemplate: text( + recommendationTemplate: optionalText( item.recommendationTemplate, `${key}.recommendationTemplate`, ), diff --git a/src/lib/readiness-rules.ts b/src/lib/readiness-rules.ts index d9dbfbd..6181a6b 100644 --- a/src/lib/readiness-rules.ts +++ b/src/lib/readiness-rules.ts @@ -313,9 +313,10 @@ export function evaluateConfiguredCriterion( return { mode: "semantic", reason: - typeof decision?.instructions === "string" - ? decision.instructions - : criterion.passCondition, + (typeof decision?.instructions === "string" && + decision.instructions.trim()) || + criterion.passCondition.trim() || + criterion.description, evidence, }; if (engine !== "deterministic") @@ -336,10 +337,13 @@ export function evaluateConfiguredCriterion( ? matched >= (minimum as number) : matched > 0; const rationale = passed - ? typeof decision?.passRationale === "string" + ? typeof decision?.passRationale === "string" && + decision.passRationale.trim() ? decision.passRationale - : criterion.passCondition - : typeof decision?.failRationale === "string" + : criterion.passCondition || + `Required evidence was found for ${criterion.name}.` + : typeof decision?.failRationale === "string" && + decision.failRationale.trim() ? decision.failRationale : `Required evidence was not established for ${criterion.name}.`; return { diff --git a/tests/lib/readiness-profile.test.ts b/tests/lib/readiness-profile.test.ts index 50261a3..5fa6c8f 100644 --- a/tests/lib/readiness-profile.test.ts +++ b/tests/lib/readiness-profile.test.ts @@ -54,4 +54,25 @@ describe("readiness profiles", () => { first.key = "__proto__"; expect(() => decodeReadinessProfile(profile)).toThrow(/unsafe/); }); + + it("accepts empty optional criterion metadata", () => { + const profile = structuredClone(bundledStandardProfile()); + profile.slug = "custom"; + const criterion = profile.activeVersion.definition.criteria[0]; + if (!criterion) throw new Error("Standard profile fixture is empty."); + criterion.category = ""; + criterion.description = ""; + criterion.passCondition = ""; + criterion.evidenceRequirement = ""; + criterion.recommendationTemplate = ""; + expect( + decodeReadinessProfile(profile).activeVersion.definition.criteria[0], + ).toMatchObject({ + category: "", + description: "", + passCondition: "", + evidenceRequirement: "", + recommendationTemplate: "", + }); + }); }); diff --git a/tests/lib/readiness-rules.test.ts b/tests/lib/readiness-rules.test.ts index 28b8540..e05adca 100644 --- a/tests/lib/readiness-rules.test.ts +++ b/tests/lib/readiness-rules.test.ts @@ -51,6 +51,28 @@ afterEach(() => { }); describe("declarative readiness rules", () => { + it("uses useful default messages when optional rationale text is empty", () => { + const inventory = repository({ + "package.json": JSON.stringify({ scripts: { test: "vitest" } }), + }); + const configured = criterion({ + passCondition: "", + decision: { + engine: "deterministic", + match: "any", + passRationale: "", + failRationale: "", + }, + }); + expect(evaluateConfiguredCriterion(configured, inventory)).toMatchObject({ + mode: "deterministic", + result: { + status: "pass", + rationale: "Required evidence was found for Tests configured.", + }, + }); + }); + it("evaluates bounded manifest predicates deterministically", () => { const inventory = repository({ "package.json": JSON.stringify({ scripts: { test: "vitest" } }), From 691bab6dc579f56f140d727bd691aaa5570bb03e Mon Sep 17 00:00:00 2001 From: Bao Tran Date: Thu, 23 Jul 2026 11:39:32 +0700 Subject: [PATCH 11/14] fix: generate contextual readiness recommendations - remove hardcoded recommendation templates from profile criteria\n- accept zero to three scan-generated next steps without generic filler\n- filter malformed placeholder actions and use display names in warnings\n- update focused readiness contract, plan, rule, and upload coverage --- src/lib/readiness-agent.ts | 2 +- src/lib/readiness-contract.ts | 5 ++-- src/lib/readiness-plan.ts | 41 ++++++++++++++++------------- src/lib/readiness-profile.ts | 6 ----- tests/lib/readiness-plan.test.ts | 24 ++++++++++------- tests/lib/readiness-profile.test.ts | 2 -- tests/lib/readiness-rules.test.ts | 1 - tests/lib/readiness-upload.test.ts | 1 - 8 files changed, 39 insertions(+), 43 deletions(-) diff --git a/src/lib/readiness-agent.ts b/src/lib/readiness-agent.ts index 00bbeb8..f3930be 100644 --- a/src/lib/readiness-agent.ts +++ b/src/lib/readiness-agent.ts @@ -192,7 +192,7 @@ export function openCodeStructuredOutputInstruction( rubricVersion = READINESS_RUBRIC_VERSION, ): string { return `Return exactly one JSON object with this shape and no markdown fence: -{"rubricVersion":"${rubricVersion}","languages":["string"],"applications":[{"path":".","description":"string","languages":["string"]}],"criteria":{"":{"status":"pass|fail|skipped","numerator":"integer or null","denominator":"positive integer","rationale":"string","evidence":["existing repository-relative path"]}},"warnings":["string"],"recommendations":["2 or 3 strings"],"model":"string or null"} +{"rubricVersion":"${rubricVersion}","languages":["string"],"applications":[{"path":".","description":"string","languages":["string"]}],"criteria":{"":{"status":"pass|fail|skipped","numerator":"integer or null","denominator":"positive integer","rationale":"string","evidence":["existing repository-relative path"]}},"warnings":["string"],"recommendations":["0 to 3 contextual actions for failed criteria"],"model":"string or null"} Use null numerator only for skipped criteria. Include every criterion listed in the Semantic rubric exactly once and omit fixed-decision criteria.`; } diff --git a/src/lib/readiness-contract.ts b/src/lib/readiness-contract.ts index ea6b4c3..0e74c5e 100644 --- a/src/lib/readiness-contract.ts +++ b/src/lib/readiness-contract.ts @@ -540,11 +540,10 @@ export function validateReadinessOutput( errors.push("warnings must be an array of strings."); if ( !Array.isArray(output.recommendations) || - output.recommendations.length < 2 || output.recommendations.length > 3 || output.recommendations.some((v) => typeof v !== "string") ) - errors.push("recommendations must contain 2 or 3 strings."); + errors.push("recommendations must contain at most 3 strings."); if (output.model !== null && typeof output.model !== "string") errors.push("model must be a string or null."); return errors; @@ -609,7 +608,7 @@ export function readinessJsonSchema( warnings: { type: "array", items: { type: "string" } }, recommendations: { type: "array", - minItems: 2, + minItems: 0, maxItems: 3, items: { type: "string" }, }, diff --git a/src/lib/readiness-plan.ts b/src/lib/readiness-plan.ts index ebefced..335d6c6 100644 --- a/src/lib/readiness-plan.ts +++ b/src/lib/readiness-plan.ts @@ -718,8 +718,9 @@ export function finalizeReadinessOutput( if (decision?.mode !== "semantic") return [id, decision?.result]; const agentResult = output.criteria?.[id]; if (!agentResult || agentResult.status === "skipped") { + const definition = plan.definitions.find((entry) => entry.key === id); warnings.push( - `${id} was not judged by the agent and was conservatively scored as failing.`, + `${definition?.name || id} was not judged by the agent and was conservatively scored as failing.`, ); return [ id, @@ -741,32 +742,34 @@ export function finalizeReadinessOutput( ]; }), ) as Record; - const configuredRecommendations = plan.definitions - .filter((definition) => criteria[definition.key]?.status === "fail") - .toSorted((left, right) => left.priority - right.priority) - .map((definition) => definition.recommendationTemplate.trim()) - .filter(Boolean); const agentRecommendations = Array.isArray(output.recommendations) - ? output.recommendations + ? output.recommendations.filter( + (recommendation) => + !/^Address\s+[A-Z]\.?$/i.test(recommendation.trim()), + ) : []; - const recommendations = [ - ...new Set([...configuredRecommendations, ...agentRecommendations]), - ].slice(0, 3); + const recommendations = [...new Set(agentRecommendations)].slice(0, 3); return { ...output, rubricVersion: plan.analyzerVersion, languages: plan.profile.languages, applications: plan.profile.applications, criteria, - warnings: [...new Set(warnings)], - recommendations: - recommendations.length >= 2 - ? recommendations - : [ - ...recommendations, - "Address the highest-impact failing readiness criteria.", - "Document and automate the repository's development workflow.", - ].slice(0, 3), + warnings: [ + ...new Set( + warnings.map((warning) => + plan.definitions.reduce( + (message, definition) => + message.replaceAll( + definition.key, + definition.name || definition.key, + ), + warning, + ), + ), + ), + ], + recommendations, model: typeof output.model === "string" ? output.model : null, }; } diff --git a/src/lib/readiness-profile.ts b/src/lib/readiness-profile.ts index 7bc708b..52f5158 100644 --- a/src/lib/readiness-profile.ts +++ b/src/lib/readiness-profile.ts @@ -29,7 +29,6 @@ export interface ReadinessCriterionConfig { applicability: unknown; evidenceLocators: unknown[]; decision: unknown; - recommendationTemplate: string; priority: number; } @@ -120,10 +119,6 @@ function decodeCriterion(value: unknown): ReadinessCriterionConfig { applicability: item.applicability, evidenceLocators: item.evidenceLocators, decision: item.decision, - recommendationTemplate: optionalText( - item.recommendationTemplate, - `${key}.recommendationTemplate`, - ), priority: integer(item.priority, `${key}.priority`), }; } @@ -314,7 +309,6 @@ export function bundledStandardProfile(): ReadinessProfile { applicability: { kind: "always" }, evidenceLocators: [], decision: { engine: "builtin", ruleKey: criterion.id }, - recommendationTemplate: `Address ${criterion.id.replaceAll("_", " ")}.`, priority: criterion.maturityLevel, })), }, diff --git a/tests/lib/readiness-plan.test.ts b/tests/lib/readiness-plan.test.ts index 274e800..6943f39 100644 --- a/tests/lib/readiness-plan.test.ts +++ b/tests/lib/readiness-plan.test.ts @@ -264,7 +264,15 @@ describe("readiness evaluation plan", () => { const plan = buildReadinessEvaluationPlan(root); const semanticCount = semanticCriterionIds(plan).length; - const skipped = finalizeReadinessOutput(agentOutput("skipped"), plan); + const skippedInput = agentOutput("skipped"); + const namedCriterion = plan.definitions.find( + (definition) => definition.name !== definition.key, + ); + if (!namedCriterion) + throw new Error("Expected a criterion with a display name."); + skippedInput.warnings = [`${namedCriterion.key} needs attention.`]; + skippedInput.recommendations = ["Address N.", "Document the workflow."]; + const skipped = finalizeReadinessOutput(skippedInput, plan); const passed = finalizeReadinessOutput(agentOutput("pass"), plan); expect(summarizeReadiness(skipped.criteria).criteriaTotal).toBe( @@ -276,16 +284,12 @@ describe("readiness evaluation plan", () => { ).toBeGreaterThanOrEqual(semanticCount); expect(skipped.languages).toEqual(["TypeScript"]); expect(skipped.applications).toEqual(passed.applications); - const highestPriorityFailure = plan.definitions - .filter( - (definition) => skipped.criteria[definition.key]?.status === "fail", - ) - .toSorted((left, right) => left.priority - right.priority)[0]; - if (!highestPriorityFailure) - throw new Error("Expected a failing criterion."); - expect(skipped.recommendations).toContain( - highestPriorityFailure.recommendationTemplate, + expect(skipped.recommendations).toContain("Document the workflow."); + expect(skipped.warnings).toContain( + `${namedCriterion.name} needs attention.`, ); + expect(skipped.warnings.join(" ")).not.toContain(namedCriterion.key); + expect(skipped.recommendations).not.toContain("Address N."); }); it("only skips database and API checks after high-confidence absence", () => { diff --git a/tests/lib/readiness-profile.test.ts b/tests/lib/readiness-profile.test.ts index 5fa6c8f..cafed5d 100644 --- a/tests/lib/readiness-profile.test.ts +++ b/tests/lib/readiness-profile.test.ts @@ -64,7 +64,6 @@ describe("readiness profiles", () => { criterion.description = ""; criterion.passCondition = ""; criterion.evidenceRequirement = ""; - criterion.recommendationTemplate = ""; expect( decodeReadinessProfile(profile).activeVersion.definition.criteria[0], ).toMatchObject({ @@ -72,7 +71,6 @@ describe("readiness profiles", () => { description: "", passCondition: "", evidenceRequirement: "", - recommendationTemplate: "", }); }); }); diff --git a/tests/lib/readiness-rules.test.ts b/tests/lib/readiness-rules.test.ts index e05adca..d966a92 100644 --- a/tests/lib/readiness-rules.test.ts +++ b/tests/lib/readiness-rules.test.ts @@ -39,7 +39,6 @@ function criterion( { type: "manifest_script", manifest: "package.json", name: "test" }, ], decision: { engine: "deterministic", match: "any" }, - recommendationTemplate: "Add tests.", priority: 1, ...overrides, }; diff --git a/tests/lib/readiness-upload.test.ts b/tests/lib/readiness-upload.test.ts index 7b592cc..5d16ef6 100644 --- a/tests/lib/readiness-upload.test.ts +++ b/tests/lib/readiness-upload.test.ts @@ -48,7 +48,6 @@ function deterministicProfile(): ReadinessProfile { }, ], decision: { engine: "deterministic", match: "any" }, - recommendationTemplate: "Add a test script.", priority: 1, }, ], From 4e7d1c1f216f132a10c9f7dbe7e22cd38cbdb626 Mon Sep 17 00:00:00 2001 From: Bao Tran Date: Thu, 23 Jul 2026 12:16:57 +0700 Subject: [PATCH 12/14] Update README --- README.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 36f0ee5..e602dc8 100644 --- a/README.md +++ b/README.md @@ -100,16 +100,13 @@ the agent's evidence against its versioned rubric, computes scores locally, rejects scans that change the working tree, and uploads the validated report. The report is a local agent evaluation, not a server-verified audit. -During development use `bun dev readiness`. Factory fixtures and benchmark -utilities live in the separate top-level `agent-readiness-benchmarks` package; -they are not shipped with or imported by the CLI. - Readiness uses each harness's configured default model for Claude Code and Codex; `codev readiness --model ` provides a one-off override for those harnesses. OpenCode readiness always uses `aigateway/MiniMax/MiniMax-M2.7`. Repair, timeout, and output limits are bounded product safeguards rather than user configuration. The selected harness must -first be configured through `codev install`. +first be configured through `codevhub install`. + The rubric version is deliberately not configurable: it identifies the actual report schema and scoring rubric shared by CLI, proxy, and dashboard. From ce6820d3a826a9083d4c71b35ec99bfebb5731e3 Mon Sep 17 00:00:00 2001 From: Bao Tran Date: Thu, 23 Jul 2026 14:10:43 +0700 Subject: [PATCH 13/14] test: isolate readiness gateway config - provide an explicit gateway base URL in the OpenCode readiness fixture\n- remove dependence on a developer's local auth.json during CI\n- assert that the configured gateway URL is preserved exactly --- tests/lib/readiness-contract.test.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/lib/readiness-contract.test.ts b/tests/lib/readiness-contract.test.ts index fd46bd4..82e2c14 100644 --- a/tests/lib/readiness-contract.test.ts +++ b/tests/lib/readiness-contract.test.ts @@ -212,12 +212,15 @@ describe("readiness contract", () => { it("writes a concrete gateway URL into the isolated OpenCode config", () => { const config = openCodeReadinessConfig( - { apiKey: "sk-test" }, + { + apiKey: "sk-test", + baseUrl: "https://gateway.example.test/v1", + }, "MiniMax/MiniMax-M2.7", ); - expect(config.provider.aigateway.options.baseURL).toMatch( - /^https?:\/\/.+\/v1$/, + expect(config.provider.aigateway.options.baseURL).toBe( + "https://gateway.example.test/v1", ); expect(JSON.stringify(config)).not.toContain("undefined/chat/completions"); }); From 7ee9e52e8e20c93455f34aab83604ffc9583c853 Mon Sep 17 00:00:00 2001 From: Bao Tran Date: Thu, 23 Jul 2026 16:34:56 +0700 Subject: [PATCH 14/14] fix: structure readiness recommendations - require agents to return criterion keys separately from remediation text\n- validate recommendation keys against the selected profile and failed criteria\n- resolve authoritative display names in the CLI before upload\n- preserve the existing web report API without another deployment --- src/lib/readiness-agent.ts | 5 +++- src/lib/readiness-contract.ts | 18 +++++++++-- src/lib/readiness-plan.ts | 51 ++++++++++++++++++++++---------- tests/lib/readiness-plan.test.ts | 9 ++++-- 4 files changed, 63 insertions(+), 20 deletions(-) diff --git a/src/lib/readiness-agent.ts b/src/lib/readiness-agent.ts index f3930be..e090455 100644 --- a/src/lib/readiness-agent.ts +++ b/src/lib/readiness-agent.ts @@ -179,6 +179,8 @@ Finish efficiently. CoDev has already performed a fresh deterministic inventory Evaluate every semantic rubric criterion below. Semantic criteria must be pass or fail, never skipped: failure means the repository does not provide enough evidence. Score each criterion once for the repository with denominator 1 and numerator 1 for pass or 0 for fail. Return only semantic criterion IDs in the criteria object; omit fixed decisions because CoDev merges them authoritatively. Evidence entries must be existing repository-relative file or directory paths; file paths may optionally be followed by a line number. Use an empty evidence array when the rationale describes something that is absent. Never cite a nonexistent placeholder, URL, or command as evidence; describe command evidence in the rationale instead. +For each recommendation, return the exact criterion key separately in criterionKey and put only the contextual remediation in action. Recommend only criteria that fail. Do not repeat a criterion key or display name inside action; CoDev resolves the user-facing rule name authoritatively. + Return only the JSON object matching the supplied schema. Do not include aggregate scores. Rubric version: ${plan?.analyzerVersion ?? READINESS_RUBRIC_VERSION} @@ -192,7 +194,7 @@ export function openCodeStructuredOutputInstruction( rubricVersion = READINESS_RUBRIC_VERSION, ): string { return `Return exactly one JSON object with this shape and no markdown fence: -{"rubricVersion":"${rubricVersion}","languages":["string"],"applications":[{"path":".","description":"string","languages":["string"]}],"criteria":{"":{"status":"pass|fail|skipped","numerator":"integer or null","denominator":"positive integer","rationale":"string","evidence":["existing repository-relative path"]}},"warnings":["string"],"recommendations":["0 to 3 contextual actions for failed criteria"],"model":"string or null"} +{"rubricVersion":"${rubricVersion}","languages":["string"],"applications":[{"path":".","description":"string","languages":["string"]}],"criteria":{"":{"status":"pass|fail|skipped","numerator":"integer or null","denominator":"positive integer","rationale":"string","evidence":["existing repository-relative path"]}},"warnings":["string"],"recommendations":[{"criterionKey":"failed_criterion_id","action":"contextual action without a criterion label"}],"model":"string or null"} Use null numerator only for skipped criteria. Include every criterion listed in the Semantic rubric exactly once and omit fixed-decision criteria.`; } @@ -442,6 +444,7 @@ export async function runReadinessAgent( readinessJsonSchema( plan ? semanticCriterionIds(plan) : undefined, plan?.analyzerVersion, + plan?.criteriaOrder, ), ), ); diff --git a/src/lib/readiness-contract.ts b/src/lib/readiness-contract.ts index 0e74c5e..fb642c9 100644 --- a/src/lib/readiness-contract.ts +++ b/src/lib/readiness-contract.ts @@ -32,13 +32,18 @@ export interface ReadinessApplication { languages: string[]; } +export interface AgentRecommendation { + criterionKey: string; + action: string; +} + export interface AgentReadinessOutput { rubricVersion: string; languages: string[]; applications: ReadinessApplication[]; criteria: Record; warnings: string[]; - recommendations: string[]; + recommendations: Array; model: string | null; } @@ -552,6 +557,7 @@ export function validateReadinessOutput( export function readinessJsonSchema( criterionIds: string[] = READINESS_CRITERION_IDS, rubricVersion = READINESS_RUBRIC_VERSION, + recommendationCriterionIds: string[] = criterionIds, ): Record { const criterionSchema = { type: "object", @@ -610,7 +616,15 @@ export function readinessJsonSchema( type: "array", minItems: 0, maxItems: 3, - items: { type: "string" }, + items: { + type: "object", + additionalProperties: false, + required: ["criterionKey", "action"], + properties: { + criterionKey: { enum: recommendationCriterionIds }, + action: { type: "string", minLength: 1 }, + }, + }, }, model: { type: ["string", "null"] }, }, diff --git a/src/lib/readiness-plan.ts b/src/lib/readiness-plan.ts index 335d6c6..877ce97 100644 --- a/src/lib/readiness-plan.ts +++ b/src/lib/readiness-plan.ts @@ -707,6 +707,17 @@ export function semanticCriterionIds(plan: ReadinessEvaluationPlan): string[] { ); } +function replaceCriterionKeys( + message: string, + definitions: ReadinessCriterionConfig[], +) { + return definitions.reduce( + (current, definition) => + current.replaceAll(definition.key, definition.name || definition.key), + message, + ); +} + export function finalizeReadinessOutput( output: AgentReadinessOutput, plan: ReadinessEvaluationPlan, @@ -742,13 +753,30 @@ export function finalizeReadinessOutput( ]; }), ) as Record; - const agentRecommendations = Array.isArray(output.recommendations) - ? output.recommendations.filter( - (recommendation) => - !/^Address\s+[A-Z]\.?$/i.test(recommendation.trim()), - ) - : []; - const recommendations = [...new Set(agentRecommendations)].slice(0, 3); + const failedKeys = new Set( + plan.criteriaOrder.filter((key) => criteria[key]?.status === "fail"), + ); + const recommendations = [ + ...new Set( + (Array.isArray(output.recommendations) + ? output.recommendations + : [] + ).flatMap((recommendation) => { + if ( + typeof recommendation === "string" || + !failedKeys.has(recommendation.criterionKey) || + !recommendation.action.trim() + ) + return []; + const definition = plan.definitions.find( + (entry) => entry.key === recommendation.criterionKey, + ); + return definition + ? [`${definition.name}: ${recommendation.action.trim()}`] + : []; + }), + ), + ].slice(0, 3); return { ...output, rubricVersion: plan.analyzerVersion, @@ -758,14 +786,7 @@ export function finalizeReadinessOutput( warnings: [ ...new Set( warnings.map((warning) => - plan.definitions.reduce( - (message, definition) => - message.replaceAll( - definition.key, - definition.name || definition.key, - ), - warning, - ), + replaceCriterionKeys(warning, plan.definitions), ), ), ], diff --git a/tests/lib/readiness-plan.test.ts b/tests/lib/readiness-plan.test.ts index 6943f39..7a001ef 100644 --- a/tests/lib/readiness-plan.test.ts +++ b/tests/lib/readiness-plan.test.ts @@ -271,7 +271,9 @@ describe("readiness evaluation plan", () => { if (!namedCriterion) throw new Error("Expected a criterion with a display name."); skippedInput.warnings = [`${namedCriterion.key} needs attention.`]; - skippedInput.recommendations = ["Address N.", "Document the workflow."]; + skippedInput.recommendations = [ + { criterionKey: namedCriterion.key, action: "Document the workflow." }, + ]; const skipped = finalizeReadinessOutput(skippedInput, plan); const passed = finalizeReadinessOutput(agentOutput("pass"), plan); @@ -284,7 +286,10 @@ describe("readiness evaluation plan", () => { ).toBeGreaterThanOrEqual(semanticCount); expect(skipped.languages).toEqual(["TypeScript"]); expect(skipped.applications).toEqual(passed.applications); - expect(skipped.recommendations).toContain("Document the workflow."); + expect(skipped.recommendations).toContain( + `${namedCriterion.name}: Document the workflow.`, + ); + expect(skipped.recommendations.join(" ")).not.toContain(namedCriterion.key); expect(skipped.warnings).toContain( `${namedCriterion.name} needs attention.`, );