diff --git a/lib/verification.js b/lib/verification.js index ecd59cf..1fb3cc4 100644 --- a/lib/verification.js +++ b/lib/verification.js @@ -7,15 +7,15 @@ import { detectSignals } from './signals.js'; // verdict an automatic reuse report carries is read off the final test, build, // lint or type-check command, never off how the turn ended. const VERIFICATION_COMMANDS = [ - /\b(?:npm|pnpm|yarn|bun)\s+(?:run\s+)?(?:test|build|lint|typecheck|type-check|check)\b/, - /\bnode\s+--test\b/, - /\b(?:jest|vitest|mocha|pytest|rspec|phpunit|tsc|eslint|ruff|mypy)\b/, - /\bgo\s+(?:test|build|vet)\b/, - /\bcargo\s+(?:test|build|check|clippy)\b/, - /\bdotnet\s+(?:test|build)\b/, - /\b(?:mvn|gradle|gradlew)\b/, - /\bmake\b/, - /\bclaude\s+plugin\s+(?:validate|test)\b/, + /^(?:npm|pnpm|yarn|bun)\s+(?:run\s+)?(?:test|build|lint|typecheck|type-check|check)\b/, + /^node\s+--test\b/, + /^(?:jest|vitest|mocha|pytest|rspec|phpunit|tsc|eslint|ruff|mypy)(?:\s|$)/, + /^go\s+(?:test|build|vet)\b/, + /^cargo\s+(?:test|build|check|clippy)\b/, + /^dotnet\s+(?:test|build)\b/, + /^(?:mvn|gradle|gradlew)(?:\s|$)/, + /^make(?:\s|$)/, + /^claude\s+plugin\s+(?:validate|test)\b/, ]; const FAILURE_SIGNALS = new Set(['test_failure', 'log_error']); @@ -31,8 +31,92 @@ const HEREDOC = /<<-?\s*(['"]?)(\w+)\1([^\n]*)\n[\s\S]*?\n\s*\2[ \t]*(?=\n|$)/g; const withoutHeredocBodies = (command) => command.replace(HEREDOC, '<<$2$3'); +// Keep separators inside strings, backticks and comments out of the command scan. A name in an argument is no +// check. This recognises direct commands and known wrappers; it does not evaluate arbitrary shell control flow. +// An escaped separator stays in the shell word and makes an immediately following hash literal. +const SHELL_TOKEN = /\\(?:[ \t;&|()]#|[\s\S])|\x60[^\x60]*(?:\x60|$)|'(?:''|[^'])*(?:'|$)|"(?:\\[\s\S]|\x60[\s\S]|[^"\\\x60])*(?:"|$)|(? path.replaceAll('\\', '/')); + let depth = 0; + let start = 0; + let output = ''; + for (const { 0: token, index } of input.matchAll(SHELL_TOKEN)) { + if (depth === 0) { + if (token === '(' || token === ')') continue; + const executable = literalExecutable(token); + const masked = executable ? '\0' + (quotedExecutables.push(executable) - 1) + '\0' : '\0'; + output += input.slice(start, index) + masked; + start = index + token.length; + } + if (token === '$(' || (depth > 0 && token === '(')) depth += 1; + if (depth > 0 && token === ')') { + depth -= 1; + if (depth === 0) start = index + 1; + } + } + return depth === 0 ? output + input.slice(start) : output; +} + +const WRAPPERS = [ + /^(?:[A-Za-z_]\w*=\S*\s+)+/, + /^env\s+(?:(?:-i|--ignore-environment)\s+|(?:-u|--unset|-C|--chdir)(?:=\S+|\s+\S+)\s+|[A-Za-z_]\w*=\S*\s+)*(?:--\s+)?/, + /^npx\s+(?:(?:-y|--yes|--no-install)\s+|(?:-p|--package)(?:=\S+|\s+\S+)\s+)*(?:--\s+)?/, + /^npm\s+exec\s+(?:(?:-y|--yes|--no)\s+|--package(?:=\S+|\s+\S+)\s+)*(?:--\s+)?/, + /^pnpm\s+(?:(?:-r|--recursive)\s+)?exec\s+(?:(?:-r|--recursive)\s+)?(?:--\s+)?/, + /^corepack\s+(?=(?:npm|pnpm|yarn)\s)/, + /^command\s+(?:-p\s+)?(?:--\s+)?/, + /^sudo\s+(?:(?:-n|--non-interactive)\s+|(?:-u|--user|-g|--group)(?:=\S+|\s+\S+)\s+)*(?:--\s+)?/, + /^time\s+(?:(?:-p|--portability)\s+)?(?:--\s+)?(?:!\s+)?/, + /^(?:nohup|bunx|exec)\s+(?:--\s+)?/, + /^timeout\s+(?:(?:--foreground|--preserve-status)\s+|(?:-s|--signal)(?:=\S+|\s+\S+)\s+)*(?:--\s+)?\d+(?:\.\d+)?[smhd]?\s+/, + /^(?:uv|poetry)\s+run\s+/, + /^python(?:3(?:\.\d+)?)?\s+-m\s+(?=pytest(?:\s|$))/, + /^yarn\s+(?=(?:jest|vitest)(?:\s|$))/, +]; +const executableName = (path) => path.split('/').at(-1).replace(/\.(?:exe|cmd|bat)$/i, ''); + +function literalExecutable(token) { + const quote = token[0]; + if ((quote !== "'" && quote !== '"') || token.length < 2 || !token.endsWith(quote)) return null; + const path = token.slice(1, -1); + const windows = /^[A-Za-z]:[\\/]/.test(path); + if (quote === '"' && (/[#$%!\x60"]/.test(path) || (!windows && path.includes('\\')))) return null; + const name = executableName(windows ? path.replaceAll('\\', '/') : path); + return /^[\w.-]+$/.test(name) ? name : null; +} + +function withoutExecutablePath(command, quotedExecutables) { + return command.replace(/^\S+/, (token) => { + const quoted = /^\0(\d+)\0$/.exec(token); + if (quoted) return quotedExecutables[Number(quoted[1])]; + return token.includes('\0') ? token : executableName(token); + }); +} + +// Only unwrap a literal script passed directly to a shell. Expansions in a double-quoted script belong to the outer shell. +const SHELL_SCRIPT = /^(?:[\w./-]*\/)?(?:bash|sh)[ \t]+-l?c[ \t]+(?:'([^']*)'|"([^"\x60$\\]*)")$/; + +function shellCommandOf(command) { + const script = SHELL_SCRIPT.exec(command.trim()); + return script ? script[1] ?? script[2] : command; +} + export function isVerificationCommand(command) { - return typeof command === 'string' && VERIFICATION_COMMANDS.some((pattern) => pattern.test(withoutHeredocBodies(command))); + if (typeof command !== 'string' || command.includes('\0')) return false; + const quotedExecutables = []; + return withoutShellData(shellCommandOf(command), quotedExecutables).split(/[;&|\r\n]+/).some((part) => { + let check = part.trim().replace(/^(?:(?:if|then|elif|else|do|!)\s+|[({]\s*)+/, ''); + while (check) { + // An assignment may contain slashes but is still a wrapper before the executable. + if (!/^[A-Za-z_]\w*=/.test(check)) check = withoutExecutablePath(check, quotedExecutables); + const wrapper = WRAPPERS.find((pattern) => pattern.test(check)); + if (!wrapper) return VERIFICATION_COMMANDS.some((pattern) => pattern.test(check)); + check = check.replace(wrapper, ''); + } + return false; + }); } /** @param {{ isError?: boolean, text?: string }} result */ diff --git a/test/app/turn-lifecycle.test.js b/test/app/turn-lifecycle.test.js index fdad62f..9b53c1b 100644 --- a/test/app/turn-lifecycle.test.js +++ b/test/app/turn-lifecycle.test.js @@ -35,6 +35,20 @@ test('a recalled strategy ends with the spec line, and its turn end reports it w assert.ok(scheduled.includes('drain')); }); +test('printing a check command does not verify a turn or raise its reuse evidence', async () => { + const { ports } = memoryPorts(); + await promptSubmitted(ports, { sessionId: 's1', prompt: PROMPT, ...WAIT }); + await toolFinished(ports, { sessionId: 's1', note: { kind: 'edit', paths: ['/proj/lib/redis.js'] } }); + await toolFinished(ports, { sessionId: 's1', note: { + kind: 'shell', command: 'echo "npm test"', exit: 0, output: 'npm test', label: 'Run tests', + } }); + const { record, reports } = await end(ports, 's1'); + assert.deepEqual(record.checks, []); + assert.equal((await ports.candidates.read()).s1, undefined); + assert.equal(reports[0].evidence.tier, 'none'); + assert.equal(reports[0].evidence.check, undefined); +}); + test('a verified turn becomes the candidate; a correction drops it before revising the reports', async () => { const { ports, queued } = memoryPorts(); await verifiedTurn(ports, 's1'); diff --git a/test/verification.test.js b/test/verification.test.js index a06c60b..8ced2a3 100644 --- a/test/verification.test.js +++ b/test/verification.test.js @@ -13,6 +13,102 @@ test('test, build, lint and type-check runs are verifications; other commands ar } }); +test('check names in arguments, quoted data and comments are not executions', () => { + for (const command of [ + 'echo "npm test"', 'echo npm test', 'printf "%s\\n" "make"', 'rg pytest README.md', + 'node -e "console.log(\'npm test\')"', 'git status # npm test', '# npm test\npwd', + 'echo "notes; npm test"', "printf 'notes\nnpm test\n'", 'echo `pwd; npm test`', + 'npx echo npm test', 'timeout --signal TERM 60 echo npm test', 'sudo -n echo npm test', 'npm exec echo "npm test"', + 'echo # note; npm test', 'echo bash -c "npm test"', 'bash -c "echo npm test"', 'echo $(printf x; npm test)', 'echo $(echo $(printf x); npm test)', + ]) { + assert.equal(isVerificationCommand(command), false, command); + } +}); + +test('a hash after an escaped separator stays in the current shell word', () => { + for (const command of [ + String.raw`echo arg\ #literal && npm test`, 'echo arg\\\t#literal; npm test', + String.raw`echo arg\\\ #literal; npm test`, String.raw`echo $(echo arg\ #literal) && npm test`, + String.raw`echo arg\;#literal && npm test`, + ]) assert.equal(isVerificationCommand(command), true, command); + for (const command of [ + String.raw`echo arg\\ # comment; npm test`, String.raw`echo arg\ # comment; npm test`, + 'echo "arg " # "$(; npm test\npwd', + ]) assert.equal(isVerificationCommand(command), false, command); + assert.equal(isVerificationCommand('echo # "$( ignored\nnpm test'), true); +}); + +test('command positions in chains, pipelines, environment assignments and npx wrappers still count', () => { + for (const command of [ + 'sleep 2; node --test', 'npm test 2>&1 | tail -5', 'CI=1 npm test', 'env CI=1 npm test', + 'npx --yes vitest run', 'npx --package=typescript tsc -p .', 'npx --package typescript tsc -p .', + 'echo "notes; npm test"\nnode --test', 'curl http://x/#f && npm test', 'git log --format=%h#%s; npm test', + 'cd $(git rev-parse --show-toplevel) && npm test', 'cd $(echo "$(pwd)") && npm test', + 'bash -c "npm test"', "sh -c 'cd app && npm test'", 'timeout 60 npm test', 'timeout --signal=TERM 60 npm test', 'sudo npm test', 'sudo -n npm test', 'time npm test', 'time -p npm test', 'nohup npm test', 'env -i npm test', + 'yarn vitest', 'bunx vitest', 'python -m pytest', 'uv run pytest', 'poetry run pytest', + '{ npm test; }', 'if true; then npm test; fi', 'C:\\tools\\make.exe', + ]) { + assert.equal(isVerificationCommand(command), true, command); + } +}); + +test('a known executable reached through a path still verifies, while a path in an argument does not', () => { + for (const command of [ + './gradlew test', 'gradlew.bat test', 'npm.cmd test', 'C:\\tools\\pnpm.cmd lint', './node_modules/.bin/tsc --noEmit', '/usr/bin/make', + 'cd app && ./gradlew test', 'CI=1 ./node_modules/.bin/tsc --noEmit', + '/usr/bin/npx --yes ./node_modules/.bin/tsc --noEmit', + ]) { + assert.equal(isVerificationCommand(command), true, command); + } + for (const command of [ + 'echo ./gradlew test', 'cat ./node_modules/.bin/tsc', 'echo /usr/bin/make', + 'echo "notes; ./gradlew test"', '/usr/bin/npx echo ./gradlew test', './scripts/make-not-a-check', + ]) { + assert.equal(isVerificationCommand(command), false, command); + } +}); + +test('supported wrapper options still leave the actual command in command position', () => { + for (const command of [ + 'sudo -u root npm test', 'sudo --user=root -- npm test', 'env -u CI npm test', + 'env --unset=CI --chdir app npm test', 'timeout -s KILL 60 npm test', 'time -p npm test', + 'exec npm test', '! npm test', 'time ! npm test', + ]) assert.equal(isVerificationCommand(command), true, command); + for (const command of [ + 'sudo -u root echo npm test', 'sudo -u npm test', 'env -u npm test', + 'env --chdir app echo npm test', 'exec echo npm test', '! echo npm test', + ]) assert.equal(isVerificationCommand(command), false, command); +}); + +test('literal shell and package-manager wrappers recognise only the command they execute', () => { + for (const command of [ + "/bin/bash -c 'npm test'", 'bash -lc "npm test"', "/usr/bin/bash -lc 'cd app && npm test'", + 'corepack pnpm test', 'corepack yarn vitest run', 'command npm test', 'command -p -- npm test', + 'npm exec -- vitest run', 'npm exec --yes --package typescript -- tsc --noEmit', + 'corepack npm exec --package=vitest -- vitest run', + 'pnpm exec vitest run', 'pnpm exec -- vitest run', 'corepack pnpm exec vitest run', 'pnpm -r exec jest', 'pnpm exec --recursive jest', + ]) assert.equal(isVerificationCommand(command), true, command); + for (const command of [ + "echo /bin/bash -c 'npm test'", '/bin/bash -c "echo npm test"', 'bash -lc "$CHECK"', "bash -lc\n'npm test'", + 'corepack enable pnpm', 'corepack echo npm test', 'command -v npm test', 'command -V npm test', + 'command -- echo npm test', 'command ! npm test', 'npm exec -- echo "vitest run"', 'npm exec --package=vitest -- echo vitest', + "npm exec -c 'echo vitest run'", + 'echo pnpm exec vitest run', 'pnpm exec echo vitest run', 'pnpm exec --help vitest', + ]) assert.equal(isVerificationCommand(command), false, command); +}); + +test('literal quoted executables count only in command position', () => { + for (const command of [ + '"./node_modules/.bin/jest"', "'./path with spaces/gradlew' build", + 'sudo -u "build user" "./node_modules/.bin/jest"', '"/usr/bin/env" -u CI "./node_modules/.bin/jest"', + '"C:\\Program Files\\nodejs\\npm.cmd" test', + ]) assert.equal(isVerificationCommand(command), true, command); + for (const command of [ + 'echo "./node_modules/.bin/jest"', '"npm test"', '"$runner/jest"', + '"./node_modules/.bin/jest"-other', '"notes; npm test"', + ]) assert.equal(isVerificationCommand(command), false, command); +}); + test('a non-zero exit fails the verification', () => { assert.equal(verificationFailed({ isError: true, text: '' }), true); });