From d2f5242e05021b9e8e9dcc4711fe0acc148ad35d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 08:31:37 -0700 Subject: [PATCH 01/53] ci: add a non-blocking Windows job for the oracle framework and gap CLI The framework, the eslint plugin, and the gap CLI have never run on Windows, and users report path problems there. Two diagnostic jobs run on windows-latest: the unit suites (framework, eslint plugin, gap fast oracles) and the gap CLI end to end (a fixture package written at run time, the plain and daemon paths, and one fixed-seed e2e unit). Both are continue-on-error and each step continues on error, so the first failure does not hide later ones. Neither is a required check or a dependency of preview. Co-authored-by: Isaac --- .github/workflows/pr.yml | 110 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index efd964b..e984b06 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -196,3 +196,113 @@ jobs: with: advanced-security: false annotations: true + # Diagnostic, not a gate: nothing needs these jobs, and they are not + # required checks. They show what breaks on Windows, where the oracle + # framework and the gap CLI have never run. Each step continues on error, + # so a later step still runs when an earlier one fails. + windows-unit: + name: Windows unit suites (non-blocking) + runs-on: windows-latest + continue-on-error: true + timeout-minutes: 25 + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Setup Tools + uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 + - name: Build framework + continue-on-error: true + run: pnpm --filter @tanstack/oracles run build + - name: Framework unit tests + continue-on-error: true + run: pnpm --filter @tanstack/oracles run test + - name: eslint plugin tests + continue-on-error: true + run: pnpm --filter @tanstack/eslint-plugin-oracles run test + - name: Gap report fast oracles + continue-on-error: true + run: pnpm --filter @tanstack/oracles run test:oracle-gaps:fast + windows-gap-cli: + name: Windows gap CLI end to end (non-blocking) + runs-on: windows-latest + continue-on-error: true + timeout-minutes: 25 + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Setup Tools + uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 + - name: Build framework + continue-on-error: true + run: pnpm --filter @tanstack/oracles run build + # A tiny package written at run time, not committed: one function with + # two operators and one oracle file that owns it. It is written in bash, + # which the runner provides on Windows too; the CLI steps below use the + # default shell, as a user's PowerShell or cmd would. + - name: Write gap CLI fixture + shell: bash + continue-on-error: true + run: | + set -euo pipefail + dir="$GITHUB_WORKSPACE/.ci-fixture" + mkdir -p "$dir/src" "$dir/tests" + cat > "$dir/src/clamp.ts" <<'EOT' + export function clamp(x: number): number { + return x < 0 ? 0 : x > 10 ? 10 : x + } + EOT + cat > "$dir/tests/clamp.property.test.ts" <<'EOT' + import fc from 'fast-check' + import { differs, oracleProperty, pass } from '../../packages/oracles/dist/esm/index.js' + import { clamp } from '../src/clamp' + + oracleProperty({ + id: `fixture.clamp`, + law: { + statement: `clamp returns its argument within 0 and 10.`, + authority: `the CI fixture for the Windows gap-CLI run`, + limits: `integers from -20 to 20`, + }, + histories: { + arbitrary: fc.integer({ min: -20, max: 20 }), + runs: 50, + fixedSeeds: [1], + states: [], + measured: [], + }, + timeout: 5_000, + model: (x: number) => (x < 0 ? 0 : x > 10 ? 10 : x), + driver: (x: number, observe: (checkpoint: string, value: unknown) => void) => { + observe(`result`, clamp(x)) + }, + check: (_x: number, trace: ReadonlyArray<{ checkpoint: string; value: unknown }>, expected: number) => { + const seen = trace.find(({ checkpoint }) => checkpoint === `result`)?.value + return differs(`result`, `clamp`, seen, expected) ?? pass + }, + }) + EOT + cat > "$dir/vitest.config.mjs" <<'EOT' + export default { + test: { include: ['tests/**/*.test.ts'], environment: 'node', coverage: { enabled: false } }, + } + EOT + # The user-facing path: the default shell, no supervisor on Windows. + - name: Gap CLI on the fixture + continue-on-error: true + run: node packages/oracles/oracle-gaps/index.mjs --package .ci-fixture --source src/clamp.ts --config vitest.config.mjs + # The daemon path: start it, check through it, then stop it. + - name: Gap CLI through the daemon + continue-on-error: true + run: node packages/oracles/oracle-gaps/index.mjs --package .ci-fixture --source src/clamp.ts --config vitest.config.mjs --daemon + - name: Stop the gap CLI daemon + if: always() + continue-on-error: true + run: node packages/oracles/oracle-gaps/index.mjs --stop-daemon --package .ci-fixture + # One fixed-seed unit of the end-to-end oracle, as the Linux job runs it. + - name: Gap report end-to-end oracle (fixed seed) + continue-on-error: true + run: pnpm --filter @tanstack/oracles exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "fixed seed" From 0fc897b4b8a5a115b292592552eb79372e60063a Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 09:02:24 -0700 Subject: [PATCH 02/53] ci(windows): diagnostic split of the fast suites, fixture package.json Diagnostic only, not a fix. The first run showed the gap fast suites and the fixed-seed end-to-end unit hang on windows-latest (cancelled at the job timeout, with no output after the static oracle and the e2e calibrations started). The fast suites now run one file per step with a step timeout and verbose output for the static oracle, so the hang is located. The fixture gets the package.json it was missing (the CLI reported ENOENT for it). The job timeouts are raised to 45 minutes so the split steps can finish. Co-authored-by: Isaac --- .github/workflows/pr.yml | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index e984b06..2fb7928 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -204,7 +204,7 @@ jobs: name: Windows unit suites (non-blocking) runs-on: windows-latest continue-on-error: true - timeout-minutes: 25 + timeout-minutes: 45 steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -221,14 +221,30 @@ jobs: - name: eslint plugin tests continue-on-error: true run: pnpm --filter @tanstack/eslint-plugin-oracles run test - - name: Gap report fast oracles + # [diagnostic] The fast suites run one file per step, each with a step + # timeout, to find which one hangs on Windows (the run was cancelled at + # 21 minutes with no output after the static oracle started). + - name: '[diagnostic] Gap report static oracle' continue-on-error: true - run: pnpm --filter @tanstack/oracles run test:oracle-gaps:fast + timeout-minutes: 8 + run: pnpm --filter @tanstack/oracles exec vitest --run --config oracle-gaps/vitest.config.mjs --reporter=verbose gap-report-static + - name: '[diagnostic] Gap report classify oracle' + continue-on-error: true + timeout-minutes: 8 + run: pnpm --filter @tanstack/oracles exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-classify + - name: '[diagnostic] Gap report probe-form oracle' + continue-on-error: true + timeout-minutes: 8 + run: pnpm --filter @tanstack/oracles exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-probe-form + - name: '[diagnostic] Gap report mutant-validity oracle' + continue-on-error: true + timeout-minutes: 10 + run: pnpm --filter @tanstack/oracles exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-mutant-validity windows-gap-cli: name: Windows gap CLI end to end (non-blocking) runs-on: windows-latest continue-on-error: true - timeout-minutes: 25 + timeout-minutes: 45 steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -250,6 +266,9 @@ jobs: set -euo pipefail dir="$GITHUB_WORKSPACE/.ci-fixture" mkdir -p "$dir/src" "$dir/tests" + cat > "$dir/package.json" <<'EOT' + {"name": "ci-fixture", "private": true, "type": "module"} + EOT cat > "$dir/src/clamp.ts" <<'EOT' export function clamp(x: number): number { return x < 0 ? 0 : x > 10 ? 10 : x @@ -303,6 +322,8 @@ jobs: continue-on-error: true run: node packages/oracles/oracle-gaps/index.mjs --stop-daemon --package .ci-fixture # One fixed-seed unit of the end-to-end oracle, as the Linux job runs it. + # [diagnostic] Verbose and a step timeout, to see which copy hangs. - name: Gap report end-to-end oracle (fixed seed) continue-on-error: true - run: pnpm --filter @tanstack/oracles exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "fixed seed" + timeout-minutes: 12 + run: pnpm --filter @tanstack/oracles exec vitest --run --config oracle-gaps/vitest.config.mjs --reporter=verbose gap-report-oracle -t "fixed seed" From 72b949554c79ef7f8c5b8bca00b5f7922418dece Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 09:24:49 -0700 Subject: [PATCH 03/53] ci(windows): diagnostic replay, error-log dump, and discovered-config run Diagnostic only. Adds a bash replay of the first failing static unit (its verdict is not printed by the run), a run of the fixture with the discovered config instead of --config, and a dump of the gap CLI's error.log files from the OS temp directory, which the daemon run's failure points to. Co-authored-by: Isaac --- .github/workflows/pr.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 2fb7928..39e8317 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -228,6 +228,16 @@ jobs: continue-on-error: true timeout-minutes: 8 run: pnpm --filter @tanstack/oracles exec vitest --run --config oracle-gaps/vitest.config.mjs --reporter=verbose gap-report-static + # [diagnostic] Replays the first failing static unit alone, with bash, to + # print its verdict (the run's own output names the law only through a + # counterexample). + - name: '[diagnostic] Replay the first failing static unit' + continue-on-error: true + timeout-minutes: 6 + shell: bash + run: | + export ORACLE_SEED=20261105 ORACLE_PATH=0:0:0:0:0:0:0:0:0:0:0:0:0:0:0 ORACLE_PROPERTY='oracle-gaps.report-static' ORACLE_UNIT='random seed: every static state' + pnpm --filter @tanstack/oracles exec vitest run --config oracle-gaps/vitest.config.mjs -t '^oracle-gaps\.report-static(?: | > )random seed: every static state$' gap-report-static - name: '[diagnostic] Gap report classify oracle' continue-on-error: true timeout-minutes: 8 @@ -321,6 +331,22 @@ jobs: if: always() continue-on-error: true run: node packages/oracles/oracle-gaps/index.mjs --stop-daemon --package .ci-fixture + # [diagnostic] The fixture without --config, to see whether the discovered + # config hits the same path bug as --config. + - name: '[diagnostic] Gap CLI on the fixture, discovered config' + continue-on-error: true + run: node packages/oracles/oracle-gaps/index.mjs --package .ci-fixture --source src/clamp.ts + # [diagnostic] The daemon's error logs live in the OS temp directory, + # which the job does not print: print them here. + - name: '[diagnostic] Print gap CLI error logs' + if: always() + continue-on-error: true + shell: pwsh + run: | + Get-ChildItem -Path $env:TEMP -Recurse -Filter error.log -ErrorAction SilentlyContinue | ForEach-Object { + "== $($_.FullName)" + Get-Content $_.FullName -TotalCount 120 + } # One fixed-seed unit of the end-to-end oracle, as the Linux job runs it. # [diagnostic] Verbose and a step timeout, to see which copy hangs. - name: Gap report end-to-end oracle (fixed seed) From dacfb88ad56ad56de58f84ad991185559c843102 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 10:52:43 -0700 Subject: [PATCH 04/53] fix(oracle-gaps): one path rule for Windows, and shell-aware replay lines Forward slashes at entry, native only at the file system. Paths enter the CLI as argv, as Vite and Vitest report them, and as realpath returns them; comparisons and keys use one spelling (oracle-gaps/paths.mjs). - The config wrapper imports the config by a POSIX specifier. A backslash in an ES module specifier is a character, so every run with a config was refused on Windows (check.mjs, wrapper). - Mutant copies are keyed and served in forward slashes, as Vite resolves and Vitest reports them. The copy map was keyed by native paths, so the copies were never served on Windows. - Module IDs are kept by isAbsolute, not by a leading slash: a Windows ID is `C:/...`. - The package root, the located packages, and the source names use the native real path on both sides (realPathOf). The JavaScript realpath kept 8.3 short names such as RUNNER~1 and refused every owns entry. - The daemon listens on a named pipe on win32 (no socket file to remove), keys by the same real path as the CLI, and its timeout names the end of its log. - A package with no package.json is refused, not an internal error. - Replay and rerun lines print PowerShell and cmd forms on Windows; the POSIX form is unchanged. Co-authored-by: Isaac --- packages/oracles/oracle-gaps/args.mjs | 23 ++--- .../oracles/oracle-gaps/capture-setup.mjs | 4 +- packages/oracles/oracle-gaps/check.mjs | 53 +++++++---- packages/oracles/oracle-gaps/daemon.mjs | 26 +++++- packages/oracles/oracle-gaps/evidence.mjs | 14 ++- packages/oracles/oracle-gaps/paths.mjs | 45 +++++++++ packages/oracles/oracle-gaps/report.mjs | 4 +- packages/oracles/oracle-gaps/stream.mjs | 29 +++++- packages/oracles/src/oracle-property.ts | 91 +++++++++++++++++-- 9 files changed, 236 insertions(+), 53 deletions(-) create mode 100644 packages/oracles/oracle-gaps/paths.mjs diff --git a/packages/oracles/oracle-gaps/args.mjs b/packages/oracles/oracle-gaps/args.mjs index bd4b15b..f3e6c69 100644 --- a/packages/oracles/oracle-gaps/args.mjs +++ b/packages/oracles/oracle-gaps/args.mjs @@ -1,17 +1,12 @@ // The CLI's arguments, shared by the CLI (index.mjs) and the daemon // (daemon.mjs), which parses each request's arguments the same way. -import { - existsSync, - readFileSync, - readdirSync, - realpathSync, - statSync, -} from 'node:fs' +import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs' import { randomBytes } from 'node:crypto' import { availableParallelism } from 'node:os' import { dirname, isAbsolute, join, relative, resolve } from 'node:path' import { loadAuthored } from './authored.mjs' import { stateDirOf } from './evidence.mjs' +import { posix, realPathOf } from './paths.mjs' import { Refusal } from './ending.mjs' const flags = new Set([`daemon`, `stop-daemon`, `ordinary`, `help`]) @@ -114,8 +109,14 @@ function optionsOf(args, cwd) { ) // Test reports name files by their real path (on macOS, /tmp is // /private/tmp), so the package path is resolved the same way. - const pkg = realpathSync.native(at) + const pkg = realPathOf(at) if (args[`stop-daemon`]) return { stop: true, pkg } + // A package is a directory with a manifest: every check reads it. + if (!existsSync(join(pkg, `package.json`))) + throw new Refusal( + `--package ${args.package} has no package.json`, + `Pass --package the directory of a package, which has a package.json.`, + ) if (!args.source) throw new Refusal( `--source is required`, @@ -244,14 +245,14 @@ export function packagePath(pkg, path) { // filesystem that ignores case. const named = relative( pkg, - existsSync(absolute) ? realpathSync.native(absolute) : absolute, + existsSync(absolute) ? realPathOf(absolute) : absolute, ) if (isOutside(named)) throw new Refusal( `${path} is outside the package`, `Name a file inside the package, relative to --package.`, ) - return named + return posix(named) } /** A run's directory name: its start time, then random bytes, unique per run. */ @@ -366,7 +367,7 @@ function locate(from, name, stamps) { stamps.push([path, stampOf(path)]) const candidate = join(dir, `node_modules`, name) if (existsSync(join(candidate, `package.json`))) - return realpathSync(candidate) + return realPathOf(candidate) if (dirname(dir) === dir) return undefined } } diff --git a/packages/oracles/oracle-gaps/capture-setup.mjs b/packages/oracles/oracle-gaps/capture-setup.mjs index 6b9db66..8872333 100644 --- a/packages/oracles/oracle-gaps/capture-setup.mjs +++ b/packages/oracles/oracle-gaps/capture-setup.mjs @@ -26,7 +26,7 @@ import { createHash } from 'node:crypto' import { appendFileSync, readFileSync } from 'node:fs' import { createRequire, syncBuiltinESMExports } from 'node:module' -import { dirname, join } from 'node:path' +import { dirname, isAbsolute, join } from 'node:path' import { afterAll, beforeEach, chai, expect, inject, vi } from 'vitest' const run = @@ -1023,7 +1023,7 @@ afterAll(() => { const loaded = [...(evaluated?.idToModuleMap ?? [])] .filter(([, node]) => node.evaluated) .map(([id]) => id.split(`?`)[0]) - .filter((path) => path.startsWith(`/`)) + .filter((path) => isAbsolute(path)) appendFileSync( state.capture, `${JSON.stringify({ file: thisFile, closed: true, loaded, ...(skippedCopy ? { skipped: true } : {}) })}\n`, diff --git a/packages/oracles/oracle-gaps/check.mjs b/packages/oracles/oracle-gaps/check.mjs index a1139a1..ddc1119 100644 --- a/packages/oracles/oracle-gaps/check.mjs +++ b/packages/oracles/oracle-gaps/check.mjs @@ -36,7 +36,13 @@ import { } from 'node:fs' import { createRequire, isBuiltin, syncBuiltinESMExports } from 'node:module' import { constants, hostname } from 'node:os' -import { basename, dirname, join, relative as relativePath } from 'node:path' +import { + basename, + dirname, + isAbsolute, + join, + relative as relativePath, +} from 'node:path' import { fileURLToPath, pathToFileURL } from 'node:url' import { applyAuthored, loadAuthored } from './authored.mjs' import { @@ -57,6 +63,13 @@ import { } from './stream.mjs' import { analyze, isRunnerCopy } from './analysis.mjs' import { Refusal } from './ending.mjs' +import { + baseOf, + isWithin, + posix, + relativeWithin, + specifierOf, +} from './paths.mjs' import { stateDirOf } from './evidence.mjs' import { baselineOf, @@ -118,7 +131,9 @@ function serveCopies() { promises.readFile = function (path, options, ...rest) { const text = typeof path === `string` - ? globalThis.__oracleGapsCopies?.get(path) + ? globalThis.__oracleGapsCopies?.get( + path.replaceAll(String.fromCharCode(92), '/'), + ) : undefined if (text === undefined) return readFile.call(this, path, options, ...rest) const encoded = @@ -456,7 +471,7 @@ export async function openSession(pkg) { import { mergeConfig } from 'vitest/config' ${ config - ? `import * as config from ${JSON.stringify(relativePath(home, join(pkg, config)))} + ? `import * as config from ${JSON.stringify(specifierOf(home, join(pkg, config)))} // The package's config, read from its namespace at run time: a config with // no default export then fails as it fails under Vitest, when it loads, and // not when Vite bundles this wrapper, which esbuild reports on stderr @@ -482,7 +497,7 @@ export default async (env) => { resolveId(source) { const copies = globalThis.__oracleGapsCopies for (const path of [source, ${JSON.stringify(pkg)} + source]) - if (copies?.has(path)) return path + if (copies?.has(path.replaceAll(String.fromCharCode(92), '/'))) return path return globalThis.__oracleGapsBundles?.get(source) }, load(id) { @@ -493,7 +508,7 @@ export default async (env) => { const at = id.indexOf('?') if (at >= 0 && [...new URLSearchParams(id.slice(at + 1)).keys()].every((key) => ['raw', 'url', 'worker', 'sharedworker', 'inline'].includes(key))) return undefined const path = at < 0 ? id : id.slice(0, at) - const copy = globalThis.__oracleGapsCopies?.get(path) + const copy = globalThis.__oracleGapsCopies?.get(path.replaceAll(String.fromCharCode(92), '/')) if (copy !== undefined) return copy // This context's authored patch, or else the instrumented source of the // current check, both set by the runner in this process. @@ -559,7 +574,7 @@ export default async (env) => { ) .replaceAll(wrapper, config === undefined ? name : join(pkg, config)) .replaceAll(basename(wrapper), name)}`, - `Make ${name} load under Vitest: run vitest --run in the package and fix the error it reports.`, + `Fix the error that the message names in ${name}, then run the check again.`, ) }) // Each warm context is a Vitest instance, and each listens on the process @@ -747,7 +762,7 @@ export default async (env) => { // This process's harness files are generated from this CLI's own files, // which every key covers, and their directory names the process, so they // are left out of every key and input. - const ownHarness = (path) => path.startsWith(`${home}/`) + const ownHarness = (path) => isWithin(home, path) // The files the Vite config loaded, for the global key. const configFiles = () => (contexts[0]?.vite?.config?.configFileDependencies ?? []).filter( @@ -875,7 +890,7 @@ export default async (env) => { // A run that spun is a timeout as it stands, even one that then ended. const spun = spinRow() const profiled = { - file: file.split(`/`).at(-1), + file: baseOf(file), mutant, select: select?.length ?? null, limitMs, @@ -987,8 +1002,7 @@ export default async (env) => { ...loaded, ]), ].filter( - (path) => - path.startsWith(`/`) && !ownHarness(path) && !isRunnerCopy(path), + (path) => isAbsolute(path) && !ownHarness(path) && !isRunnerCopy(path), ) } @@ -1008,9 +1022,11 @@ export default async (env) => { // file's do, and the wrapper's plugin serves its text: nothing is // written there. const copies = items.map((item) => { - const copy = join( - dirname(item.file), - `.oracle-gaps-${process.pid}-${++copyCounter}-${basename(item.file)}`, + const copy = posix( + join( + dirname(item.file), + `.oracle-gaps-${process.pid}-${++copyCounter}-${basename(item.file)}`, + ), ) globalThis.__oracleGapsCopies.set(copy, readFileSync(item.file, `utf8`)) return copy @@ -1133,7 +1149,9 @@ export default async (env) => { .filter(Boolean)) { const row = JSON.parse(line) const copy = copies.find( - (c) => row.key?.startsWith(`${c} > `) || row.file === c, + (c) => + posix(row.key ?? ``).startsWith(`${c} > `) || + posix(row.file ?? ``) === c, ) if (copy && row.skipped) skipped.add(copy) if (copy && row.closed) loaded.set(copy, row.loaded ?? []) @@ -1481,7 +1499,7 @@ export default async (env) => { const looked = Date.now() const valid = lookup(id, index) span(`lookup`, looked, { - file: file.split(`/`).at(-1), + file: baseOf(file), spec, hit: Boolean(valid), }) @@ -2377,9 +2395,8 @@ function identityOf(options, authored, testFiles, oracleCount) { // the order of those names, so the same inputs in another checkout, or // the same CLI in another directory, have the same identity. const name = (file) => - file.startsWith(`${options.pkg}/`) - ? file.slice(options.pkg.length + 1) - : `cli:${relativePath(here, file)}` + relativeWithin(options.pkg, file) ?? + `cli:${posix(relativePath(here, file))}` const files = [ ...options.sources.map((file) => join(options.pkg, file)), ...testFiles, diff --git a/packages/oracles/oracle-gaps/daemon.mjs b/packages/oracles/oracle-gaps/daemon.mjs index d887d58..6ea0215 100644 --- a/packages/oracles/oracle-gaps/daemon.mjs +++ b/packages/oracles/oracle-gaps/daemon.mjs @@ -43,7 +43,8 @@ import { connect, createServer } from 'node:net' import { dirname, join, resolve } from 'node:path' import { fileURLToPath, pathToFileURL } from 'node:url' import { Refusal } from './ending.mjs' -import { socketOf, stateDirOf } from './evidence.mjs' +import { removeSocket, socketOf, stateDirOf } from './evidence.mjs' +import { realPathOf } from './paths.mjs' const here = dirname(fileURLToPath(import.meta.url)) const idleMs = 30 * 60_000 @@ -110,6 +111,19 @@ const errorSince = (pkg, since) => { return undefined } +/** The end of the package's daemon log, for a failure that names none. */ +function daemonLogTail(pkg) { + try { + const lines = readFileSync( + join(stateDirOf(pkg), `daemon.log`), + `utf8`, + ).split(`\n`) + return `; its log ends:\n${lines.slice(-30).join(`\n`)}` + } catch { + return `; it wrote no log` + } +} + /** Starts a daemon for the package, detached from this process. */ function startDaemon(pkg) { mkdirSync(stateDirOf(pkg), { recursive: true }) @@ -180,7 +194,7 @@ export async function requestDaemon(pkg, argv, code) { if (reply) mayStart = true if (Date.now() - began - held > WAIT_MS) throw new Error( - `the package's daemon did not answer within ${WAIT_MS / 1000} s`, + `the package's daemon did not answer within ${WAIT_MS / 1000} s${daemonLogTail(pkg)}`, ) await sleep(100) } @@ -277,7 +291,7 @@ async function serve(pkg) { const socket = socketOf(pkg) if (!(await takeLock(lock, socket))) process.exit(0) // A dead daemon's socket; this daemon holds the lock. - rmSync(socket, { force: true }) + removeSocket(socket) let closing = false // Why the daemon could not load or open its session. let failure @@ -377,7 +391,7 @@ async function serve(pkg) { await session?.close() await store?.close() if (readText(lock) === token) { - rmSync(socket, { force: true }) + removeSocket(socket) rmSync(lock, { force: true }) } server.close() @@ -419,5 +433,7 @@ async function serve(pkg) { if (import.meta.url === pathToFileURL(realpathSync(process.argv[1])).href) { const at = process.argv.indexOf(`--package`) - await serve(realpathSync.native(resolve(process.argv[at + 1]))) + // The CLI keys the daemon by the same real path (paths.mjs), so the two + // compute one socket and one state directory. + await serve(realPathOf(resolve(process.argv[at + 1]))) } diff --git a/packages/oracles/oracle-gaps/evidence.mjs b/packages/oracles/oracle-gaps/evidence.mjs index 6becbe9..7a1d4bd 100644 --- a/packages/oracles/oracle-gaps/evidence.mjs +++ b/packages/oracles/oracle-gaps/evidence.mjs @@ -10,6 +10,7 @@ import { mkdirSync, readFileSync, renameSync, + rmSync, writeFileSync, } from 'node:fs' import { createHash } from 'node:crypto' @@ -18,9 +19,18 @@ import { join } from 'node:path' // A package's key: its real path, which names the repository checkout too. const keyOf = (pkg) => createHash(`sha1`).update(pkg).digest(`hex`).slice(0, 12) -/** The socket of a package's daemon. A socket path must stay short. */ +/** + * The socket of a package's daemon. A Unix socket path must stay short, and + * on Windows a socket is a named pipe, which has no file to remove. + */ export const socketOf = (pkg) => - join(tmpdir(), `oracle-gaps-${keyOf(pkg)}.sock`) + process.platform === `win32` + ? `\\\\.\\pipe\\oracle-gaps-${keyOf(pkg)}` + : join(tmpdir(), `oracle-gaps-${keyOf(pkg)}.sock`) +/** Removes a daemon's socket file; a named pipe has none. */ +export const removeSocket = (socket) => { + if (process.platform !== `win32`) rmSync(socket, { force: true }) +} /** * A package's state: the daemon's lock, log, and evidence, the workspace * dependencies' one-file bundles, and the default output directories. It is diff --git a/packages/oracles/oracle-gaps/paths.mjs b/packages/oracles/oracle-gaps/paths.mjs new file mode 100644 index 0000000..7de4db3 --- /dev/null +++ b/packages/oracles/oracle-gaps/paths.mjs @@ -0,0 +1,45 @@ +// One rule for paths inside the gap CLI. A path is written with forward +// slashes as it enters the CLI (its argv, the paths Vite and Vitest report, +// and the results of realpath), so every comparison and every key is a +// comparison of one spelling. Node's file system calls accept forward slashes +// on Windows too, so a path goes back to the file system as it is. The one +// real path, `realpathSync.native`, is taken on both sides of a comparison: +// the JavaScript `realpathSync` keeps an 8.3 short name (RUNNER~1), which the +// native call expands, so the two never agree on Windows. +import { realpathSync } from 'node:fs' +import { relative } from 'node:path' + +/** The forward-slash spelling of `path`. */ +export const posix = (path) => path.replaceAll(`\\`, `/`) + +/** The real path of `path`, in forward slashes. */ +export const realPathOf = (path) => posix(realpathSync.native(path)) + +/** + * The forward-slash path of `file` relative to `dir` when `file` is `dir` or + * inside it, and undefined otherwise. Both are compared in one spelling, so a + * drive letter's case, or a `\` from a join, does not decide it. + */ +export function relativeWithin(dir, file) { + const base = posix(dir).replace(/\/+$/, ``) + const path = posix(file) + if (path === base) return `.` + if (!path.startsWith(`${base}/`)) return undefined + return path.slice(base.length + 1) +} + +/** Whether `file` is `dir` or lies inside it. */ +export const isWithin = (dir, file) => relativeWithin(dir, file) !== undefined + +/** + * An ES module specifier for `to` from the directory `from`: forward slashes, + * and starting with `./` or `../`, as an import needs. The one the generated + * config wrapper uses; a backslash in it is a character, not a separator. + */ +export function specifierOf(from, to) { + const rel = posix(relative(from, to)) + return /^\.\.?\//.test(rel) ? rel : `./${rel}` +} + +/** The last segment of `path`, in forward slashes. */ +export const baseOf = (path) => posix(path).split(`/`).at(-1) diff --git a/packages/oracles/oracle-gaps/report.mjs b/packages/oracles/oracle-gaps/report.mjs index 59b394f..f9b0390 100644 --- a/packages/oracles/oracle-gaps/report.mjs +++ b/packages/oracles/oracle-gaps/report.mjs @@ -30,6 +30,7 @@ import { streamLine, verdictLine, } from './stream.mjs' +import { relativeWithin } from './paths.mjs' /** Outcomes that are gaps. Only a law verdict on a reproducible route kills. */ export const gapOutcomes = new Set([ @@ -419,8 +420,7 @@ export function renderStdout({ (r) => !r.probeGuard && r.outcome === `unowned`, ) const killed = results.filter((r) => r.outcome === `killed` && !r.probeGuard) - const relative = (file) => - file.startsWith(`${pkg}/`) ? file.slice(pkg.length + 1) : file + const relative = (file) => relativeWithin(pkg, file) ?? file const section = baselineSection(results, baseline, config) // Every line is a line of the stream (stream.mjs): the totals, one // directive per gap class, each function's heading over its gap lines, diff --git a/packages/oracles/oracle-gaps/stream.mjs b/packages/oracles/oracle-gaps/stream.mjs index ed78725..79db349 100644 --- a/packages/oracles/oracle-gaps/stream.mjs +++ b/packages/oracles/oracle-gaps/stream.mjs @@ -77,6 +77,21 @@ export function testNamesOf(testCase) { // A word that a POSIX shell reads back as it is: plain, or in single quotes. const shellWord = (word) => /^[\w./-]+$/.test(word) ? word : `'${word.replaceAll(`'`, `'\\''`)}'` +// The same word for PowerShell (a single quote is doubled) and for cmd (a +// double-quoted word, which cmd reads as it is). +const powerShellWord = (word) => + /^[\w./-]+$/.test(word) ? word : `'${word.replaceAll(`'`, `''`)}'` +const cmdWord = (word) => + /^[\w./-]+$/.test(word) ? word : `"${word.replaceAll(`"`, `\\"`)}"` +// The replay line of a command. A POSIX shell reads one line; on Windows +// there are two, one for PowerShell and one for cmd, since the words differ. +const replayLines = (words) => + process.platform === `win32` + ? [ + ` replay (PowerShell): ${words.map(powerShellWord).join(` `)}`, + ` replay (cmd): ${words.map(cmdWord).join(` `)}`, + ] + : [` replay: ${words.map(shellWord).join(` `)}`] // A control character, or a line or paragraph separator, as its `\uXXXX` // escape: the line stays one line, and the escape is regular-expression @@ -141,8 +156,8 @@ function violationLines(message) { export function baselineLines(files, config) { const vitest = config === undefined - ? `vitest run` - : `vitest run --config ${shellWord(config)}` + ? [`vitest`, `run`] + : [`vitest`, `run`, `--config`, config] return files.flatMap(({ file, ordinary, failure, tests }) => { const name = `${file}${ordinary ? ` (ordinary)` : ``}` const ran = tests.filter( @@ -164,7 +179,7 @@ export function baselineLines(files, config) { `baseline`, `fail ${name}: ${tests.length === 0 ? `did not load` : `failed outside its tests`}: ${codeSpan(escapeControls(failure.split(`\n`)[0]))}`, ), - ` replay: ${vitest} ${shellWord(file)}`, + ...replayLines([...vitest, file]), ] const message = first.message ?? `` return [ @@ -174,7 +189,13 @@ export function baselineLines(files, config) { ), ` failing: ${codeSpan(escapeControls((first.names ?? [first.fullName]).join(` > `)))}: ${codeSpan(escapeControls(message.split(`\n`)[0]))}`, ...violationLines(message), - ` replay: ${vitest} ${shellWord(file)}${first.names === undefined ? `` : ` -t ${shellWord(escapeControls(replayPattern(first.names)))}`}`, + ...replayLines([ + ...vitest, + file, + ...(first.names === undefined + ? [] + : [`-t`, escapeControls(replayPattern(first.names))]), + ]), ] }) } diff --git a/packages/oracles/src/oracle-property.ts b/packages/oracles/src/oracle-property.ts index 8ef46d1..4054df5 100644 --- a/packages/oracles/src/oracle-property.ts +++ b/packages/oracles/src/oracle-property.ts @@ -2201,7 +2201,7 @@ function failure( options: { id: string; law: OracleLaw }, verdict: Extract, history: THistory, - rerun: string | undefined, + rerun: Command | undefined, ): OracleViolation { globalThis.__oracleVerdict?.({ property: options.id, @@ -2216,7 +2216,7 @@ function violationError( options: { id: string; law: OracleLaw }, verdict: Extract, history: THistory, - rerun: string | undefined, + rerun: Command | undefined, ): OracleViolation { const { id, law } = options const error = new OracleViolation( @@ -2227,7 +2227,7 @@ function violationError( `History: ${reportText(history)}`, `Authority: ${law.authority}`, `Known limits: ${law.limits}`, - ...(rerun === undefined ? [] : [`Rerun: ${rerun}`]), + ...commandLines(`Rerun`, rerun), ].join(`\n`), verdict.cause === undefined ? undefined : { cause: verdict.cause }, ) @@ -2246,12 +2246,76 @@ function violationError( * a space, Vitest 5 joined by ` > `, so the pattern accepts either between * them. */ -function rerunOf(task: NamedTask): string { +function rerunPatternOf(task: NamedTask): string { const names: Array = [] for (let at: NamedTask | undefined = task; at; at = at.suite) names.unshift(at.name) const escape = (text: string) => text.replace(/[.*+?^${}()|[\]\\]/g, `\\$&`) - return `-t ${shellQuoted(`^${names.map(escape).join(`(?: | > )`)}$`)}` + return `^${names.map(escape).join(`(?: | > )`)}$` +} + +function rerunOf(task: NamedTask): Command { + return commandOf([], undefined, rerunPatternOf(task)) +} + +/** + * A command that a failure prints, in the form each shell on the platform + * reads: POSIX words, or on Windows a PowerShell line and a cmd line. The + * POSIX form is the one a failure always printed. + */ +interface Command { + readonly posix: string + readonly powershell: string + readonly cmd: string +} + +/** + * The command that sets `env` (a variable's `quote` flag says whether its + * POSIX value is quoted), runs `command`, and reruns the test `pattern`. + */ +function commandOf( + env: ReadonlyArray, + command: string | undefined, + pattern: string, +): Command { + const psQuoted = (text: string) => `'${text.replace(/'/g, `''`)}'` + const cmdQuoted = (text: string) => `"${text.replace(/"/g, `\\"`)}"` + const words = (parts: ReadonlyArray) => + parts.filter((part) => part !== undefined && part !== ``).join(` `) + return { + posix: words([ + ...env.map( + ([name, value, quote]) => + `${name}=${quote ? shellQuoted(value) : value}`, + ), + command, + `-t ${shellQuoted(pattern)}`, + ]), + powershell: words([ + ...env.map(([name, value]) => `$env:${name}=${psQuoted(value)};`), + command, + `-t ${psQuoted(pattern)}`, + ]), + cmd: words([ + ...env.map(([name, value]) => `set "${name}=${value}" &&`), + command, + `-t ${cmdQuoted(pattern)}`, + ]), + } +} + +/** The lines that print a command under `label`: one per shell on Windows. */ +function commandLines( + label: string, + command: Command | undefined, +): Array { + if (command === undefined) return [] + return process.platform === `win32` + ? [ + `${label} (PowerShell): ${command.powershell}`, + `${label} (cmd): ${command.cmd}`, + ] + : [`${label}: ${command.posix}`] } /** Text in a POSIX shell's single quotes. */ @@ -2350,7 +2414,7 @@ function generatedTests( shrinkUntil: number, // The rerun line a violation prints; a random unit's comes only once // fast-check reports its path. - rerun: string | undefined, + rerun: Command | undefined, // Each violation thrown, with its verdict and history. violations: Map, ) => { @@ -2489,7 +2553,16 @@ function generatedTests( const replay = path === null ? undefined - : `${oracleEnvName(`SEED`)}=${details.seed} ${oracleEnvName(`PATH`)}=${path} ${oracleEnvName(`PROPERTY`)}=${shellQuoted(id)} ${oracleEnvName(`UNIT`)}=${shellQuoted(replayed.name)} vitest run ${rerunOf(replayed)}` + : commandOf( + [ + [oracleEnvName(`SEED`), `${details.seed}`, false], + [oracleEnvName(`PATH`), `${path}`, false], + [oracleEnvName(`PROPERTY`), id, true], + [oracleEnvName(`UNIT`), replayed.name, true], + ], + `vitest run`, + rerunPatternOf(replayed), + ) // A random unit's violation names its replay as its rerun, so it is // built again, once, now that the path is known; without a path it // prints no rerun line. @@ -2501,8 +2574,8 @@ function generatedTests( throw new Error( [ report, - ...(rerun === undefined ? [] : [`Rerun: ${rerun}`]), - ...(replay === undefined ? [] : [`Replay: ${replay}`]), + ...commandLines(`Rerun`, rerun), + ...commandLines(`Replay`, replay), ].join(`\n`), { cause }, ) From 8d96432aae958701b20ac5fd2455ec63a0798e63 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 10:52:46 -0700 Subject: [PATCH 05/53] test: portable npm spawns, a segment rule for production files, native realpath - npm is an npm.cmd shim on Windows; spawnSync runs it through a shell there (package.test.ts, shipped-citations.test.js). - The probe-form oracle's model decides production by a directory named src, split on either separator, not by a substring with `/`. Production was right; the model was wrong on Windows. - The owns test uses the native real path, as the CLI does. Co-authored-by: Isaac --- .../tests/probe-form-oracle.property.test.js | 5 ++++- .../tests/shipped-citations.test.js | 3 ++- .../tests/oracle-gaps-owns-through-calls.test.ts | 5 ++++- packages/oracles/tests/package.test.ts | 15 +++++++++++++-- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/packages/eslint-plugin-oracles/tests/probe-form-oracle.property.test.js b/packages/eslint-plugin-oracles/tests/probe-form-oracle.property.test.js index 8701be8..66031f2 100644 --- a/packages/eslint-plugin-oracles/tests/probe-form-oracle.property.test.js +++ b/packages/eslint-plugin-oracles/tests/probe-form-oracle.property.test.js @@ -107,7 +107,10 @@ const TEST_FILE = join( // test file has it off. Production source is `src` under a package or at the // root, as the production config's globs say. function predictSeverity(filePath) { - return filePath.includes(`/src/`) ? `error` : `off` + // A file is production when one of its directories is `src`. The path's + // separators are either kind on Windows, so the directories are split on both. + const directories = filePath.split(/[\\/]+/).slice(0, -1) + return directories.includes(`src`) ? `error` : `off` } // The rule's behavior is linted through the plugin's production config alone: diff --git a/packages/eslint-plugin-oracles/tests/shipped-citations.test.js b/packages/eslint-plugin-oracles/tests/shipped-citations.test.js index c8c67fd..1a4f424 100644 --- a/packages/eslint-plugin-oracles/tests/shipped-citations.test.js +++ b/packages/eslint-plugin-oracles/tests/shipped-citations.test.js @@ -47,7 +47,8 @@ function shippedFiles(dir) { const packed = spawnSync( `npm`, [`pack`, `--dry-run`, `--json`, `--ignore-scripts`], - { cwd: dir, encoding: `utf8` }, + // On Windows npm is an npm.cmd shim, which only a shell runs. + { cwd: dir, encoding: `utf8`, shell: process.platform === `win32` }, ) return JSON.parse(packed.stdout)[0].files.map((file) => file.path) } diff --git a/packages/oracles/tests/oracle-gaps-owns-through-calls.test.ts b/packages/oracles/tests/oracle-gaps-owns-through-calls.test.ts index d5ea06d..99b41dc 100644 --- a/packages/oracles/tests/oracle-gaps-owns-through-calls.test.ts +++ b/packages/oracles/tests/oracle-gaps-owns-through-calls.test.ts @@ -18,7 +18,10 @@ import { analyze } from '../oracle-gaps/analysis.mjs' // generate a history where the owning entry is in another file. describe(`gap report ownership through the call trees`, () => { test(`a selected source owned only through another file's call tree is not refused`, () => { - const pkg = realpathSync(mkdtempSync(join(tmpdir(), `owns-through-calls-`))) + // The native real path, as the CLI takes it: JavaScript's keeps an 8.3 name. + const pkg = realpathSync.native( + mkdtempSync(join(tmpdir(), `owns-through-calls-`)), + ) mkdirSync(join(pkg, `src`)) mkdirSync(join(pkg, `tests`)) writeFileSync( diff --git a/packages/oracles/tests/package.test.ts b/packages/oracles/tests/package.test.ts index 78a3540..fc65b77 100644 --- a/packages/oracles/tests/package.test.ts +++ b/packages/oracles/tests/package.test.ts @@ -135,7 +135,12 @@ describe(`@tanstack/oracles`, () => { const pack = spawnSync( `npm`, [`pack`, `--dry-run`, `--json`, `--ignore-scripts`], - { cwd: packageDirectory, encoding: `utf8` }, + // On Windows npm is an npm.cmd shim, which only a shell runs. + { + cwd: packageDirectory, + encoding: `utf8`, + shell: process.platform === `win32`, + }, ) expect(pack.status, pack.stderr).toBe(0) const [{ files }] = JSON.parse(pack.stdout) as [ @@ -232,7 +237,13 @@ console.log(JSON.stringify({ failed, unreached }))`, ), ) const run = (command: string, args: Array, cwd: string) => { - const ran = spawnSync(command, args, { cwd, encoding: `utf8`, env }) + const ran = spawnSync(command, args, { + cwd, + encoding: `utf8`, + env, + // npm is an npm.cmd shim on Windows, which only a shell runs. + shell: command === `npm` && process.platform === `win32`, + }) expect(ran.status, `${ran.stdout}\n${ran.stderr}`).toBe(0) return ran } From ef9205d172427272bd291b9a42ea7f9e6101863f Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 10:52:50 -0700 Subject: [PATCH 06/53] ci(windows): sharded Windows jobs in place of the diagnostic ones The diagnostic steps are gone. The Windows jobs mirror the Linux shards: the unit suites in one job, the static gap oracle and the fixed-seed end-to-end oracle in three shards each (pr-shards.mjs), and the gap CLI on a fixture, including the discovered config and the daemon. Each job has a 15-minute backstop. The jobs are still continue-on-error until every one is green; that line is removed then. Co-authored-by: Isaac --- .github/workflows/pr.yml | 145 ++++++++++++++++++--------------------- 1 file changed, 67 insertions(+), 78 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 39e8317..b1eb96a 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -196,15 +196,15 @@ jobs: with: advanced-security: false annotations: true - # Diagnostic, not a gate: nothing needs these jobs, and they are not - # required checks. They show what breaks on Windows, where the oracle - # framework and the gap CLI have never run. Each step continues on error, - # so a later step still runs when an earlier one fails. + # Windows is a required part of CI: the framework, the eslint plugin, and + # the gap CLI run on windows-latest, split the way the Linux jobs split + # them. Each job stays under 10 minutes; each has a timeout backstop. windows-unit: - name: Windows unit suites (non-blocking) + name: Windows unit suites runs-on: windows-latest + # Not blocking until every Windows job is green; remove this line then. continue-on-error: true - timeout-minutes: 45 + timeout-minutes: 15 steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -212,49 +212,68 @@ jobs: persist-credentials: false - name: Setup Tools uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - - name: Build framework - continue-on-error: true - run: pnpm --filter @tanstack/oracles run build - - name: Framework unit tests - continue-on-error: true - run: pnpm --filter @tanstack/oracles run test - - name: eslint plugin tests - continue-on-error: true - run: pnpm --filter @tanstack/eslint-plugin-oracles run test - # [diagnostic] The fast suites run one file per step, each with a step - # timeout, to find which one hangs on Windows (the run was cancelled at - # 21 minutes with no output after the static oracle started). - - name: '[diagnostic] Gap report static oracle' - continue-on-error: true - timeout-minutes: 8 - run: pnpm --filter @tanstack/oracles exec vitest --run --config oracle-gaps/vitest.config.mjs --reporter=verbose gap-report-static - # [diagnostic] Replays the first failing static unit alone, with bash, to - # print its verdict (the run's own output names the law only through a - # counterexample). - - name: '[diagnostic] Replay the first failing static unit' - continue-on-error: true - timeout-minutes: 6 + - run: pnpm --filter @tanstack/oracles run build + - run: pnpm --filter @tanstack/oracles run test + - run: pnpm --filter @tanstack/eslint-plugin-oracles run test + windows-gap-oracles: + name: Windows gap report oracles (${{ matrix.shard }}/3) + runs-on: windows-latest + # Not blocking until every Windows job is green; remove this line then. + continue-on-error: true + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Setup Tools + uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 + - run: pnpm --filter @tanstack/oracles run build + # The same partition of the static oracle that Linux runs, by pattern. + - name: Gap report oracles shell: bash + working-directory: packages/oracles + env: + SHARD: ${{ matrix.shard }} run: | - export ORACLE_SEED=20261105 ORACLE_PATH=0:0:0:0:0:0:0:0:0:0:0:0:0:0:0 ORACLE_PROPERTY='oracle-gaps.report-static' ORACLE_UNIT='random seed: every static state' - pnpm --filter @tanstack/oracles exec vitest run --config oracle-gaps/vitest.config.mjs -t '^oracle-gaps\.report-static(?: | > )random seed: every static state$' gap-report-static - - name: '[diagnostic] Gap report classify oracle' - continue-on-error: true - timeout-minutes: 8 - run: pnpm --filter @tanstack/oracles exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-classify - - name: '[diagnostic] Gap report probe-form oracle' - continue-on-error: true - timeout-minutes: 8 - run: pnpm --filter @tanstack/oracles exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-probe-form - - name: '[diagnostic] Gap report mutant-validity oracle' - continue-on-error: true - timeout-minutes: 10 - run: pnpm --filter @tanstack/oracles exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-mutant-validity + if [ "$SHARD" = 3 ]; then + pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-classify gap-report-probe-form gap-report-mutant-validity + fi + pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-static -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-static "$SHARD")" + windows-gap-end-to-end: + name: Windows gap report end-to-end oracle (fixed seed, ${{ matrix.shard }}/3) + runs-on: windows-latest + # Not blocking until every Windows job is green; remove this line then. + continue-on-error: true + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Setup Tools + uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 + - run: pnpm --filter @tanstack/oracles run build + - name: Gap report end-to-end oracle + shell: bash + working-directory: packages/oracles + env: + SHARD: ${{ matrix.shard }} + run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-fixed-seed "$SHARD")" windows-gap-cli: - name: Windows gap CLI end to end (non-blocking) + name: Windows gap CLI on a fixture runs-on: windows-latest + # Not blocking until every Windows job is green; remove this line then. continue-on-error: true - timeout-minutes: 45 + timeout-minutes: 15 steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -262,16 +281,9 @@ jobs: persist-credentials: false - name: Setup Tools uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - - name: Build framework - continue-on-error: true - run: pnpm --filter @tanstack/oracles run build - # A tiny package written at run time, not committed: one function with - # two operators and one oracle file that owns it. It is written in bash, - # which the runner provides on Windows too; the CLI steps below use the - # default shell, as a user's PowerShell or cmd would. + - run: pnpm --filter @tanstack/oracles run build - name: Write gap CLI fixture shell: bash - continue-on-error: true run: | set -euo pipefail dir="$GITHUB_WORKSPACE/.ci-fixture" @@ -319,37 +331,14 @@ jobs: test: { include: ['tests/**/*.test.ts'], environment: 'node', coverage: { enabled: false } }, } EOT - # The user-facing path: the default shell, no supervisor on Windows. + # The user-facing path: PowerShell, the default shell, as a user runs it. - name: Gap CLI on the fixture - continue-on-error: true run: node packages/oracles/oracle-gaps/index.mjs --package .ci-fixture --source src/clamp.ts --config vitest.config.mjs + - name: Gap CLI on the fixture, discovered config + run: node packages/oracles/oracle-gaps/index.mjs --package .ci-fixture --source src/clamp.ts # The daemon path: start it, check through it, then stop it. - name: Gap CLI through the daemon - continue-on-error: true run: node packages/oracles/oracle-gaps/index.mjs --package .ci-fixture --source src/clamp.ts --config vitest.config.mjs --daemon - name: Stop the gap CLI daemon if: always() - continue-on-error: true run: node packages/oracles/oracle-gaps/index.mjs --stop-daemon --package .ci-fixture - # [diagnostic] The fixture without --config, to see whether the discovered - # config hits the same path bug as --config. - - name: '[diagnostic] Gap CLI on the fixture, discovered config' - continue-on-error: true - run: node packages/oracles/oracle-gaps/index.mjs --package .ci-fixture --source src/clamp.ts - # [diagnostic] The daemon's error logs live in the OS temp directory, - # which the job does not print: print them here. - - name: '[diagnostic] Print gap CLI error logs' - if: always() - continue-on-error: true - shell: pwsh - run: | - Get-ChildItem -Path $env:TEMP -Recurse -Filter error.log -ErrorAction SilentlyContinue | ForEach-Object { - "== $($_.FullName)" - Get-Content $_.FullName -TotalCount 120 - } - # One fixed-seed unit of the end-to-end oracle, as the Linux job runs it. - # [diagnostic] Verbose and a step timeout, to see which copy hangs. - - name: Gap report end-to-end oracle (fixed seed) - continue-on-error: true - timeout-minutes: 12 - run: pnpm --filter @tanstack/oracles exec vitest --run --config oracle-gaps/vitest.config.mjs --reporter=verbose gap-report-oracle -t "fixed seed" From 8d2d037d81ab15412e2a3fc1665366a5f116e909 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 10:58:33 -0700 Subject: [PATCH 07/53] fix(oracle-gaps): give TypeScript and the owner map one spelling of a path The owners and the instrumenter named files with the platform's separators, and TypeScript matches a file's name as given, so on Windows the owner map keyed `src\x.js` while the sources were `src/x.js`, and CommonJS sites lost their functions. The compiler now gets forward-slash names, and owner keys are forward-slash (schemata.mjs, owners.mjs, analysis.mjs). A path's drive letter is capitalised in the one spelling (paths.mjs), so a drive's case never decides a path. New oracle: gap-report-path-spelling (a metamorphic check). Law 1: the instrumenter gives the same mutant ids for a source under any spelling of its path. Law 2: relativeWithin answers from the segments alone. Evidence block - Law: a path's separators and a drive letter's case do not change what the gap CLI decides (paths.mjs, Windows support, 2026-11-13). - Reach: 200 generated histories (fixed seed 20261113 and random), over four module forms, drives in both cases, mixed separators, short and spaced names. Reach is not declared (states: []); the histories are not measured as reaching a state. - RED: on the parent commit (72b9495), the Windows jobs of https://github.com/TanStack/oracles/pull/58 failed: the config wrapper import (check.mjs), the copy map, the owner keys (owners.mjs), and the 8.3 short names. On Linux the enumeration law does NOT reproduce the Windows failure: the instrumenter gave the same ids on the parent. The Windows failure is a platform path.relative difference, which Linux cannot produce. The RED for those classes is the Windows CI run. - GREEN: the Windows jobs (see the PR). Local, on macOS: the new oracle 5 of 5, the framework 114 of 114, the eslint plugin 7335 of 7335. - Mutant: relativeWithin as it was before (raw string comparison) fails the relation checkpoint; dropping the drive capitalisation fails it too. Both run on the parent tree with the oracle and both fail (exit 1). - Limits: no declared reach; Law 1 uses the forward-slash spelling as its reference, so it claims only that the spelling changes nothing; owners, realpath, and OS short names are judged by the Windows end-to-end run, not by this oracle. Co-authored-by: Isaac --- .github/workflows/pr.yml | 5 +- packages/oracles/oracle-gaps/analysis.mjs | 3 +- ...ort-path-spelling-oracle.property.test.mjs | 161 ++++++++++++++++++ packages/oracles/oracle-gaps/owners.mjs | 14 +- packages/oracles/oracle-gaps/paths.mjs | 8 +- packages/oracles/oracle-gaps/schemata.mjs | 11 +- packages/oracles/package.json | 2 +- 7 files changed, 193 insertions(+), 11 deletions(-) create mode 100644 packages/oracles/oracle-gaps/gap-report-path-spelling-oracle.property.test.mjs diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index b1eb96a..b81ceae 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -99,7 +99,7 @@ jobs: # and the classify, probe-form, and mutant-validity oracles in a third. - run: | if [ "$SHARD" = 3 ]; then - exec pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-classify gap-report-probe-form gap-report-mutant-validity + exec pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-classify gap-report-probe-form gap-report-mutant-validity gap-report-path-spelling fi pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-static -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-static "$SHARD")" working-directory: packages/oracles @@ -241,7 +241,7 @@ jobs: SHARD: ${{ matrix.shard }} run: | if [ "$SHARD" = 3 ]; then - pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-classify gap-report-probe-form gap-report-mutant-validity + pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-classify gap-report-probe-form gap-report-mutant-validity gap-report-path-spelling fi pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-static -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-static "$SHARD")" windows-gap-end-to-end: @@ -302,6 +302,7 @@ jobs: import { clamp } from '../src/clamp' oracleProperty({ + owns: ['src/clamp.ts#clamp'], id: `fixture.clamp`, law: { statement: `clamp returns its argument within 0 and 10.`, diff --git a/packages/oracles/oracle-gaps/analysis.mjs b/packages/oracles/oracle-gaps/analysis.mjs index c74a097..b884cee 100644 --- a/packages/oracles/oracle-gaps/analysis.mjs +++ b/packages/oracles/oracle-gaps/analysis.mjs @@ -7,6 +7,7 @@ import { existsSync, readFileSync, realpathSync } from 'node:fs' import { basename, join } from 'node:path' import { packagePath } from './args.mjs' +import { posix } from './paths.mjs' import { Refusal } from './ending.mjs' import { ownership, readOwns, splitEntry } from './owners.mjs' import { instrument } from './schemata.mjs' @@ -124,7 +125,7 @@ export function analyze(ts, pkg, sources, testFiles) { .flat() .map((entry) => splitEntry(entry)[0]), ]), - ].map((source) => join(pkg, source)), + ].map((source) => posix(join(pkg, source))), compilerOptions(ts), hostFor(ts), ) diff --git a/packages/oracles/oracle-gaps/gap-report-path-spelling-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-path-spelling-oracle.property.test.mjs new file mode 100644 index 0000000..89d1a8d --- /dev/null +++ b/packages/oracles/oracle-gaps/gap-report-path-spelling-oracle.property.test.mjs @@ -0,0 +1,161 @@ +/** + * # A path's separators do not change what the gap CLI decides + * + * The law (the path rule of paths.mjs, Windows support, 2026-11-13): a file + * is one file however its separators are spelled. A path enters the CLI with + * either separator, and the CLI compares, keys, and enumerates it in one + * spelling, the forward-slash one. TypeScript compares a file's name as it was + * given, so a name with backslashes is a different file to it: the scope and + * binding questions of the mutant instrumenter then answer differently, and + * CommonJS sites (`exports`, `module`) lose their mutants. Within a package, a + * path lies inside another exactly when its segments start with the other's, + * whichever separator spells them, and a drive letter's case does not decide + * it. + * + * Law 1 (enumeration): for every source text and every spelling of its path, + * the instrumenter yields the same mutant ids as for the forward-slash + * spelling. Law 2 (relation): `relativeWithin(dir, file)` is the forward-slash + * path of `file` after `dir`'s segments when `dir`'s segments start `file`'s + * (compared with a drive letter in any case), and undefined otherwise. + * + * History: a source text from a short fixed list (the module forms the + * instrumenter treats apart: an ES module, a CommonJS `module.exports` + * assignment, `exports.x =`, and a `require`), a drive, one to three + * directory names, one to two file names (names that a Windows + * spelling exercises: a short name such as `RUNNER~1`, a name with a space), + * and a separator for each joint, drawn independently. + * + * Model: Law 2 is answered from the segments alone: split both paths on either + * separator, compare the directory's segments with the file's (the drive + * case-insensitively), and the file's rest is the answer. Law 1 has no model + * of its own: the forward-slash spelling is the reference, and the check + * compares each spelling with it. That reference is production's answer on + * the one spelling the rule names as the reference, so the law claims only + * that spelling changes nothing; it does not claim the forward-slash answer is + * right, which the static oracle judges. + * + * Driver: runs the instrumenter on the forward-slash spelling and on the + * history's spelling, and runs `relativeWithin` on the history's directory and + * file. + * + * Limits: the instrumenter runs on the source's text only (no package, so no + * owners or imports), and the file names are not on disk, so a case that + * needs a real file (realpath, a short name that the OS expands) belongs to the + * end-to-end oracle, which runs on Windows in CI. + */ +import fc from 'fast-check' +import { createRequire } from 'node:module' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { differs, oracleProperty, pass } from '../src/oracle-property.ts' +import { relativeWithin } from './paths.mjs' +import { instrument } from './schemata.mjs' + +const here = dirname(fileURLToPath(import.meta.url)) +const ts = createRequire(join(here, `../package.json`))( + `typescript-for-tanstack-oracles`, +) + +// The module forms the instrumenter treats apart. Each parses on its own. +const TEXTS = [ + `export function run(x: number): number {\n return x < 0 ? 0 : x\n}\n`, + `module.exports = exports = function (x, out) {\n if (x) out.push(x)\n return x ? 1 : 2\n}\n`, + `exports.default = (x, out) => {\n if (x) out.push(x)\n}\n`, + `const other = require('./b')\nexports.run = () => other.help() || 2\n`, +] + +const NAMES = [`src`, `RUNNER~1`, `my dir`, `pkg`, `lib`, `a.js`, `b.ts`] + +const historyArbitrary = fc.record({ + text: fc.integer({ min: 0, max: TEXTS.length - 1 }), + // The directory's drive letter may differ in case from the file's. + drive: fc.constantFrom(`C`, `c`, `D`), + dirDrive: fc.constantFrom(`C`, `c`), + root: fc.array(fc.constantFrom(...NAMES), { minLength: 1, maxLength: 3 }), + rest: fc.array(fc.constantFrom(...NAMES), { minLength: 1, maxLength: 2 }), + // One separator for each joint: the drive to the first name, and each name. + joints: fc.array(fc.constantFrom(`/`, `\\`), { minLength: 8, maxLength: 8 }), +}) + +/** The segments of a history's spelling: the drive, then the names. */ +const segmentsOf = ({ drive, dirDrive, root, rest }) => ({ + directory: [`${dirDrive}:`, ...root], + file: [`${drive}:`, ...root, ...rest], +}) + +/** The path with each joint's separator, as a Windows or POSIX spelling makes it. */ +function spell(segments, joints) { + let path = segments[0] + for (let i = 1; i < segments.length; i += 1) + path += joints[i - 1] + segments[i] + return path +} + +/** The forward-slash spelling: the reference the enumeration is compared with. */ +const posixOf = (segments) => segments.join(`/`) + +/** The mutant ids the instrumenter gives a text at `path`, sorted. */ +function idsAt(path, text) { + const { mutants } = instrument(ts, path, text, `src/a.js`) + return mutants.map((m) => m.id).sort() +} + +// Law 2's answer, from the segments alone: the directory's segments start the +// file's (the drive compared case-insensitively), and the rest is the answer. +function relationModel({ root, rest, drive, dirDrive }) { + const dir = [dirDrive.toUpperCase(), ...root] + const file = [drive.toUpperCase(), ...root, ...rest] + const starts = dir.every((segment, i) => file[i] === segment) + return starts ? rest.join(`/`) : undefined +} + +const model = (history) => ({ + relation: relationModel(history), +}) + +const check = (history, trace, expected) => { + const find = (checkpoint) => + trace.find((row) => row.checkpoint === checkpoint)?.value + const reference = find(`reference`) + const spelled = find(`spelled`) + return ( + differs(`enumeration`, `the forward-slash spelling`, spelled, reference) ?? + differs(`relation`, `segments`, find(`relation`), expected.relation) ?? + pass + ) +} + +oracleProperty({ + id: `oracle-gaps.report-path-spelling`, + law: { + statement: `A path's separators change nothing the gap CLI decides: the mutants of a source are the same under any spelling of its path, and a path lies inside a directory exactly when its segments start with the directory's, a drive letter's case aside.`, + authority: `The path rule of packages/oracles/oracle-gaps/paths.mjs (Windows support, 2026-11-13): paths are forward-slash at entry, and TypeScript and Vite compare them in that one spelling.`, + limits: `A source that runs through owners and imports, a real file (realpath, an OS short name), and the enumeration of the whole package are not generated here: the static oracle and the Windows end-to-end run judge them.`, + }, + histories: { + arbitrary: historyArbitrary, + runs: 200, + fixedSeeds: [20261113], + states: [], + measured: [], + }, + timeout: 10_000, + model, + driver: (history, observe) => { + const { directory, file } = segmentsOf(history) + const text = TEXTS[history.text] + observe(`reference`, idsAt(posixOf(file), text)) + observe( + `spelled`, + idsAt(spell(file, history.joints.slice(0, file.length - 1)), text), + ) + observe( + `relation`, + relativeWithin( + spell(directory, history.joints.slice(0, directory.length - 1)), + spell(file, history.joints.slice(0, file.length - 1)), + ), + ) + }, + check, +}) diff --git a/packages/oracles/oracle-gaps/owners.mjs b/packages/oracles/oracle-gaps/owners.mjs index d41a42c..f268c61 100644 --- a/packages/oracles/oracle-gaps/owners.mjs +++ b/packages/oracles/oracle-gaps/owners.mjs @@ -66,6 +66,7 @@ // reported as `unowned`, a warning rather than a gap. import { relative, resolve } from 'node:path' import { isOutside } from './args.mjs' +import { posix } from './paths.mjs' /** * An owns entry's file and function. The file ends at the first `#`, so a @@ -148,7 +149,12 @@ function isAssertion(ts, call, names) { * CLI cannot read would hide gaps. */ export function readOwns(ts, path, text) { - const file = ts.createSourceFile(path, text, ts.ScriptTarget.Latest, true) + const file = ts.createSourceFile( + posix(path), + text, + ts.ScriptTarget.Latest, + true, + ) const result = { declares: false, entries: [], problem: undefined } const assertions = assertionBindings(ts, file) // `compared` is true inside an assertion's arguments. @@ -300,7 +306,7 @@ export function ownership(ts, program, pkg, oracles) { const indexed = new Map() const functionsOf = (source) => { if (!indexed.has(source)) { - const file = program.getSourceFile(resolve(pkg, source)) + const file = program.getSourceFile(posix(resolve(pkg, source))) indexed.set(source, file ? functionsIn(ts, file) : []) } return indexed.get(source) @@ -311,7 +317,7 @@ export function ownership(ts, program, pkg, oracles) { const targetsOf = (declaration, constructed) => { const file = declaration.getSourceFile() if (!inPackage(file.fileName)) return [] - const functions = functionsOf(relative(pkg, file.fileName)) + const functions = functionsOf(posix(relative(pkg, file.fileName))) if (ts.isClassDeclaration(declaration) && declaration.name) return functions.filter( (fn) => @@ -380,7 +386,7 @@ export function ownership(ts, program, pkg, oracles) { // Module data owns only its own sites, not the functions it holds. const functions = ts.isVariableDeclaration(fn.node) ? [] - : functionsOf(relative(pkg, fn.node.getSourceFile().fileName)) + : functionsOf(posix(relative(pkg, fn.node.getSourceFile().fileName))) for ( let at = functions.indexOf(fn) + 1; at < functions.length && functions[at].start < fn.end; diff --git a/packages/oracles/oracle-gaps/paths.mjs b/packages/oracles/oracle-gaps/paths.mjs index 7de4db3..27711dd 100644 --- a/packages/oracles/oracle-gaps/paths.mjs +++ b/packages/oracles/oracle-gaps/paths.mjs @@ -9,8 +9,12 @@ import { realpathSync } from 'node:fs' import { relative } from 'node:path' -/** The forward-slash spelling of `path`. */ -export const posix = (path) => path.replaceAll(`\\`, `/`) +/** + * The forward-slash spelling of `path`, with a drive letter in capitals: a + * Windows drive's letter is one letter whatever its case. + */ +export const posix = (path) => + path.replaceAll(`\\`, `/`).replace(/^[a-z]:/, (drive) => drive.toUpperCase()) /** The real path of `path`, in forward slashes. */ export const realPathOf = (path) => posix(realpathSync.native(path)) diff --git a/packages/oracles/oracle-gaps/schemata.mjs b/packages/oracles/oracle-gaps/schemata.mjs index 525d53e..2366c8c 100644 --- a/packages/oracles/oracle-gaps/schemata.mjs +++ b/packages/oracles/oracle-gaps/schemata.mjs @@ -110,11 +110,20 @@ function anchorOf(ts, file, node) { // because one switch serves them all. A mutant names no function: the runner // names it by the function declaration whose lines hold it (owners.mjs), so // keys and ownership read one source. +import { posix } from './paths.mjs' + const placedRune = /^\$(state(\.raw)?|derived(\.by)?|effect(\.pre)?|props(\.id)?|bindable|inspect(\.trace)?)$/ export function instrument(ts, path, text, source, offset = 0) { - const file = ts.createSourceFile(path, text, ts.ScriptTarget.Latest, true) + // TypeScript keeps a file's name as given and matches it with forward + // slashes, so the file is named in them (paths.mjs). + const file = ts.createSourceFile( + posix(path), + text, + ts.ScriptTarget.Latest, + true, + ) // Without the word `super` no site can be in a super region: skip the walk. const mentionsSuper = /\bsuper\b/.test(text) const svelte = /\.svelte\.[cm]?[jt]s$/.test(path) diff --git a/packages/oracles/package.json b/packages/oracles/package.json index d6fa476..834822d 100644 --- a/packages/oracles/package.json +++ b/packages/oracles/package.json @@ -63,7 +63,7 @@ "test": "vitest --run", "test:oracle-framework": "vitest --run --config vitest.oracle-framework.config.mjs", "test:oracle-gaps": "vitest --run --config oracle-gaps/vitest.config.mjs", - "test:oracle-gaps:fast": "vitest --run --config oracle-gaps/vitest.config.mjs gap-report-static gap-report-classify gap-report-probe-form gap-report-mutant-validity", + "test:oracle-gaps:fast": "vitest --run --config oracle-gaps/vitest.config.mjs gap-report-static gap-report-classify gap-report-probe-form gap-report-mutant-validity gap-report-path-spelling", "test:oracle-gaps:shards": "node scripts/e2e-shards.mjs check", "test:pr-shards": "node scripts/pr-shards.mjs check", "typecheck": "tsc --noEmit" From 29c695994fbe21f955b7fd6902c75ed79cafb332 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 11:09:12 -0700 Subject: [PATCH 08/53] ci(windows): P10 done-line, refusal, early-stop, and per-shell replay witnesses - P10: the Windows CLI runs assert the done line on both streams, and that it matches the exit code (scripts/windows-witness.mjs done). - A package with no package.json is refused with exit 2 and a done line that names it (Windows witness). - One early stop: SIGINT once the CLI prints its first line; the run must end with the stop's done line on both streams, or with no output (#57). - Per-shell replay (framework, accepted by the coordinator): a failing property's PowerShell and cmd lines each reproduce the failure in their own shell. The POSIX line is run in PowerShell and must not reproduce (the RED). A run that printed only the POSIX line fails here (the mutant). - The tsc link lookup in package.test.ts reads npm's Windows shim, not a symlink. - The eslint plugin suite runs even when the framework suite fails. - Job timeouts are 20 minutes, as high backstops (2026-11-12). Co-authored-by: Isaac --- .github/workflows/pr.yml | 91 ++++++++- packages/oracles/scripts/windows-witness.mjs | 195 +++++++++++++++++++ packages/oracles/tests/package.test.ts | 12 +- 3 files changed, 287 insertions(+), 11 deletions(-) create mode 100644 packages/oracles/scripts/windows-witness.mjs diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index b81ceae..afc357d 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -204,7 +204,7 @@ jobs: runs-on: windows-latest # Not blocking until every Windows job is green; remove this line then. continue-on-error: true - timeout-minutes: 15 + timeout-minutes: 20 steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -214,13 +214,15 @@ jobs: uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - run: pnpm --filter @tanstack/oracles run build - run: pnpm --filter @tanstack/oracles run test - - run: pnpm --filter @tanstack/eslint-plugin-oracles run test + # Runs even when the framework tests fail, so each suite reports. + - if: always() + run: pnpm --filter @tanstack/eslint-plugin-oracles run test windows-gap-oracles: name: Windows gap report oracles (${{ matrix.shard }}/3) runs-on: windows-latest # Not blocking until every Windows job is green; remove this line then. continue-on-error: true - timeout-minutes: 15 + timeout-minutes: 20 strategy: fail-fast: false matrix: @@ -249,7 +251,7 @@ jobs: runs-on: windows-latest # Not blocking until every Windows job is green; remove this line then. continue-on-error: true - timeout-minutes: 15 + timeout-minutes: 20 strategy: fail-fast: false matrix: @@ -273,7 +275,7 @@ jobs: runs-on: windows-latest # Not blocking until every Windows job is green; remove this line then. continue-on-error: true - timeout-minutes: 15 + timeout-minutes: 20 steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -332,14 +334,87 @@ jobs: test: { include: ['tests/**/*.test.ts'], environment: 'node', coverage: { enabled: false } }, } EOT + # A failing property for the replay witness: every history fails, so a + # random unit prints its replay lines. Written in bash, as the fixture is. + - name: Write replay fixture + shell: bash + run: | + set -euo pipefail + dir="$GITHUB_WORKSPACE/.ci-replay" + mkdir -p "$dir/tests" + cat > "$dir/tests/fail.property.test.ts" <<'EOT' + import fc from 'fast-check' + import { differs, oracleProperty, pass } from '../../packages/oracles/dist/esm/index.js' + + oracleProperty({ + id: `fixture.replay`, + law: { + statement: `a fixture that always fails, so its replay lines are printed.`, + authority: `the CI witness for the per-shell replay lines`, + limits: `integers from -20 to 20`, + }, + histories: { + arbitrary: fc.integer({ min: -20, max: 20 }), + runs: 20, + states: [], + measured: [], + }, + timeout: 5_000, + model: (x: number) => x, + driver: (x: number, observe: (checkpoint: string, value: unknown) => void) => { + observe(`result`, x + 1) + }, + check: (_x: number, trace: ReadonlyArray<{ checkpoint: string; value: unknown }>, expected: number) => + differs(`result`, `plus one`, trace.find(({ checkpoint }) => checkpoint === `result`)?.value, expected) ?? pass, + }) + EOT + printf '%s\n' "export default { test: { include: ['tests/**/*.test.ts'], environment: 'node', coverage: { enabled: false } } }" > "$dir/vitest.config.mjs" # The user-facing path: PowerShell, the default shell, as a user runs it. + # Each run's stdout and stderr are kept apart, and the done line is + # checked on both streams against the exit code (P10). - name: Gap CLI on the fixture - run: node packages/oracles/oracle-gaps/index.mjs --package .ci-fixture --source src/clamp.ts --config vitest.config.mjs + run: | + node packages/oracles/oracle-gaps/index.mjs --package .ci-fixture --source src/clamp.ts --config vitest.config.mjs > cli-out.txt 2> cli-err.txt + $code = $LASTEXITCODE + node packages/oracles/scripts/windows-witness.mjs done cli-out.txt cli-err.txt $code + if ($LASTEXITCODE -ne 0) { throw "the done line failed" } - name: Gap CLI on the fixture, discovered config - run: node packages/oracles/oracle-gaps/index.mjs --package .ci-fixture --source src/clamp.ts + run: | + node packages/oracles/oracle-gaps/index.mjs --package .ci-fixture --source src/clamp.ts > cli-out2.txt 2> cli-err2.txt + $code = $LASTEXITCODE + node packages/oracles/scripts/windows-witness.mjs done cli-out2.txt cli-err2.txt $code + if ($LASTEXITCODE -ne 0) { throw "the done line failed" } # The daemon path: start it, check through it, then stop it. - name: Gap CLI through the daemon - run: node packages/oracles/oracle-gaps/index.mjs --package .ci-fixture --source src/clamp.ts --config vitest.config.mjs --daemon + run: | + node packages/oracles/oracle-gaps/index.mjs --package .ci-fixture --source src/clamp.ts --config vitest.config.mjs --daemon > cli-out3.txt 2> cli-err3.txt + $code = $LASTEXITCODE + node packages/oracles/scripts/windows-witness.mjs done cli-out3.txt cli-err3.txt $code + if ($LASTEXITCODE -ne 0) { throw "the done line failed" } - name: Stop the gap CLI daemon if: always() run: node packages/oracles/oracle-gaps/index.mjs --stop-daemon --package .ci-fixture + # A package with no package.json is refused, with exit 2 and a done line + # that says so. + - name: Gap CLI refuses a package with no package.json + run: | + New-Item -ItemType Directory -Force .ci-no-manifest | Out-Null + node packages/oracles/oracle-gaps/index.mjs --package .ci-no-manifest --source src/x.ts > nm-out.txt 2> nm-err.txt + $code = $LASTEXITCODE + if ($code -ne 2) { throw "want a refusal with exit 2, got $code" } + if (-not (Select-String -Path nm-out.txt -Pattern 'has no package.json' -Quiet)) { throw "the refusal does not name package.json" } + node packages/oracles/scripts/windows-witness.mjs done nm-out.txt nm-err.txt $code + if ($LASTEXITCODE -ne 0) { throw "the done line failed" } + # One early stop: SIGINT once the CLI has printed its first line. The + # run must end with the stop's done line on both streams, or with no + # output at all (the all-or-nothing law, #57). + - name: Early stop of the gap CLI + run: node packages/oracles/scripts/windows-witness.mjs stop .ci-fixture --package . --source src/clamp.ts --config vitest.config.mjs + # The per-shell replay lines: a failing property's PowerShell and cmd + # lines each reproduce its failure in their own shell (RED: the POSIX + # line does not, in PowerShell). + - name: Replay lines reproduce the failure in PowerShell and cmd + run: | + node "$env:GITHUB_WORKSPACE\packages\oracles\node_modules\vitest\vitest.mjs" run --root .ci-replay > replay-out.txt 2>&1 + node packages/oracles/scripts/windows-witness.mjs replay replay-out.txt "$env:GITHUB_WORKSPACE\.ci-replay" + if ($LASTEXITCODE -ne 0) { throw "the replay witness failed" } diff --git a/packages/oracles/scripts/windows-witness.mjs b/packages/oracles/scripts/windows-witness.mjs new file mode 100644 index 0000000..16ced2d --- /dev/null +++ b/packages/oracles/scripts/windows-witness.mjs @@ -0,0 +1,195 @@ +// Windows witnesses for the gap CLI and the framework's replay lines. Each +// subcommand is one check that CI runs on windows-latest, and each exits +// non-zero when the check fails. Node runs them on every platform. +// +// node scripts/windows-witness.mjs done +// The done line (P10): each stream has exactly one `oracle-gaps: done` +// line, and it is the stream's last line; its status matches the exit +// code (completed 0, refused 2, error 3, a stop 130 or 143). +// +// node scripts/windows-witness.mjs replay +// The replay lines of a failing property: the PowerShell line and the +// cmd line each reproduce the failure when run in their own shell. The +// POSIX line is run in PowerShell too, and must not reproduce (that is the +// RED of the framework's per-shell lines). A run whose output has only +// the POSIX line on win32 fails here, because the per-shell lines are +// missing. +// +// node scripts/windows-witness.mjs stop +// One early stop: the CLI starts, the witness sends SIGINT once the CLI +// has printed its first line, and the run must end in one of the two +// outcomes of the all-or-nothing law (#57): (a) the stop's done line on +// both streams, matching the exit code (130 or 143); (b) no output at all. +// Anything else, including a partial output, fails. +import { spawn, spawnSync } from 'node:child_process' +import { readFileSync } from 'node:fs' +import { delimiter, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +const here = fileURLToPath(new URL(`.`, import.meta.url)) +const cli = resolve(here, `../oracle-gaps/index.mjs`) +const [command, ...args] = process.argv.slice(2) + +const fail = (message) => { + console.error(`windows-witness ${command}: ${message}`) + process.exitCode = 1 +} + +// The done status each exit code names (P10). +const statusOf = { completed: 0, refused: 2, error: 3 } +const exitCodeOf = (status) => { + if (status in statusOf) return statusOf[status] + return undefined +} + +function doneLinesOf(text) { + return text + .split(/\r?\n/) + .filter((line) => line.startsWith(`oracle-gaps: done `)) +} + +function checkDone(stream, text, exitCode) { + const lines = text.split(/\r?\n/).filter((line) => line !== ``) + const done = doneLinesOf(text) + if (done.length !== 1) + return `${stream}: ${done.length} done lines, want exactly 1` + if (!lines.at(-1).startsWith(`oracle-gaps: done `)) + return `${stream}: the done line is not the last line` + const status = done[0].slice(`oracle-gaps: done `.length).split(/[ ,]/)[0] + // A stop names no exit status of its own: it ends with 130 or 143. + const want = exitCodeOf(status) + const ok = + want === undefined ? [130, 143].includes(exitCode) : want === exitCode + return ok ? undefined : `${stream}: done ${status} but exit code ${exitCode}` +} + +if (command === `done`) { + const [stdoutFile, stderrFile, code] = args + const exitCode = Number(code) + const problems = [ + checkDone(`stdout`, readFileSync(stdoutFile, `utf8`), exitCode), + checkDone(`stderr`, readFileSync(stderrFile, `utf8`), exitCode), + ].filter(Boolean) + if (problems.length > 0) fail(problems.join('; ')) + else console.log(`done line ok on both streams, exit ${exitCode}`) +} else if (command === `replay`) { + const [outputFile, dir] = args + const output = readFileSync(outputFile, `utf8`).replace(/\x1b\[[0-9;]*m/g, ``) + // The text after a label on the first line that carries it, wherever the + // error text indents it. + const lineOf = (label) => { + const line = output.split(/\r?\n/).find((text) => text.includes(label)) + return line === undefined + ? undefined + : line.slice(line.indexOf(label) + label.length) + } + const powershell = lineOf(`Replay (PowerShell): `) + const cmd = lineOf(`Replay (cmd): `) + const posix = lineOf(`Replay: `) + // The shells find the test runner in this package's own binaries. + const env = { + ...process.env, + PATH: [ + resolve(here, `../node_modules/.bin`), + join(dir, `node_modules`, `.bin`), + process.env.PATH, + ].join(delimiter), + } + const runIn = (shell, line) => + spawnSync( + shell === `pwsh` ? `pwsh` : `cmd`, + shell === `pwsh` + ? [`-NoProfile`, `-Command`, line] + : [`/d`, `/s`, `/c`, line], + // cmd reads its line as it is; Node's own quoting would break it. + { + cwd: dir, + env, + encoding: `utf8`, + windowsVerbatimArguments: shell !== `pwsh`, + }, + ) + const reproduces = (result) => + result.status !== 0 && + /violated|failed/.test(`${result.stdout}${result.stderr}`) + if (powershell === undefined || cmd === undefined) { + fail( + `the failure printed no per-shell replay lines (PowerShell: ${powershell === undefined ? `missing` : `ok`}, cmd: ${cmd === undefined ? `missing` : `ok`})`, + ) + } else { + const psRun = runIn(`pwsh`, powershell) + const cmdRun = runIn(`cmd`, cmd) + if (!reproduces(psRun)) + fail( + `the PowerShell line did not reproduce the failure: ${psRun.stderr || psRun.stdout}`.slice( + 0, + 800, + ), + ) + else if (!reproduces(cmdRun)) + fail( + `the cmd line did not reproduce the failure: ${cmdRun.stderr || cmdRun.stdout}`.slice( + 0, + 800, + ), + ) + else console.log(`PowerShell and cmd lines each reproduce the failure`) + } + if (posix !== undefined) { + const posixRun = runIn(`pwsh`, posix) + if (reproduces(posixRun)) + console.log(`note: the POSIX line also reproduces in PowerShell`) + else + console.log( + `RED: the POSIX line does not reproduce in PowerShell (exit ${posixRun.status})`, + ) + } +} else if (command === `stop`) { + const [dir, ...cliArgs] = args + const child = spawn(process.execPath, [cli, ...cliArgs], { + cwd: dir, + stdio: [`ignore`, `pipe`, `pipe`], + }) + let stdout = `` + let stderr = `` + let sent = false + child.stdout.on(`data`, (chunk) => { + stdout += chunk + if (!sent && stdout.length > 0) { + sent = true + child.kill(`SIGINT`) + } + }) + child.stderr.on(`data`, (chunk) => { + stderr += chunk + }) + child.on(`close`, (code, signal) => { + if (!sent) { + // The run ended before it printed anything: outcome (b) if it printed + // nothing at all, which is the only way to end before a first line. + if (stdout === `` && stderr === ``) + console.log(`stop outcome (b): no output`) + else + fail( + `the run ended before any stop was sent, with ${JSON.stringify(stdout.slice(0, 200))}`, + ) + return + } + if (stdout === `` && stderr === ``) { + console.log(`stop outcome (b): no output`) + return + } + const problems = [ + checkDone(`stdout`, stdout, code ?? -1), + checkDone(`stderr`, stderr, code ?? -1), + ].filter(Boolean) + if (problems.length > 0) + fail( + `stop was not all-or-nothing (signal ${signal}, exit ${code}): ${problems.join('; ')}`, + ) + else + console.log(`stop outcome (a): done line on both streams, exit ${code}`) + }) +} else { + fail(`unknown command ${command}`) +} diff --git a/packages/oracles/tests/package.test.ts b/packages/oracles/tests/package.test.ts index fc65b77..13b4751 100644 --- a/packages/oracles/tests/package.test.ts +++ b/packages/oracles/tests/package.test.ts @@ -292,9 +292,15 @@ console.log(JSON.stringify({ failed, unreached }))`, npm(directory, `install`, ...specs) return directory } - // Where the project's `tsc` command links. - const tscOf = (directory: string) => - readlinkSync(join(directory, `node_modules/.bin/tsc`)) + // Where the project's `tsc` command links. npm makes a symlink on POSIX + // and, on Windows, a shell shim that names the same target. + const tscOf = (directory: string) => { + const bin = join(directory, `node_modules/.bin/tsc`) + if (process.platform !== `win32`) return readlinkSync(bin) + const target = /\.\.\/(.+?)\/bin\/tsc/.exec(readFileSync(bin, `utf8`)) + if (target === null) throw new Error(`no tsc target in ${bin}`) + return join(`..`, target[1], `bin`, `tsc`) + } const aliasTsc = join( `..`, `typescript-for-tanstack-oracles`, From 00456e7c6824aac188e830923bf13f5832edfa45 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 11:29:06 -0700 Subject: [PATCH 09/53] fix(oracle-gaps): oracles read the platform's replay lines, and scratch packages have a manifest Windows CI failed the classify oracle's baseline law: the CLI prints a PowerShell and a cmd replay line there, and the law read only the POSIX `replay:` line (Difference: "the replay of tests/f0.test.ts is undefined"). The law is unchanged: each replay line the platform prints must read as `vitest run -t ` in its own shell, and the owed test is the one it selects. On win32 the law owes both per-shell lines; elsewhere the POSIX line (replayLinesOf, powerShellWords, cmdWords). The static oracle's scratch packages had no package.json, which the CLI now refuses (args.mjs). Each fixture now writes a minimal manifest; its content and its histories are unchanged. Evidence block - Law: the replay line of each shell reads as the owed command (Law 9, baseline; the CLI's replay lines, stream.mjs). - Reach: the existing classify histories (276 tests); no new histories. - RED: Windows run 43478073715, classify shard 3 (baseline law, 11 violations) and the static shard 1 refusal (the missing manifest). - GREEN: local POSIX classify 276 of 276. The win32 branch runs in the Windows job (see the PR). - Mutant: not run; the win32 branch is exercised only by the Windows job. - Limits: the win32 branch is checked on Windows only, not by the oracle's own generated histories. Co-authored-by: Isaac --- ...p-report-classify-oracle.property.test.mjs | 163 ++++++++++++++---- ...gap-report-static-oracle.property.test.mjs | 6 + 2 files changed, 136 insertions(+), 33 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs index 062d5bd..d312fba 100644 --- a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs @@ -2412,7 +2412,7 @@ function check(h, observations, expected) { withoutSpans( stdout .split(`\n`) - .filter((line) => !line.startsWith(` replay: `)) + .filter((line) => !/^ replay( \(\w+\))?: /.test(line)) .join(`\n`), ).replace(/ L\? /g, ` L `), ) @@ -2703,50 +2703,147 @@ function baselineBreach(h, observations, expected) { : `${file} fails by an error, and its baseline names a law: ${JSON.stringify(block)}`, ) } - const command = block - .find((line) => line.startsWith(` replay: `)) - ?.slice(` replay: `.length) - const words = command === undefined ? undefined : shellWords(command) - if (!words) + // The platform's replay lines: one POSIX line, or on Windows one line for + // PowerShell and one for cmd. Each reads as the same words. + const replays = replayLinesOf(block) + if (replays === undefined) return breach( - `the replay of ${file} is ${JSON.stringify(command)}, which a POSIX shell does not read as plain words`, + `${file}'s replay lines are not the platform's: ${JSON.stringify(block.filter((line) => line.startsWith(` replay`)))}`, ) - const [vitest, run, target, flag, pattern, ...more] = words - const path = relativeName(h, owed.fi) - if (vitest !== `vitest` || run !== `run` || target !== path) - return breach(`the replay of ${file} is not \`vitest run ${path}\``) - if (owed.test === undefined) { - if (flag !== undefined) + for (const { command, read } of replays) { + const words = read(command) + if (!words) return breach( - `${file} did not load, and its replay selects tests: ${command}`, - ) - continue - } - if (flag !== `-t` || pattern === undefined || more.length > 0) - return breach( - `the replay of ${file} has no single -t pattern: ${command}`, - ) - for (const join of [` `, ` > `]) { - let selected - try { - selected = selectedBy(h, owed.fi, pattern, join) - } catch (error) { - return breach( - `the replay pattern ${pattern} is not a regular expression: ${error.message}`, + `the replay of ${file} is ${JSON.stringify(command)}, which the shell does not read as plain words`, ) + const [vitest, run, target, flag, pattern, ...more] = words + const path = relativeName(h, owed.fi) + if (vitest !== `vitest` || run !== `run` || target !== path) + return breach(`the replay of ${file} is not \`vitest run ${path}\``) + if (owed.test === undefined) { + if (flag !== undefined) + return breach( + `${file} did not load, and its replay selects tests: ${command}`, + ) + continue } - // The owed test, and any test whose joined names equal its own, which - // no pattern can tell apart from it. - const owedSet = sameNamed(h, owed.fi, owed.test, join) - if (JSON.stringify(selected) !== JSON.stringify(owedSet)) + if (flag !== `-t` || pattern === undefined || more.length > 0) return breach( - `the replay pattern ${JSON.stringify(pattern)} of ${file} runs tests ${JSON.stringify(selected)} when Vitest joins names with ${JSON.stringify(join)}, owed ${JSON.stringify(owedSet)}`, + `the replay of ${file} has no single -t pattern: ${command}`, ) + for (const join of [` `, ` > `]) { + let selected + try { + selected = selectedBy(h, owed.fi, pattern, join) + } catch (error) { + return breach( + `the replay pattern ${pattern} is not a regular expression: ${error.message}`, + ) + } + // The owed test, and any test whose joined names equal its own, which + // no pattern can tell apart from it. + const owedSet = sameNamed(h, owed.fi, owed.test, join) + if (JSON.stringify(selected) !== JSON.stringify(owedSet)) + return breach( + `the replay pattern ${JSON.stringify(pattern)} of ${file} runs tests ${JSON.stringify(selected)} when Vitest joins names with ${JSON.stringify(join)}, owed ${JSON.stringify(owedSet)}`, + ) + } } } return undefined } +// The replay lines the platform prints, each with the reader of its shell: +// one POSIX line elsewhere; on Windows a PowerShell line and a cmd line. +// Undefined when the block holds a different set of forms. +function replayLinesOf(block) { + const forms = + process.platform === `win32` + ? [ + [` replay (PowerShell): `, powerShellWords], + [` replay (cmd): `, cmdWords], + ] + : [[` replay: `, shellWords]] + const lines = block.filter((line) => /^ replay( \(\w+\))?: /.test(line)) + if (lines.length !== forms.length) return undefined + const found = forms.map(([prefix, read]) => { + const line = lines.find((text) => text.startsWith(prefix)) + return line === undefined + ? undefined + : { command: line.slice(prefix.length), read } + }) + return found.every(Boolean) ? found : undefined +} + +// The words a PowerShell command line splits into: single quotes hold text, +// a doubled single quote is one quote, and nothing expands. +function powerShellWords(line) { + const words = [] + let word + for (let i = 0; i < line.length;) { + const c = line[i] + if (c === ` `) { + if (word !== undefined) words.push(word) + word = undefined + i += 1 + continue + } + word ??= `` + if (c === `'`) { + let j = i + 1 + for (;;) { + if (j >= line.length) return undefined + if (line[j] === `'` && line[j + 1] === `'`) { + word += `'` + j += 2 + } else if (line[j] === `'`) break + else { + word += line[j] + j += 1 + } + } + i = j + 1 + } else if (/[$"()\`;&|<>]/.test(c)) { + return undefined + } else { + word += c + i += 1 + } + } + if (word !== undefined) words.push(word) + return words +} + +// The words a cmd command line splits into: double quotes hold text, and +// nothing expands or chains. +function cmdWords(line) { + const words = [] + let word + for (let i = 0; i < line.length;) { + const c = line[i] + if (c === ` `) { + if (word !== undefined) words.push(word) + word = undefined + i += 1 + continue + } + word ??= `` + if (c === `"`) { + const end = line.indexOf(`"`, i + 1) + if (end < 0) return undefined + word += line.slice(i + 1, end) + i = end + 1 + } else if (/[%&|<>^]/.test(c)) { + return undefined + } else { + word += c + i += 1 + } + } + if (word !== undefined) words.push(word) + return words +} + // A command line as a POSIX shell splits it into words (POSIX.1-2017, Shell // Command Language 2.2 and 2.6), when nothing in it expands, globs, or // redirects: plain characters, single-quoted text, backslash escapes, and diff --git a/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs index d2ef2d0..9cf098b 100644 --- a/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs @@ -751,6 +751,12 @@ async function driver(plan, observe) { const scratch = realpathSync(mkdtempSync(join(here, `.scratch-`))) const dir = plan.configPlace === `parent` ? join(scratch, `pkg`) : scratch mkdirSync(dir, { recursive: true }) + // Every package has a manifest, which the CLI refuses without (args.mjs); + // the fixture's content is unchanged, so its histories are too. + writeFileSync( + join(dir, `package.json`), + `{"name":"fixture","private":true}\n`, + ) try { // A symbolic link to the package, for sources spelled through it. symlinkSync(dir, `${dir}.link`, `dir`) From f79aae7d23893c115c89782d557934892407490b Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 12:33:51 -0700 Subject: [PATCH 10/53] fix(windows): forward-slash report paths, cmd quotes, six shards, and shipped citations Run 43483101749 (Windows, the first with the witnesses) showed: - The CLI printed `report \out\summary.md` and `tests\a.test.ts` in the done line and in the heartbeat. Report paths and display paths are now forward-slash (posix, check.mjs, ending.mjs), as the register's keys rule requires; the file system still gets native paths. - The classify oracle's cmd reader rejected `\"` inside a double-quoted argument, which Node's argument parser reads as a quote. It now reads it so. - The static and classify oracles' second scratch package had no manifest. - Shipped citations joined paths with backslashes, and npm names files with forward slashes. The comparison now uses forward slashes. - The refusal witness searches both streams (a refusal prints to stderr). - The static and fixed-seed e2e took 3 to 6 times their Linux time on Windows, and the Linux shards hit the limit. Windows runs six shards of each (pr-shards.mjs `oracle-gaps-static-windows`, `oracle-gaps-fixed-seed-windows`, checked by `pr-shards.mjs check`: each test in exactly one shard). The classify, probe-form, mutant-validity, and path-spelling oracles run in their own job. Evidence block - Law: report and display paths are forward-slash; a cmd line reads its quoted quotes as Node does (register: deterministic replay; the CLI's keys rule). - Reach: the existing e2e and static histories, unchanged. - RED: run 43483101749, which printed backslash report paths and failed classify on `\"`. - GREEN: pending the next Windows run; local POSIX suites pass (classify 276, shipped citations 2). - Mutant: not run for this commit. - Limits: the Windows e2e and static shard timings are measured on the next run before the budget claim (P12) is made. Co-authored-by: Isaac --- .github/workflows/pr.yml | 37 ++- .../tests/shipped-citations.test.js | 3 +- packages/oracles/oracle-gaps/check.mjs | 6 +- packages/oracles/oracle-gaps/ending.mjs | 3 +- ...p-report-classify-oracle.property.test.mjs | 17 +- ...gap-report-static-oracle.property.test.mjs | 4 + packages/oracles/scripts/pr-shards.mjs | 253 ++++++++++++++++++ 7 files changed, 302 insertions(+), 21 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index afc357d..4a9873c 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -218,7 +218,7 @@ jobs: - if: always() run: pnpm --filter @tanstack/eslint-plugin-oracles run test windows-gap-oracles: - name: Windows gap report oracles (${{ matrix.shard }}/3) + name: Windows gap report static oracle (${{ matrix.shard }}/6) runs-on: windows-latest # Not blocking until every Windows job is green; remove this line then. continue-on-error: true @@ -226,7 +226,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [1, 2, 3] + shard: [1, 2, 3, 4, 5, 6] steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -236,18 +236,31 @@ jobs: uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - run: pnpm --filter @tanstack/oracles run build # The same partition of the static oracle that Linux runs, by pattern. - - name: Gap report oracles + - name: Gap report static oracle shell: bash working-directory: packages/oracles env: SHARD: ${{ matrix.shard }} - run: | - if [ "$SHARD" = 3 ]; then - pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-classify gap-report-probe-form gap-report-mutant-validity gap-report-path-spelling - fi - pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-static -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-static "$SHARD")" + run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-static -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-static-windows "$SHARD")" + windows-gap-classify: + name: Windows gap report classify, probe-form, and mutant-validity oracles + runs-on: windows-latest + timeout-minutes: 20 + # Not blocking until every Windows job is green; remove this line then. + continue-on-error: true + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Setup Tools + uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 + - run: pnpm --filter @tanstack/oracles run build + - name: Gap report oracles + working-directory: packages/oracles + run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-classify gap-report-probe-form gap-report-mutant-validity gap-report-path-spelling windows-gap-end-to-end: - name: Windows gap report end-to-end oracle (fixed seed, ${{ matrix.shard }}/3) + name: Windows gap report end-to-end oracle (fixed seed, ${{ matrix.shard }}/6) runs-on: windows-latest # Not blocking until every Windows job is green; remove this line then. continue-on-error: true @@ -255,7 +268,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [1, 2, 3] + shard: [1, 2, 3, 4, 5, 6] steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -269,7 +282,7 @@ jobs: working-directory: packages/oracles env: SHARD: ${{ matrix.shard }} - run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-fixed-seed "$SHARD")" + run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-fixed-seed-windows "$SHARD")" windows-gap-cli: name: Windows gap CLI on a fixture runs-on: windows-latest @@ -402,7 +415,7 @@ jobs: node packages/oracles/oracle-gaps/index.mjs --package .ci-no-manifest --source src/x.ts > nm-out.txt 2> nm-err.txt $code = $LASTEXITCODE if ($code -ne 2) { throw "want a refusal with exit 2, got $code" } - if (-not (Select-String -Path nm-out.txt -Pattern 'has no package.json' -Quiet)) { throw "the refusal does not name package.json" } + if (-not (Select-String -Path nm-out.txt, nm-err.txt -Pattern 'has no package.json' -Quiet)) { throw "the refusal does not name package.json" } node packages/oracles/scripts/windows-witness.mjs done nm-out.txt nm-err.txt $code if ($LASTEXITCODE -ne 0) { throw "the done line failed" } # One early stop: SIGINT once the CLI has printed its first line. The diff --git a/packages/eslint-plugin-oracles/tests/shipped-citations.test.js b/packages/eslint-plugin-oracles/tests/shipped-citations.test.js index 1a4f424..d381dee 100644 --- a/packages/eslint-plugin-oracles/tests/shipped-citations.test.js +++ b/packages/eslint-plugin-oracles/tests/shipped-citations.test.js @@ -87,7 +87,8 @@ function strays(dir, file, text) { .filter((cited) => { const ships = [dirname(file), `.`].some((base) => [cited, cited.replace(/\.js$/, `.ts`)].some((name) => - shipped.includes(join(base, name)), + // npm names a shipped file with forward slashes, on every platform. + shipped.includes(join(base, name).replaceAll(`\\`, `/`)), ), ) const names = diff --git a/packages/oracles/oracle-gaps/check.mjs b/packages/oracles/oracle-gaps/check.mjs index ddc1119..c6c48cc 100644 --- a/packages/oracles/oracle-gaps/check.mjs +++ b/packages/oracles/oracle-gaps/check.mjs @@ -367,7 +367,7 @@ function discoveredConfig(pkg) { ) for (let dir = pkg; ; dir = dirname(dir)) { const name = names.find((file) => existsSync(join(dir, file))) - if (name) return relativePath(pkg, join(dir, name)) + if (name) return posix(relativePath(pkg, join(dir, name))) if (dirname(dir) === dir) return undefined } } @@ -804,7 +804,7 @@ export default async (env) => { pulse.log( streamLine( `busy`, - `${label} in ${relativePath(pkg, file)} for ${Math.floor(elapsed / 1000)}s`, + `${label} in ${posix(relativePath(pkg, file))} for ${Math.floor(elapsed / 1000)}s`, ), ) } @@ -2366,7 +2366,7 @@ export default async (env) => { baseline: baselineFiles, config: options.config, paths: { - report: join(options.out, `summary.md`), + report: posix(join(options.out, `summary.md`)), results: join(options.out, `results.json`), out: options.out, }, diff --git a/packages/oracles/oracle-gaps/ending.mjs b/packages/oracles/oracle-gaps/ending.mjs index f26a94c..14d40bf 100644 --- a/packages/oracles/oracle-gaps/ending.mjs +++ b/packages/oracles/oracle-gaps/ending.mjs @@ -14,6 +14,7 @@ import { mkdirSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' +import { posix } from './paths.mjs' /** A problem the author can fix: its cause, and the one step that fixes it. */ export class Refusal extends Error { @@ -42,7 +43,7 @@ export function endRun(error, out = error?.out) { results, `Refused: ${message}\n\nFix: ${error.fix}\n`, ) - ? `oracle-gaps: done refused, report ${join(out, `summary.md`)}` + ? `oracle-gaps: done refused, report ${posix(join(out, `summary.md`))}` : `oracle-gaps: done refused` return { exitCode: 2, stderr: `${[...block, done].join(`\n`)}\n`, done } } diff --git a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs index d312fba..6e2402a 100644 --- a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs @@ -2829,10 +2829,19 @@ function cmdWords(line) { } word ??= `` if (c === `"`) { - const end = line.indexOf(`"`, i + 1) - if (end < 0) return undefined - word += line.slice(i + 1, end) - i = end + 1 + // Inside double quotes, `\"` is a quote, as the program's argument parser + // reads it (Node's on Windows); cmd itself does not escape it. + let text = `` + let j = i + 1 + for (; j < line.length && line[j] !== `"`; j += 1) { + if (line[j] === `\\` && line[j + 1] === `"`) { + text += `"` + j += 1 + } else text += line[j] + } + if (j >= line.length) return undefined + word += text + i = j + 1 } else if (/[%&|<>^]/.test(c)) { return undefined } else { diff --git a/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs index 9cf098b..8dd1f0e 100644 --- a/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs @@ -214,6 +214,10 @@ function analyzeFixture(dir, files, plan) { // state, and which pairs of outputs are the same directory. function defaultOutputsOf(dir, sources) { const other = realpathSync(mkdtempSync(join(here, `.scratch-`))) + writeFileSync( + join(other, `package.json`), + `{"name":"other","private":true}\n`, + ) // The other package holds the same source files, which --source must name. for (const source of sources) { mkdirSync(dirname(join(other, source)), { recursive: true }) diff --git a/packages/oracles/scripts/pr-shards.mjs b/packages/oracles/scripts/pr-shards.mjs index 2a81190..9e16288 100644 --- a/packages/oracles/scripts/pr-shards.mjs +++ b/packages/oracles/scripts/pr-shards.mjs @@ -313,6 +313,259 @@ export const partitions = { 3: rest, }, }, + 'oracle-gaps-static-windows': { + args: [`--config`, `oracle-gaps/vitest.config.mjs`, `gap-report-static`], + suite: `oracle-gaps.report-static`, + select: ``, + // Windows runs on its own partition: the same tests, in six shards, + // because each Linux shard takes 3 to 6 times as long on windows-latest. + shards: { + 1: anyOf([ + "calibration: a derived class's super call is made conditional", + 'calibration: a refusal names another cause', + 'calibration: production code is marked as probe code', + 'determinism: constructed history: a declaration fault: outsideSource', + 'determinism: constructed history: a source in another letter case', + 'determinism: constructed history: an owns that is not a literal in a expect-aliased assertion refuses nothing', + 'determinism: constructed history: an owns that is not a literal in a node-assert-default assertion refuses nothing', + 'determinism: constructed history: bad arguments: directoryConfig', + 'determinism: constructed history: bad arguments: mutantLacksField', + 'determinism: constructed history: bad arguments: repeatedMutantIds', + 'determinism: constructed history: no oracle file refuses the run', + 'determinism: generated 1', + 'fixed seed 20261105: a declaration fault: outsideSource', + 'fixed seed 20261105: a source in another letter case', + 'fixed seed 20261105: an owns that is not a literal in a expect-aliased assertion refuses nothing', + 'fixed seed 20261105: an owns that is not a literal in a node-assert-default assertion refuses nothing', + 'fixed seed 20261105: bad arguments: directoryConfig', + 'fixed seed 20261105: bad arguments: mutantLacksField', + 'fixed seed 20261105: bad arguments: repeatedMutantIds', + 'fixed seed 20261105: module data owned by readers and by name', + 'random seed: a bad entry in a run with no owner names the bad entry', + 'random seed: a derived class calling super after ||', + 'random seed: a whole-file owner', + 'random seed: an owns that is not a literal in a expect-computed assertion refuses nothing', + 'random seed: an owns that is not a literal in a node-assert-namespace assertion refuses nothing', + 'random seed: bad arguments: envPrefix', + 'random seed: bad arguments: noSource', + 'random seed: bad arguments: unknown', + 'random seed: no owns entry refuses the run', + ]), + 2: anyOf([ + 'calibration: a key drops its statement', + 'calibration: a refusal prints a stack trace, as a crash does', + 'calibration: the instrumented derived class differs with no mutant active', + 'determinism: constructed history: a derived class calling super after &&', + 'determinism: constructed history: a source through a link to the package', + 'determinism: constructed history: an owns that is not a literal in a expect-argument assertion refuses nothing', + 'determinism: constructed history: an owns that is not a literal in a node-assert-named assertion refuses nothing', + 'determinism: constructed history: bad arguments: directorySource', + 'determinism: constructed history: bad arguments: noPackage', + 'determinism: constructed history: bad arguments: seed', + 'determinism: constructed history: no owns entry refuses the run', + 'determinism: generated 2', + 'fixed seed 20261105: a derived class calling super after &&', + 'fixed seed 20261105: a source through a link to the package', + 'fixed seed 20261105: an owns that is not a literal in a expect-argument assertion refuses nothing', + 'fixed seed 20261105: an owns that is not a literal in a node-assert-named assertion refuses nothing', + 'fixed seed 20261105: bad arguments: directorySource', + 'fixed seed 20261105: bad arguments: noPackage', + 'fixed seed 20261105: bad arguments: seed', + 'fixed seed 20261105: no oracle file refuses the run', + 'random seed: a declaration fault: badEntry', + 'random seed: a derived class calling super in an arrow', + 'random seed: an owns that is not a literal in a assert assertion refuses nothing', + 'random seed: an owns that is not a literal in a expect-not assertion refuses nothing', + 'random seed: bad arguments: absentSource', + 'random seed: bad arguments: missingConfig', + 'random seed: bad arguments: noValue', + 'random seed: every static state', + 'random seed: owns read from both declarations', + ]), + 3: anyOf([ + 'calibration: a mutant is dropped', + "calibration: a refusal's fix offers a way around it", + 'determinism: constructed history: a bad entry in a run with no owner names the bad entry', + 'determinism: constructed history: a derived class calling super after ||', + 'determinism: constructed history: a whole-file owner', + 'determinism: constructed history: an owns that is not a literal in a expect-computed assertion refuses nothing', + 'determinism: constructed history: an owns that is not a literal in a node-assert-namespace assertion refuses nothing', + 'determinism: constructed history: bad arguments: envPrefix', + 'determinism: constructed history: bad arguments: noSource', + 'determinism: constructed history: bad arguments: unknown', + 'determinism: constructed history: owns read from both declarations', + 'fixed seed 20261105: a bad entry in a run with no owner names the bad entry', + 'fixed seed 20261105: a derived class calling super after ||', + 'fixed seed 20261105: a whole-file owner', + 'fixed seed 20261105: an owns that is not a literal in a expect-computed assertion refuses nothing', + 'fixed seed 20261105: an owns that is not a literal in a node-assert-namespace assertion refuses nothing', + 'fixed seed 20261105: bad arguments: envPrefix', + 'fixed seed 20261105: bad arguments: noSource', + 'fixed seed 20261105: bad arguments: unknown', + 'fixed seed 20261105: no owns entry refuses the run', + 'random seed: a declaration fault: missingSource', + 'random seed: a literal owns in an assertion makes no oracle file', + 'random seed: an owns that is not a literal in a chai assertion refuses nothing', + 'random seed: an owns that is not a literal in a expect-soft assertion refuses nothing', + 'random seed: bad arguments: badMutantJson', + 'random seed: bad arguments: missingMutants', + 'random seed: bad arguments: outsideSource', + 'random seed: generated', + 'random seed: owns read from oracleProperty', + ]), + 4: anyOf([ + 'calibration: a refusal ends without its done line', + 'calibration: a rune is moved from its place', + 'determinism: constructed history: a declaration fault: badEntry', + 'determinism: constructed history: a derived class calling super in an arrow', + 'determinism: constructed history: an owns that is not a literal in a assert assertion refuses nothing', + 'determinism: constructed history: an owns that is not a literal in a expect-not assertion refuses nothing', + 'determinism: constructed history: bad arguments: absentSource', + 'determinism: constructed history: bad arguments: missingConfig', + 'determinism: constructed history: bad arguments: noValue', + 'determinism: constructed history: every static state', + 'determinism: constructed history: owns read from oracleProperty', + 'fixed seed 20261105: a declaration fault: badEntry', + 'fixed seed 20261105: a derived class calling super in an arrow', + 'fixed seed 20261105: an owns that is not a literal in a assert assertion refuses nothing', + 'fixed seed 20261105: an owns that is not a literal in a expect-not assertion refuses nothing', + 'fixed seed 20261105: bad arguments: absentSource', + 'fixed seed 20261105: bad arguments: missingConfig', + 'fixed seed 20261105: bad arguments: noValue', + 'fixed seed 20261105: every static state', + 'fixed seed 20261105: owns read from both declarations', + 'random seed: a declaration fault: nonLiteral', + 'random seed: a plain-parameter derived class', + 'random seed: an owns that is not a literal in a expect assertion refuses nothing', + 'random seed: an owns that is not a literal in a expect-type-of assertion refuses nothing', + 'random seed: bad arguments: count', + 'random seed: bad arguments: missingPackage', + 'random seed: bad arguments: pool', + 'random seed: module data named by an entry leaves its methods to their callers', + 'random seed: unnamed exports owned through an import', + ]), + 5: anyOf([ + 'calibration: a refusal exits 1, as a crash does', + 'calibration: an edit moves a key', + 'determinism: constructed history: a declaration fault: missingSource', + 'determinism: constructed history: a literal owns in an assertion makes no oracle file', + 'determinism: constructed history: an owns that is not a literal in a chai assertion refuses nothing', + 'determinism: constructed history: an owns that is not a literal in a expect-soft assertion refuses nothing', + 'determinism: constructed history: bad arguments: badMutantJson', + 'determinism: constructed history: bad arguments: missingMutants', + 'determinism: constructed history: bad arguments: outsideSource', + 'determinism: constructed history: module data named by an entry leaves its methods to their callers', + 'determinism: constructed history: unnamed exports owned through an import', + 'fixed seed 20261105: a declaration fault: missingSource', + 'fixed seed 20261105: a literal owns in an assertion makes no oracle file', + 'fixed seed 20261105: an owns that is not a literal in a chai assertion refuses nothing', + 'fixed seed 20261105: an owns that is not a literal in a expect-soft assertion refuses nothing', + 'fixed seed 20261105: bad arguments: badMutantJson', + 'fixed seed 20261105: bad arguments: missingMutants', + 'fixed seed 20261105: bad arguments: outsideSource', + 'fixed seed 20261105: generated', + 'fixed seed 20261105: owns read from oracleProperty', + 'random seed: a declaration fault: outsideSource', + 'random seed: a source in another letter case', + 'random seed: an owns that is not a literal in a expect-aliased assertion refuses nothing', + 'random seed: an owns that is not a literal in a node-assert-default assertion refuses nothing', + 'random seed: bad arguments: directoryConfig', + 'random seed: bad arguments: mutantLacksField', + 'random seed: bad arguments: repeatedMutantIds', + 'random seed: module data owned by readers and by name', + ]), + 6: anyOf([ + 'calibration: a refusal gives no fix', + 'calibration: an owned mutant is reported unowned', + 'determinism: constructed history: a declaration fault: nonLiteral', + 'determinism: constructed history: a plain-parameter derived class', + 'determinism: constructed history: an owns that is not a literal in a expect assertion refuses nothing', + 'determinism: constructed history: an owns that is not a literal in a expect-type-of assertion refuses nothing', + 'determinism: constructed history: bad arguments: count', + 'determinism: constructed history: bad arguments: missingPackage', + 'determinism: constructed history: bad arguments: pool', + 'determinism: constructed history: module data owned by readers and by name', + 'determinism: generated 0', + 'fixed seed 20261105: a declaration fault: nonLiteral', + 'fixed seed 20261105: a plain-parameter derived class', + 'fixed seed 20261105: an owns that is not a literal in a expect assertion refuses nothing', + 'fixed seed 20261105: an owns that is not a literal in a expect-type-of assertion refuses nothing', + 'fixed seed 20261105: bad arguments: count', + 'fixed seed 20261105: bad arguments: missingPackage', + 'fixed seed 20261105: bad arguments: pool', + 'fixed seed 20261105: module data named by an entry leaves its methods to their callers', + 'fixed seed 20261105: unnamed exports owned through an import', + 'random seed: a derived class calling super after &&', + 'random seed: a source through a link to the package', + 'random seed: an owns that is not a literal in a expect-argument assertion refuses nothing', + 'random seed: an owns that is not a literal in a node-assert-named assertion refuses nothing', + 'random seed: bad arguments: directorySource', + 'random seed: bad arguments: noPackage', + 'random seed: bad arguments: seed', + 'random seed: no oracle file refuses the run', + ]), + }, + }, + 'oracle-gaps-fixed-seed-windows': { + args: [`--config`, `oracle-gaps/vitest.config.mjs`, `gap-report-oracle`], + suite: `oracle-gaps.report`, + select: `fixed seed 20261105: `, + previously: `fixed seed`, + // Windows runs on its own partition: the same tests, in six shards, + // because each Linux shard takes 3 to 6 times as long on windows-latest. + shards: { + 1: anyOf([ + 'a .cjs config that calls require', + 'a declaration fault: brokenOwner', + 'a declaration fault: missingSource', + 'a failing owner under a config that --config names', + 'a run with eleven contexts', + 'an edit sequence through the daemon', + "test.projects in a parent's config", + ]), + 2: anyOf([ + 'a .ts config in a package with no module type that calls require', + 'a declaration fault: brokenVitest', + 'a declaration fault: noTypescript', + "a failing owner under its parent's config", + 'a second daemon check of the budget witness', + 'an edit that changed the cold result after a hang replaced the only context', + 'two authored mutants', + ]), + 3: anyOf([ + 'a config fault in a run that owns nothing', + 'a declaration fault: configNoDefault', + 'a declaration fault: nonLiteral', + 'a lazy module loaded before a hang in the same batch', + 'a stale authored mutant and a reformat', + 'an edit to a module that only mutated runs load', + ]), + 4: anyOf([ + 'a config that --config names', + 'a declaration fault: configNull', + 'a declaration fault: outsideSource', + 'a load failure beside a hang', + 'an ambiguous authored mutant', + 'every outcome class', + ]), + 5: anyOf([ + 'a declaration fault: apiLessTypescript', + 'a declaration fault: configSyntax', + 'a declaration fault: projects', + 'a package whose parent directory has its config', + 'an awaiting config in a package with no module type', + 'generated', + ]), + 6: anyOf([ + 'a declaration fault: badEntry', + 'a declaration fault: configThrows', + 'a declaration fault: setupThrows', + 'a package with no config', + 'an edit sequence over the witness module', + 'ordinary tests left out by default', + ]), + }, + }, } /** A shard's pattern over the name that follows the suite's. */ From 8697227178a3c9e4e735431369f1a34e760f4dfa Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 13:06:23 -0700 Subject: [PATCH 11/53] fix(oracle-gaps): keys and report paths are forward-slash in the CLI and its models Run 43493431349 (Windows) failed the static and e2e oracles on report paths: the done line names its report in forward slashes (the register's Windows paths rule), but the model built the expected line from the native output directory, and the declared oracle files were keyed by native relative paths (`tests\a.test.ts`), so a heartbeat's file was not an owner. - analysis.mjs, check.mjs: declared oracle files and copies are keyed by their forward-slash relative path (relativeWithin). - gap-report-model-oracle.mjs: the model's expected done line is forward-slash (inline, the model does not import production). - shipped-citations.test.js: the rejection test's strings are forward-slash. Evidence block - Law: keys and report paths use forward slashes on every OS (register, Windows paths, #58); filesystem calls get native paths. - Reach: the existing static and e2e histories; no new histories. - RED: run 43493431349: static 40 refusal-ending violations, e2e 17 baseline lead, 12 report-shaped ending, 4 streaming, 3 greppable stream. - GREEN: pending the next Windows run. Local POSIX: shipped citations 2 of 2, CLI lints clean; the Linux keys are unchanged. - Mutant: not run. - Limits: the Windows e2e and static timings are measured on the next run. Co-authored-by: Isaac --- .../eslint-plugin-oracles/tests/shipped-citations.test.js | 3 ++- packages/oracles/oracle-gaps/analysis.mjs | 4 ++-- packages/oracles/oracle-gaps/check.mjs | 2 +- packages/oracles/oracle-gaps/gap-report-model-oracle.mjs | 5 ++++- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/eslint-plugin-oracles/tests/shipped-citations.test.js b/packages/eslint-plugin-oracles/tests/shipped-citations.test.js index d381dee..1c77c1d 100644 --- a/packages/eslint-plugin-oracles/tests/shipped-citations.test.js +++ b/packages/eslint-plugin-oracles/tests/shipped-citations.test.js @@ -103,7 +103,8 @@ function strays(dir, file, text) { .map((match) => match[1]) .filter((name) => repoOnlyNames.has(name)) return [...paths, ...names].map( - (cited) => `${relative(repo, join(dir, file))}: ${cited}`, + (cited) => + `${relative(repo, join(dir, file)).replaceAll(`\\`, `/`)}: ${cited}`, ) } diff --git a/packages/oracles/oracle-gaps/analysis.mjs b/packages/oracles/oracle-gaps/analysis.mjs index b884cee..763c92d 100644 --- a/packages/oracles/oracle-gaps/analysis.mjs +++ b/packages/oracles/oracle-gaps/analysis.mjs @@ -7,7 +7,7 @@ import { existsSync, readFileSync, realpathSync } from 'node:fs' import { basename, join } from 'node:path' import { packagePath } from './args.mjs' -import { posix } from './paths.mjs' +import { posix, relativeWithin } from './paths.mjs' import { Refusal } from './ending.mjs' import { ownership, readOwns, splitEntry } from './owners.mjs' import { instrument } from './schemata.mjs' @@ -60,7 +60,7 @@ function hostFor(ts) { * has an owner through the call trees, throws too. */ export function analyze(ts, pkg, sources, testFiles) { - const relative = (file) => file.slice(pkg.length + 1) + const relative = (file) => relativeWithin(pkg, file) ?? posix(file) // An oracle file is a test file that calls `oracleProperty`, or passes a // literal `owns` list to any call, such as a wrapper's (owners.mjs). const declarations = new Map() diff --git a/packages/oracles/oracle-gaps/check.mjs b/packages/oracles/oracle-gaps/check.mjs index c6c48cc..aa4fd19 100644 --- a/packages/oracles/oracle-gaps/check.mjs +++ b/packages/oracles/oracle-gaps/check.mjs @@ -1667,7 +1667,7 @@ export default async (env) => { const vitest = contexts[0] const fileSpecs = new Map(contextSpecs.get(vitest)) - const relative = (file) => file.slice(pkg.length + 1) + const relative = (file) => relativeWithin(pkg, file) ?? posix(file) const oracleFiles = new Set(declarations.keys()) const runIdentity = identityOf( options, diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index f670836..6e5a5d4 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -5478,7 +5478,10 @@ function endingOf(exit, stderr, out, name = (text) => text, stdout = ``) { done, stdout: printed.map(name), namesOutput: - out !== undefined && done === name(`refused, report ${out}/summary.md`), + // The done line names the report in forward slashes on every platform + // (the register's Windows paths rule). + out !== undefined && + done === name(`refused, report ${out.replaceAll(`\\`, `/`)}/summary.md`), // An internal error's log beside a refusal would be another run's. errorLog: out !== undefined && existsSync(join(out, `error.log`)), stackFrames: /^\s+at /m.test(text), From 0c9f38d688490ee9c1e35fc7116d8abf39c29807 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 13:26:25 -0700 Subject: [PATCH 12/53] ci(windows): the replay witness runs after a failed stop witness The early-stop step failed on Windows (SIGINT ends the process through TerminateProcess, so the CLI's stop handler never runs; the run ended with a null exit code after its done line), which skipped the replay witness behind it. The replay witness now runs regardless, so each witness reports. Co-authored-by: Isaac --- .github/workflows/pr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 4a9873c..2a2d373 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -427,6 +427,7 @@ jobs: # lines each reproduce its failure in their own shell (RED: the POSIX # line does not, in PowerShell). - name: Replay lines reproduce the failure in PowerShell and cmd + if: always() run: | node "$env:GITHUB_WORKSPACE\packages\oracles\node_modules\vitest\vitest.mjs" run --root .ci-replay > replay-out.txt 2>&1 node packages/oracles/scripts/windows-witness.mjs replay replay-out.txt "$env:GITHUB_WORKSPACE\.ci-replay" From dc71773283d391ce770b50212e2ce7e8c8c00014 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 13:50:29 -0700 Subject: [PATCH 13/53] ci(windows): print the replay output the witness reads (diagnostic) The replay witness found no per-shell lines in the fixture's failing output. This prints the first lines of that output in the same step, so the next run shows what vitest printed. Diagnostic, to be removed with the witness fix. Co-authored-by: Isaac --- .github/workflows/pr.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 2a2d373..a6d5fba 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -430,5 +430,7 @@ jobs: if: always() run: | node "$env:GITHUB_WORKSPACE\packages\oracles\node_modules\vitest\vitest.mjs" run --root .ci-replay > replay-out.txt 2>&1 + # Diagnostic: the output the witness reads, so a missing line is visible. + Get-Content replay-out.txt -TotalCount 120 node packages/oracles/scripts/windows-witness.mjs replay replay-out.txt "$env:GITHUB_WORKSPACE\.ci-replay" if ($LASTEXITCODE -ne 0) { throw "the replay witness failed" } From b8b48d70fe453f24d2b64d32801689599de44938 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 14:14:25 -0700 Subject: [PATCH 14/53] ci(windows): the replay fixture declares its fixed seed The replay fixture failed to load (TypeError reading fixedSeeds[0]), so the witness saw no failure. It now declares fixedSeeds: [1]. Co-authored-by: Isaac --- .github/workflows/pr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index a6d5fba..b5726ca 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -369,6 +369,7 @@ jobs: histories: { arbitrary: fc.integer({ min: -20, max: 20 }), runs: 20, + fixedSeeds: [1], states: [], measured: [], }, From c1c34d2a8ab75095b28086864f17db5f2393d65c Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 14:58:19 -0700 Subject: [PATCH 15/53] fix(windows): the stop law on win32, the stop hook, the argv cap, and forward-slash done lines - Stop (#57, all-or-nothing) on the unsupervised win32 path: a stop of a run that has started now writes its `oracle-gaps: done stopped ()` line on both streams and exits 128 + signal, as the supervisor does. A stop before the run starts still prints nothing (index.mjs, runStarted). - Test-only stop hook: when the file ORACLE_GAPS_TEST_STOP names appears, the CLI takes the same stop as a SIGINT (index.mjs). The Windows stop witness uses it (a non-console parent cannot deliver Ctrl+C there; child.kill on win32 is TerminateProcess, which is a SIGKILL-like limit, so no line is owed). - Static oracle: the `unknown` long-argument case uses 30,000 characters on win32. Windows caps a command line at 32,767 characters; a longer one never starts (status null). Narrowing, recorded in the PR: the POSIX runs keep 100,000. - e2e and static model: the expected done line's report path is forward-slash (inline in the model; the e2e corruption check likewise). - The diagnostic Get-Content line is removed. Evidence block - Law: the all-or-nothing stop law (#57) on every platform; keys and report paths forward-slash (register: Windows paths; stop window on every platform). - Reach: the existing static and e2e histories; the stop witness is one run. - RED: run 43498457502, stop witness `stop was not all-or-nothing (signal SIGINT, exit null)`; static `with unknown arguments, the refusal exited null`; e2e `the last stdout line is ... report D:/...`, not `...\out\summary.md`. - GREEN: pending the next Windows run. Local: the CLI lints and loads; the stop witness is queued on the e2e lock for a fixture run. - Mutant: pending (the replay witness, on a throwaway branch). - Limits: OS delivery of Ctrl+C is not exercised on Windows; TerminateProcess is treated like SIGKILL; the win32 argv cap narrows the long-argument case. Co-authored-by: Isaac --- .github/workflows/pr.yml | 2 - .../oracle-gaps/gap-report-model-oracle.mjs | 2 +- .../gap-report-oracle.property.test.mjs | 2 +- ...gap-report-static-oracle.property.test.mjs | 9 ++++- packages/oracles/oracle-gaps/index.mjs | 26 ++++++++++++- packages/oracles/scripts/windows-witness.mjs | 39 ++++++++++--------- 6 files changed, 54 insertions(+), 26 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index b5726ca..ae96eea 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -431,7 +431,5 @@ jobs: if: always() run: | node "$env:GITHUB_WORKSPACE\packages\oracles\node_modules\vitest\vitest.mjs" run --root .ci-replay > replay-out.txt 2>&1 - # Diagnostic: the output the witness reads, so a missing line is visible. - Get-Content replay-out.txt -TotalCount 120 node packages/oracles/scripts/windows-witness.mjs replay replay-out.txt "$env:GITHUB_WORKSPACE\.ci-replay" if ($LASTEXITCODE -ne 0) { throw "the replay witness failed" } diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index 6e5a5d4..6ebe9f3 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -4226,7 +4226,7 @@ const verdictLine = (files, failing) => * the path of its full report in output directory `out`. */ const doneLine = (gaps, provisional, out) => - `oracle-gaps: done completed, ${gaps} gap${gaps === 1 ? `` : `s`}${provisional ? ` (provisional)` : ``}, report ${out}/summary.md` + `oracle-gaps: done completed, ${gaps} gap${gaps === 1 ? `` : `s`}${provisional ? ` (provisional)` : ``}, report ${out.replaceAll(`\\`, `/`)}/summary.md` function model(plan) { const refused = refusal(plan) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 86cc8cc..e986703 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -2777,7 +2777,7 @@ const calibrations = [ // only the law that the run must refuse rejects it. corrupt: corruptCli((cli) => { if (cli.exit === 0) return undefined - const done = `oracle-gaps: done completed, 0 gaps, report ${cli.outDir}/summary.md` + const done = `oracle-gaps: done completed, 0 gaps, report ${cli.outDir.replaceAll(`\\`, `/`)}/summary.md` const completed = (stream) => ({ ...stream, last: done, dones: 1 }) return { ...cli, diff --git a/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs index 8dd1f0e..97e4a23 100644 --- a/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs @@ -433,7 +433,14 @@ function argumentFault(bad, dir, sources) { // An option that takes a value, given last with none. noValue: [[...valid, bad.option], bad.option], unknown: [ - [...valid, `--sources${bad.long ? `x`.repeat(100_000) : ``}`, sources], + // Windows caps a command line at 32,767 characters, and a longer one never + // starts (its status is null): there the long argument is 30,000 (a + // narrowing recorded in the PR; the POSIX runs keep 100,000). + [ + ...valid, + `--sources${bad.long ? `x`.repeat(process.platform === `win32` ? 30_000 : 100_000) : ``}`, + sources, + ], `--sources`, ], missingMutants: [ diff --git a/packages/oracles/oracle-gaps/index.mjs b/packages/oracles/oracle-gaps/index.mjs index d2639fe..d55502d 100755 --- a/packages/oracles/oracle-gaps/index.mjs +++ b/packages/oracles/oracle-gaps/index.mjs @@ -122,7 +122,7 @@ // ended 2 s later is killed, and so is any process left in the group when // the run ends. The run kills its group itself if the CLI dies first. import { spawn } from 'node:child_process' -import { realpathSync } from 'node:fs' +import { existsSync, realpathSync } from 'node:fs' import { constants } from 'node:os' import { pathToFileURL } from 'node:url' import { parseArgs, usage } from './args.mjs' @@ -343,11 +343,32 @@ else if (main) { // its ending, it says so to the supervisor, and a stop no longer cuts it // off: its done line is then the last line of each stream. let writingEnding = false + // A run that has started owes its stop one stopped line on each stream + // (#57, all-or-nothing). A supervised run's supervisor writes it; an + // unsupervised run (win32, no process groups) writes it here. A stop before + // the run starts prints nothing, as the law allows. + let runStarted = false const onStop = (signal) => { - if (!writingEnding) process.exit(128 + constants.signals[signal]) + if (writingEnding) return + const code = 128 + constants.signals[signal] + if (!runStarted || supervised) return process.exit(code) + const line = `oracle-gaps: done stopped (${signal})\n` + process.stdout.write(line, () => + process.stderr.write(line, () => process.exit(code)), + ) } process.on(`SIGINT`, onStop) process.on(`SIGTERM`, onStop) + // A test-only hook for the CLI's own oracle, not for users (as + // ORACLE_GAPS_TEST_HOLD_EXIT_MS): when the file ORACLE_GAPS_TEST_STOP names + // appears, the CLI takes the same stop as a SIGINT. CI's Windows witness uses + // it, since a non-console parent cannot deliver Ctrl+C there. + if (process.env.ORACLE_GAPS_TEST_STOP !== undefined) { + const file = process.env.ORACLE_GAPS_TEST_STOP + setInterval(() => { + if (existsSync(file)) onStop(`SIGINT`) + }, 50).unref() + } const holdStops = () => { writingEnding = true process.send?.(`ending`) @@ -369,6 +390,7 @@ else if (main) { if (options.help) console.log(usage) else if (options.stop) await stopDaemon(options.pkg) else { + runStarted = true const summary = await (options.daemon ? checkThroughDaemon : checkCold)( options, ) diff --git a/packages/oracles/scripts/windows-witness.mjs b/packages/oracles/scripts/windows-witness.mjs index 16ced2d..d0d2441 100644 --- a/packages/oracles/scripts/windows-witness.mjs +++ b/packages/oracles/scripts/windows-witness.mjs @@ -22,7 +22,7 @@ // both streams, matching the exit code (130 or 143); (b) no output at all. // Anything else, including a partial output, fails. import { spawn, spawnSync } from 'node:child_process' -import { readFileSync } from 'node:fs' +import { readFileSync, rmSync, writeFileSync } from 'node:fs' import { delimiter, join, resolve } from 'node:path' import { fileURLToPath } from 'node:url' @@ -145,38 +145,37 @@ if (command === `done`) { ) } } else if (command === `stop`) { + // The stop is the CLI's test-only hook (ORACLE_GAPS_TEST_STOP): a file that + // appears takes the same stop as a SIGINT. Node on Windows cannot deliver a + // Ctrl+C from a non-console parent, and child.kill('SIGINT') there is + // TerminateProcess, a limit like SIGKILL, so no line is owed for it. const [dir, ...cliArgs] = args + const stopFile = join(dir, `.stop-requested`) + rmSync(stopFile, { force: true }) const child = spawn(process.execPath, [cli, ...cliArgs], { cwd: dir, + env: { ...process.env, ORACLE_GAPS_TEST_STOP: stopFile }, stdio: [`ignore`, `pipe`, `pipe`], }) let stdout = `` let stderr = `` - let sent = false + let requested = false child.stdout.on(`data`, (chunk) => { stdout += chunk - if (!sent && stdout.length > 0) { - sent = true - child.kill(`SIGINT`) + // The run has begun once it prints its first line: the stop lands then. + if (!requested && stdout.length > 0) { + requested = true + writeFileSync(stopFile, ``) } }) child.stderr.on(`data`, (chunk) => { stderr += chunk }) child.on(`close`, (code, signal) => { - if (!sent) { - // The run ended before it printed anything: outcome (b) if it printed - // nothing at all, which is the only way to end before a first line. - if (stdout === `` && stderr === ``) - console.log(`stop outcome (b): no output`) - else - fail( - `the run ended before any stop was sent, with ${JSON.stringify(stdout.slice(0, 200))}`, - ) - return - } - if (stdout === `` && stderr === ``) { - console.log(`stop outcome (b): no output`) + if (!requested) { + fail( + `the run ended before its first line (exit ${code}, signal ${signal}), so no stop was sent`, + ) return } const problems = [ @@ -188,7 +187,9 @@ if (command === `done`) { `stop was not all-or-nothing (signal ${signal}, exit ${code}): ${problems.join('; ')}`, ) else - console.log(`stop outcome (a): done line on both streams, exit ${code}`) + console.log( + `stop outcome (a): stopped line on both streams, exit ${code}`, + ) }) } else { fail(`unknown command ${command}`) From ee1a5fd4278509189a1d0d132600ccd70328c432 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 15:27:30 -0700 Subject: [PATCH 16/53] fix(oracle-gaps): the temporary directory's real path for state, socket, and logs The default output directory is under the OS temporary directory, which on windows-latest is `C:\Users\RUNNER~1\...` (an 8.3 short name). The model's expected done line names the package by its real path, so the report path never matched (e2e: `the last stdout line is ... report C:/Users/RUNNER~1/...`). The temporary directory is now taken by its real path (realPathOf) for the state directory, the socket, and the error log. Evidence block - Law: keys and report paths use one spelling; a real path on both sides (register: Windows paths; the realpath rule). - Reach: the existing e2e histories (the done-line law). - RED: e2e run 43515364499, done line with RUNNER~1 vs the package path. - GREEN: pending the next Windows run. - Mutant: pending; the replay witness mutant runs on the throwaway PR. - Limits: the stdout gap-count mismatch (`2 gap lines for 0 gaps`) and the cold internal error in the incremental law are not yet explained. Co-authored-by: Isaac --- packages/oracles/oracle-gaps/ending.mjs | 4 ++-- packages/oracles/oracle-gaps/evidence.mjs | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/oracles/oracle-gaps/ending.mjs b/packages/oracles/oracle-gaps/ending.mjs index 14d40bf..23d279a 100644 --- a/packages/oracles/oracle-gaps/ending.mjs +++ b/packages/oracles/oracle-gaps/ending.mjs @@ -14,7 +14,7 @@ import { mkdirSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { posix } from './paths.mjs' +import { posix, realPathOf } from './paths.mjs' /** A problem the author can fix: its cause, and the one step that fixes it. */ export class Refusal extends Error { @@ -51,7 +51,7 @@ export function endRun(error, out = error?.out) { const stack = `${error?.stack ?? message}\n` let log = join(out ?? ``, `error.log`) if (!write(out, results, `Internal error: ${message}\n`, stack)) { - log = join(tmpdir(), `oracle-gaps-error-${process.pid}.log`) + log = join(realPathOf(tmpdir()), `oracle-gaps-error-${process.pid}.log`) writeFileSync(log, stack) } const done = `oracle-gaps: done error, log ${log}` diff --git a/packages/oracles/oracle-gaps/evidence.mjs b/packages/oracles/oracle-gaps/evidence.mjs index 7a1d4bd..dd018c1 100644 --- a/packages/oracles/oracle-gaps/evidence.mjs +++ b/packages/oracles/oracle-gaps/evidence.mjs @@ -15,6 +15,7 @@ import { } from 'node:fs' import { createHash } from 'node:crypto' import { tmpdir } from 'node:os' +import { realPathOf } from './paths.mjs' import { join } from 'node:path' // A package's key: its real path, which names the repository checkout too. @@ -23,10 +24,12 @@ const keyOf = (pkg) => createHash(`sha1`).update(pkg).digest(`hex`).slice(0, 12) * The socket of a package's daemon. A Unix socket path must stay short, and * on Windows a socket is a named pipe, which has no file to remove. */ +/** The OS temporary directory, by its real path (paths.mjs): a short name is expanded. */ +const temporaryDirectory = () => realPathOf(tmpdir()) export const socketOf = (pkg) => process.platform === `win32` ? `\\\\.\\pipe\\oracle-gaps-${keyOf(pkg)}` - : join(tmpdir(), `oracle-gaps-${keyOf(pkg)}.sock`) + : join(temporaryDirectory(), `oracle-gaps-${keyOf(pkg)}.sock`) /** Removes a daemon's socket file; a named pipe has none. */ export const removeSocket = (socket) => { if (process.platform !== `win32`) rmSync(socket, { force: true }) @@ -39,7 +42,8 @@ export const removeSocket = (socket) => { * directory (the v20 freeze's namespacing decision). A state the OS removed * is a cold start. */ -export const stateDirOf = (pkg) => join(tmpdir(), `oracle-gaps-${keyOf(pkg)}`) +export const stateDirOf = (pkg) => + join(temporaryDirectory(), `oracle-gaps-${keyOf(pkg)}`) // A run keeps a few versions, newest first, each under the inputs it read, // as a build system keeps several verifying traces: an edit and its undo From 70b118d480804b7fcb2f4d1489729c84873bf83b Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 15:48:53 -0700 Subject: [PATCH 17/53] fix(oracle-gaps): the e2e fixture's temporary directory by its native real path The e2e oracle realpaths the OS temporary directory with the JavaScript realpath, which keeps the 8.3 name (C:\Users\RUNNER~1). The CLI prints the native real path, so the expected report line never matched. Use realpathSync.native, as the CLI does. Evidence block - Law: one real path on both sides (the realpath rule; Windows paths). - Reach: the existing e2e histories. - RED: run 43515364499 and ee1a5fd: the done line names RUNNER~1. - GREEN: pending the next Windows run. - Limits: the static `output namespace` violations (82 in the last run) are not yet traced. Co-authored-by: Isaac --- .../oracles/oracle-gaps/gap-report-oracle.property.test.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index e986703..9f0fe4c 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -1578,7 +1578,7 @@ async function driver(plan, observe) { const scratch = realpathSync( mkdtempSync( outsideRepository(plan) - ? join(realpathSync(tmpdir()), `gap-fixture-`) + ? join(realpathSync.native(tmpdir()), `gap-fixture-`) : join(here, `.scratch-`), ), ) From ebc39e6e963c2697aaa4f6711240c94faa0a307f Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 16:15:44 -0700 Subject: [PATCH 18/53] ci(windows): split the blocking Windows jobs from the non-blocking ones; fix the tsc lookup type - Blocking now: the Windows unit suites, the classify, probe-form, mutant-validity, and path-spelling job, and the gap CLI fixture job with its witnesses (done line, refusal, stop, replay). Their continue-on-error is removed. - Non-blocking, stated in the workflow: the Windows static and e2e jobs, which fail on Windows (tracked in the follow-up PR's body by class). - Linux `Test` failed the typecheck on package.test.ts (the tsc lookup's optional capture). Fixed; tsc --noEmit passes locally. Co-authored-by: Isaac --- .github/workflows/pr.yml | 12 ++++-------- packages/oracles/tests/package.test.ts | 8 +++++--- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index ae96eea..d3982d2 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -202,8 +202,6 @@ jobs: windows-unit: name: Windows unit suites runs-on: windows-latest - # Not blocking until every Windows job is green; remove this line then. - continue-on-error: true timeout-minutes: 20 steps: - name: Checkout @@ -220,7 +218,8 @@ jobs: windows-gap-oracles: name: Windows gap report static oracle (${{ matrix.shard }}/6) runs-on: windows-latest - # Not blocking until every Windows job is green; remove this line then. + # Not blocking yet: these suites fail on Windows (tracked in the follow-up + # PR's body, by class). Remove this line when they are green. continue-on-error: true timeout-minutes: 20 strategy: @@ -246,8 +245,6 @@ jobs: name: Windows gap report classify, probe-form, and mutant-validity oracles runs-on: windows-latest timeout-minutes: 20 - # Not blocking until every Windows job is green; remove this line then. - continue-on-error: true steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -262,7 +259,8 @@ jobs: windows-gap-end-to-end: name: Windows gap report end-to-end oracle (fixed seed, ${{ matrix.shard }}/6) runs-on: windows-latest - # Not blocking until every Windows job is green; remove this line then. + # Not blocking yet: these suites fail on Windows (tracked in the follow-up + # PR's body, by class). Remove this line when they are green. continue-on-error: true timeout-minutes: 20 strategy: @@ -286,8 +284,6 @@ jobs: windows-gap-cli: name: Windows gap CLI on a fixture runs-on: windows-latest - # Not blocking until every Windows job is green; remove this line then. - continue-on-error: true timeout-minutes: 20 steps: - name: Checkout diff --git a/packages/oracles/tests/package.test.ts b/packages/oracles/tests/package.test.ts index 13b4751..e91fe1a 100644 --- a/packages/oracles/tests/package.test.ts +++ b/packages/oracles/tests/package.test.ts @@ -297,9 +297,11 @@ console.log(JSON.stringify({ failed, unreached }))`, const tscOf = (directory: string) => { const bin = join(directory, `node_modules/.bin/tsc`) if (process.platform !== `win32`) return readlinkSync(bin) - const target = /\.\.\/(.+?)\/bin\/tsc/.exec(readFileSync(bin, `utf8`)) - if (target === null) throw new Error(`no tsc target in ${bin}`) - return join(`..`, target[1], `bin`, `tsc`) + const target = /\.\.\/(.+?)\/bin\/tsc/.exec( + readFileSync(bin, `utf8`), + )?.[1] + if (target === undefined) throw new Error(`no tsc target in ${bin}`) + return join(`..`, target, `bin`, `tsc`) } const aliasTsc = join( `..`, From 318e95e063ebb2a2b403b3b12c52a8d9320f4412 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 16:39:36 -0700 Subject: [PATCH 19/53] fix(oracle-gaps): the static oracle's temp directory by its native real path (output namespace) The output-namespace law (checkpoint `static`) took the OS temp directory by JavaScript's realpath, which keeps an 8.3 short name (C:\Users\RUNNER~1), while the CLI's default output is under the native real path. So every run's output failed "not under the OS temp directory" on windows-latest (82 hits in run 43525797122). The oracle now uses realpathSync.native, the same rule the CLI uses, and drops the raw tmpdir() alternative. Evidence block - Law: a run's default output is under the OS temporary directory's real path, never in its package (maintainer decision, 2026-11-11; register: the realpath rule). - Reach: the existing static histories (the outputs of three runs per history, two of one package and one of another). - RED: Windows run 43525797122, static shards: 82 violations, law "output namespace", "a run's default output is not under the OS temp directory". - GREEN: pending the next Windows run on this branch. - Mutant (Windows-only): the CLI's temporary directory taken raw (tmpdir() without realpath) puts the output under RUNNER~1, which the fixed law rejects on Windows. On Linux the two spellings coincide, so the Linux run cannot kill this mutant; the Windows run is its witness. - Limits: Linux does not exercise the 8.3 spelling. Co-authored-by: Isaac --- .../oracle-gaps/gap-report-static-oracle.property.test.mjs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs index 97e4a23..99251a4 100644 --- a/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs @@ -227,14 +227,15 @@ function defaultOutputsOf(dir, sources) { const run = (pkg) => parseArgs([`--package`, pkg, `--source`, sources.join(`,`)], pkg).out const outs = [run(dir), run(dir), run(other)] - const temp = realpathSync(tmpdir()) + sep + // The OS temporary directory by its native real path (the realpath rule): + // a JavaScript realpath keeps an 8.3 name such as RUNNER~1 on Windows. + const temp = realpathSync.native(tmpdir()) + sep return { // Under the OS temp directory, which the OS cleans, and never in the // package (maintainer decision, 2026-11-11). contained: outs.map( (out, i) => - (out.startsWith(temp) || out.startsWith(tmpdir() + sep)) && - !out.startsWith([dir, dir, other][i] + sep), + out.startsWith(temp) && !out.startsWith([dir, dir, other][i] + sep), ), // Each package's outputs lie in a directory of their own, which is not // the temp directory itself or a shared `oracle-gaps` directory. From d0ea5af7b86feb90200b07d69f1859ff734e6c3d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 16:40:31 -0700 Subject: [PATCH 20/53] fix(oracle-gaps): the e2e driver names a scratch path in both spellings (baseline lead) The e2e driver replaces scratch paths with and in the CLI's output, but only in the native spelling. The CLI now names its report and package paths in forward slashes (the Windows paths rule), so the done line kept `D:/a/.../.scratch-XXXX/out/summary.md` and the baseline-lead law (18 hits in run 43525797122) read it as not `/out/summary.md`. Evidence block - Law: the CLI's done line names its report in the package's placeholder spelling (law: baseline lead; register: Windows paths). - Reach: the existing e2e histories. - RED: run 43525797122, e2e shards: 18 violations, law "baseline lead". - GREEN: pending the next Windows run on this branch. - Mutant: a CLI that prints the native spelling (backslashes) in the done line is now caught by the same check, since the placeholder is only applied to the spelling the driver names; not run in CI. - Limits: the driver names two spellings of each scratch path (native and forward-slash), not every possible form. Co-authored-by: Isaac --- .../oracles/oracle-gaps/gap-report-oracle.property.test.mjs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 9f0fe4c..40f5d05 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -943,10 +943,13 @@ function namedScratch(text, names) { const longestFirst = names .filter(([path]) => path !== undefined) .sort(([a], [b]) => b.length - a.length) + // The CLI names its paths in forward slashes (the Windows paths rule), so a + // scratch path is named in both spellings. for (const [path, name] of longestFirst) named = named .replaceAll(pathToFileURL(path).href, name) .replaceAll(path, name) + .replaceAll(path.replaceAll(`\\`, `/`), name) return named .replaceAll(/\.oracle-gaps-\d+/g, `.oracle-gaps-`) .replaceAll(/session-\d+/g, `session-`) From 09df6bbfd1d558bc4fdb9976aad7bedff44fcb1d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 16:41:34 -0700 Subject: [PATCH 21/53] fix(oracle-gaps): mutants' owner files are forward-slash in the e2e driver (streaming) The e2e driver's mutant observations named each owner file with path.relative, which returns backslashes on Windows (tests\a.test.ts). The CLI names a heartbeat's file in forward slashes, so every heartbeat failed "names sut.ts:M1 in tests/a.test.ts, which does not run it" (the streaming law, 4 hits in run 43525797122, plus greppable-stream instances). Evidence block - Law: the heartbeat names a test file that runs the mutant (streaming law); keys and report paths are forward-slash (register: Windows paths). - Reach: the existing e2e histories with a slow owner. - RED: run 43525797122, e2e shards: "a heartbeat names ... which does not run it". - GREEN: pending the next Windows run on this branch. - Mutant: not run. - Limits: the stdout gap-line count mismatch (`122 gap lines for 121 gaps`, greppable stream) is not yet explained; left for the next run. Co-authored-by: Isaac --- .../oracle-gaps/gap-report-oracle.property.test.mjs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 40f5d05..6d5a748 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -1157,8 +1157,13 @@ function readRun(dir, out, status, stderr, cli, stdout) { mutants: (results ?? []).map((r) => ({ id: r.id, // The owner files whose tests reached it, relative to the package. - oracleFiles: (r.oracleFiles ?? []).map((file) => relative(dir, file)), - ordinaryFiles: (r.ordinaryFiles ?? []).map((file) => relative(dir, file)), + // Forward slashes, as the CLI names its files (the Windows paths rule). + oracleFiles: (r.oracleFiles ?? []).map((file) => + relative(dir, file).replaceAll(`\\`, `/`), + ), + ordinaryFiles: (r.ordinaryFiles ?? []).map((file) => + relative(dir, file).replaceAll(`\\`, `/`), + ), key: r.key, source: r.source, function: r.function, From 50b4ab5245b70b8b21ed20c6f93466157629a0da Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 17:16:35 -0700 Subject: [PATCH 22/53] fix(windows): stop witness, replay quoting by the shells' grammar, and review items F1-F8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of #58 at ebc39e6 (NOT READY). This commit fixes: - F1 (stop never stops): the witness's stop file is absolute (the CLI's cwd is the package), a stop requires the stopped line on both streams with exit 130, and the stop lands 400 ms after the run starts (the fixture's output is written in one burst near its end, so the first output line is no clock). - F2/F3 (replay quoting): PowerShell reads U+2018 to U+201B as quotes, so every such character is doubled inside single quotes. cmd toggles quoting on every `"` and expands `%`; a name with either has no cmd form, and its cmd line says `use the PowerShell line for this name` (stream.mjs, src/oracle-property.ts). The classify oracle reads each shell's line by that shell's grammar (the PowerShell quote set; cmd's quote toggling, `^`, and `%`), checks the win32 printer on every host, and generates adversarial names (curly quotes, `& | < > ( ) ^ % !`, quotes, backslashes), drawn as the last field of the test record (the fixed seeds' later draws shift; recorded here). - F(f) (POSIX hook): the test-only stop hook lives in the supervisor and goes through its stop, which forwards to the child; a hooked stop gives `done stopped (SIGINT)`, exit 130 (local run: outcome (a)). Unsupervised runs keep the hook in the process. - F5: posix() acts only on win32; on POSIX a backslash is a name's character. - F6: check.mjs realpath of workspace links, report.mjs results path, and ending.mjs error log path are forward-slash and use the native real path. - F7: the blocking Windows jobs are in preview's needs. - F8: the non-blocking jobs point at #72. - The Windows replay witness runs each printed PowerShell line and its cmd line, for two fixtures with hostile names (curly quote and metacharacters; a double quote and `%`). Evidence block - Law: the printed replay line replays in the shell it names (register: deterministic replay across shells; each printed line must replay on Windows); a hooked stop is a stopped line with exit 130 (register: stop window on every platform). - Reach: the classify oracle's 276 generated histories (new adversarial name dimension, drawn last); the Windows replay fixtures; one stop per run. - RED: the parent printer (ebc39e6, with only the platform parameter added) fails the new classify oracle on two names: `"vitest run tests/f0.test.ts -t '^’ t0$'"` (the PowerShell line ends its quote at U+2019) and the cmd line for `q"t` (`which the shell does not read as plain words`). Run locally, exit 1. - GREEN: classify 276/276 on Linux (current code); POSIX stop witness outcome (a), exit 130 (local, under the e2e lock). Windows: pending CI on this head. - Mutant: the parent printer is the mutant (the curly-quote and cmd forms before this commit); the RED run above is its rejection. - Limits: - the stop timing (400 ms after spawn) is an assumption about the fixture's length; a run that ends first is reported as a failure, not passed; - cmd's `!` is literal (delayed expansion off), an assumption; - the classify fixed seeds' later draws shift (the new field is drawn last); - the win32 argv cap, the pipe name's machine scope, the POSIX state paths moving to the temporary directory's real path (an older daemon is found only after its idle timeout), and the unsupervised win32 process-tree exemption (#74) are recorded in the PR body. Co-authored-by: Isaac --- .github/workflows/pr.yml | 44 +++++- packages/oracles/oracle-gaps/check.mjs | 3 +- packages/oracles/oracle-gaps/ending.mjs | 4 +- ...p-report-classify-oracle.property.test.mjs | 137 ++++++++++++------ packages/oracles/oracle-gaps/index.mjs | 13 +- packages/oracles/oracle-gaps/paths.mjs | 10 +- packages/oracles/oracle-gaps/report.mjs | 4 +- packages/oracles/oracle-gaps/stream.mjs | 55 ++++--- packages/oracles/scripts/windows-witness.mjs | 107 ++++++++------ packages/oracles/src/oracle-property.ts | 29 +++- 10 files changed, 276 insertions(+), 130 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index d3982d2..2a81a5c 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -168,6 +168,11 @@ jobs: - test-oracle-gaps-fast - test-oracle-gaps-end-to-end - test-vitest-5 + # The blocking Windows jobs (the non-blocking static and e2e jobs are not + # a dependency until they are green). + - windows-unit + - windows-gap-classify + - windows-gap-cli runs-on: ubuntu-latest steps: - name: Checkout @@ -218,8 +223,9 @@ jobs: windows-gap-oracles: name: Windows gap report static oracle (${{ matrix.shard }}/6) runs-on: windows-latest - # Not blocking yet: these suites fail on Windows (tracked in the follow-up - # PR's body, by class). Remove this line when they are green. + # Not blocking yet: these suites fail on Windows, tracked by class in + # https://github.com/TanStack/oracles/issues/72. Remove this line when they + # are green. continue-on-error: true timeout-minutes: 20 strategy: @@ -259,8 +265,9 @@ jobs: windows-gap-end-to-end: name: Windows gap report end-to-end oracle (fixed seed, ${{ matrix.shard }}/6) runs-on: windows-latest - # Not blocking yet: these suites fail on Windows (tracked in the follow-up - # PR's body, by class). Remove this line when they are green. + # Not blocking yet: these suites fail on Windows, tracked by class in + # https://github.com/TanStack/oracles/issues/72. Remove this line when they + # are green. continue-on-error: true timeout-minutes: 20 strategy: @@ -356,7 +363,34 @@ jobs: import { differs, oracleProperty, pass } from '../../packages/oracles/dist/esm/index.js' oracleProperty({ - id: `fixture.replay`, + id: `fixture ’ & | < > ( ) ^ ! replay`, + law: { + statement: `a fixture that always fails, so its replay lines are printed.`, + authority: `the CI witness for the per-shell replay lines`, + limits: `integers from -20 to 20`, + }, + histories: { + arbitrary: fc.integer({ min: -20, max: 20 }), + runs: 20, + fixedSeeds: [1], + states: [], + measured: [], + }, + timeout: 5_000, + model: (x: number) => x, + driver: (x: number, observe: (checkpoint: string, value: unknown) => void) => { + observe(`result`, x + 1) + }, + check: (_x: number, trace: ReadonlyArray<{ checkpoint: string; value: unknown }>, expected: number) => + differs(`result`, `plus one`, trace.find(({ checkpoint }) => checkpoint === `result`)?.value, expected) ?? pass, + }) + EOT + cat > "$dir/tests/hostile.property.test.ts" <<'EOT' + import fc from 'fast-check' + import { differs, oracleProperty, pass } from '../../packages/oracles/dist/esm/index.js' + + oracleProperty({ + id: `fixture "quoted" 100% replay`, law: { statement: `a fixture that always fails, so its replay lines are printed.`, authority: `the CI witness for the per-shell replay lines`, diff --git a/packages/oracles/oracle-gaps/check.mjs b/packages/oracles/oracle-gaps/check.mjs index aa4fd19..9ca8b5b 100644 --- a/packages/oracles/oracle-gaps/check.mjs +++ b/packages/oracles/oracle-gaps/check.mjs @@ -67,6 +67,7 @@ import { baseOf, isWithin, posix, + realPathOf, relativeWithin, specifierOf, } from './paths.mjs' @@ -259,7 +260,7 @@ export async function workspaceBundles(pkg) { if (linked.has(name)) continue const link = join(dir, `node_modules`, name) if (!existsSync(link)) continue - const real = realpathSync(link) + const real = realPathOf(link) if (real.includes(`/node_modules/`) || real === pkg) continue linked.set(name, { dir: real, from: dir }) queue.push(real) diff --git a/packages/oracles/oracle-gaps/ending.mjs b/packages/oracles/oracle-gaps/ending.mjs index 23d279a..ff58f9b 100644 --- a/packages/oracles/oracle-gaps/ending.mjs +++ b/packages/oracles/oracle-gaps/ending.mjs @@ -54,10 +54,10 @@ export function endRun(error, out = error?.out) { log = join(realPathOf(tmpdir()), `oracle-gaps-error-${process.pid}.log`) writeFileSync(log, stack) } - const done = `oracle-gaps: done error, log ${log}` + const done = `oracle-gaps: done error, log ${posix(log)}` return { exitCode: 3, - stderr: `oracle-gaps internal error: ${message}\nThis is a bug in oracle-gaps, not in your oracle. Its stack is in ${log}.\n${done}\n`, + stderr: `oracle-gaps internal error: ${message}\nThis is a bug in oracle-gaps, not in your oracle. Its stack is in ${posix(log)}.\n${done}\n`, done, } } diff --git a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs index 6e2402a..fb8af3b 100644 --- a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs @@ -510,6 +510,21 @@ const testRecord = fc.record({ }, ), ms: fc.nat({ max: 40 }), + // Names a Windows shell reads as quotes, metacharacters, or expansions (the + // replay line must replay in PowerShell and cmd, or say it cannot). Drawn + // last, so the fixed seeds keep their other draws. + hostile: fc.constantFrom( + ``, + ``, + ``, + `\u2019 `, + `\u201C\u201D `, + `& | < > `, + `( ) ^ `, + `% ! `, + `use "x & y `, + `back\\ `, + ), }) const records = fc.record({ files: fc.array( @@ -614,7 +629,7 @@ const byPath = (h) => (a, b) => // distinct. Its full name joins them with ` > `. const namesOf = (h, fi, ti) => { const t = h.files[fi].tests[ti] - return [...t.suites, `${t.label}t${ti}${t.tail}`] + return [...t.suites, `${t.label}${t.hostile ?? ``}t${ti}${t.tail}`] } const nameOf = (h, fi, ti) => namesOf(h, fi, ti).join(` > `) const keyOf = (h, fi, ti) => `${fileName(h, fi)} > ${nameOf(h, fi, ti)}` @@ -2128,6 +2143,10 @@ async function driver(h, observe) { }) // The baseline the printer shows before any mutant (law 9). observe(`baseline`, printer?.baselineLines(baselineGiven) ?? []) + observe( + `baseline-win32`, + printer?.baselineLines(baselineGiven, undefined, `win32`) ?? [], + ) for (const r of results) { states.add( r.authored ? `authored ${r.class ?? r.outcome}` : (r.class ?? r.outcome), @@ -2627,7 +2646,20 @@ function check(h, observations, expected) { // replay command, read as a POSIX shell reads it, which runs exactly the // owed test of the file under either Vitest's join of its names. function baselineBreach(h, observations, expected) { - const lines = observations.find((o) => o.checkpoint === `baseline`)?.value + // The host's baseline, and the baseline the printer gives for win32: the + // Windows lines are judged on every host (review F2, F3). + for (const [checkpoint, platform] of [ + [`baseline`, process.platform], + [`baseline-win32`, `win32`], + ]) { + const lines = observations.find((o) => o.checkpoint === checkpoint)?.value + const breach = printedBreach(h, lines, expected, platform) + if (breach) return breach + } + return undefined +} + +function printedBreach(h, lines, expected, platform) { if (!Array.isArray(lines)) return violation(`baseline`, `no baseline was printed`, `baseline`) const breach = (difference) => @@ -2705,12 +2737,26 @@ function baselineBreach(h, observations, expected) { } // The platform's replay lines: one POSIX line, or on Windows one line for // PowerShell and one for cmd. Each reads as the same words. - const replays = replayLinesOf(block) + const replays = replayLinesOf(block, platform) if (replays === undefined) return breach( `${file}'s replay lines are not the platform's: ${JSON.stringify(block.filter((line) => line.startsWith(` replay`)))}`, ) for (const { command, read } of replays) { + // cmd has no form for a name with a double quote or `%`: its line says so, + // and only such a test may print that line. + if (command === CMD_FALLBACK) { + // The pattern the PowerShell line runs (the names carry the labels). + const psLine = replays.find((r) => r.read === powerShellWords) + const pattern = psLine + ? powerShellWords(psLine.command)?.[4] + : undefined + if (pattern === undefined || !/["%]/.test(pattern)) + return breach( + `the replay of ${file} says cmd cannot replay it, but no name in it needs that`, + ) + continue + } const words = read(command) if (!words) return breach( @@ -2756,9 +2802,9 @@ function baselineBreach(h, observations, expected) { // The replay lines the platform prints, each with the reader of its shell: // one POSIX line elsewhere; on Windows a PowerShell line and a cmd line. // Undefined when the block holds a different set of forms. -function replayLinesOf(block) { +function replayLinesOf(block, platform) { const forms = - process.platform === `win32` + platform === `win32` ? [ [` replay (PowerShell): `, powerShellWords], [` replay (cmd): `, cmdWords], @@ -2775,8 +2821,13 @@ function replayLinesOf(block) { return found.every(Boolean) ? found : undefined } -// The words a PowerShell command line splits into: single quotes hold text, -// a doubled single quote is one quote, and nothing expands. +// The words a PowerShell command line splits into, as PowerShell reads it: a +// word is plain characters or single-quoted text. The ASCII single quote and +// U+2018 to U+201B each open and close a quote; inside one, a doubled quote +// character is that character. Anything else outside quotes is not a word the +// printer makes. +const CMD_FALLBACK = `use the PowerShell line for this name` +const PS_QUOTES = [`'`, `‘`, `’`, `‚`, `‛`] function powerShellWords(line) { const words = [] let word @@ -2789,66 +2840,67 @@ function powerShellWords(line) { continue } word ??= `` - if (c === `'`) { + if (PS_QUOTES.includes(c)) { let j = i + 1 for (;;) { if (j >= line.length) return undefined - if (line[j] === `'` && line[j + 1] === `'`) { - word += `'` - j += 2 - } else if (line[j] === `'`) break - else { - word += line[j] - j += 1 + if (PS_QUOTES.includes(line[j])) { + if (line[j + 1] === line[j]) { + word += line[j] + j += 2 + continue + } + break } + word += line[j] + j += 1 } i = j + 1 - } else if (/[$"()\`;&|<>]/.test(c)) { - return undefined - } else { + } else if (/[\w./-]/.test(c)) { word += c i += 1 - } + } else return undefined } if (word !== undefined) words.push(word) return words } -// The words a cmd command line splits into: double quotes hold text, and -// nothing expands or chains. +// The words a cmd command line splits into, as cmd reads it under `/d /s /c`: +// a double quote toggles quoting; outside quotes `^` escapes the next +// character, and `& | < > ( )` separate or redirect; `%` expands even inside +// quotes. `!` is literal, since delayed expansion is off. Undefined otherwise. function cmdWords(line) { const words = [] let word + let quoted = false for (let i = 0; i < line.length;) { const c = line[i] + if (c === `%`) return undefined + if (quoted) { + if (c === `"`) quoted = false + else word = (word ?? ``) + c + i += 1 + continue + } if (c === ` `) { if (word !== undefined) words.push(word) word = undefined i += 1 - continue - } - word ??= `` - if (c === `"`) { - // Inside double quotes, `\"` is a quote, as the program's argument parser - // reads it (Node's on Windows); cmd itself does not escape it. - let text = `` - let j = i + 1 - for (; j < line.length && line[j] !== `"`; j += 1) { - if (line[j] === `\\` && line[j + 1] === `"`) { - text += `"` - j += 1 - } else text += line[j] - } - if (j >= line.length) return undefined - word += text - i = j + 1 - } else if (/[%&|<>^]/.test(c)) { - return undefined - } else { - word += c + } else if (c === `"`) { + quoted = true + word ??= `` + i += 1 + } else if (c === `^`) { + if (i + 1 >= line.length) return undefined + word = (word ?? ``) + line[i + 1] + i += 2 + } else if (/[&|<>()]/.test(c)) return undefined + else { + word = (word ?? ``) + c i += 1 } } + if (quoted) return undefined if (word !== undefined) words.push(word) return words } @@ -3179,6 +3231,7 @@ const plainTest = (effect = {}, test = {}) => ({ suites: [], label: ``, tail: ``, + hostile: ``, fails: false, runOrder: 0, work: 10, diff --git a/packages/oracles/oracle-gaps/index.mjs b/packages/oracles/oracle-gaps/index.mjs index d55502d..4321082 100755 --- a/packages/oracles/oracle-gaps/index.mjs +++ b/packages/oracles/oracle-gaps/index.mjs @@ -222,6 +222,15 @@ function superviseRun() { // starts is a stop too (re-review N3). process.on(`SIGINT`, stop) process.on(`SIGTERM`, stop) + // The test-only stop hook (ORACLE_GAPS_TEST_STOP, review F(f)): in a + // supervised run it goes through this supervisor's stop, which forwards to + // the child and writes the stopped line, as a SIGINT does. + if (process.env.ORACLE_GAPS_TEST_STOP !== undefined) { + const file = process.env.ORACLE_GAPS_TEST_STOP + setInterval(() => { + if (existsSync(file)) stop(`SIGINT`) + }, 50).unref() + } // An inspector flag would bind the CLI's own port a second time. child = spawn( process.execPath, @@ -363,7 +372,9 @@ else if (main) { // ORACLE_GAPS_TEST_HOLD_EXIT_MS): when the file ORACLE_GAPS_TEST_STOP names // appears, the CLI takes the same stop as a SIGINT. CI's Windows witness uses // it, since a non-console parent cannot deliver Ctrl+C there. - if (process.env.ORACLE_GAPS_TEST_STOP !== undefined) { + // Unsupervised runs only (win32, no supervisor); a supervised child's stop + // is the supervisor's. + if (process.env.ORACLE_GAPS_TEST_STOP !== undefined && !supervised) { const file = process.env.ORACLE_GAPS_TEST_STOP setInterval(() => { if (existsSync(file)) onStop(`SIGINT`) diff --git a/packages/oracles/oracle-gaps/paths.mjs b/packages/oracles/oracle-gaps/paths.mjs index 27711dd..e12eaaa 100644 --- a/packages/oracles/oracle-gaps/paths.mjs +++ b/packages/oracles/oracle-gaps/paths.mjs @@ -11,10 +11,16 @@ import { relative } from 'node:path' /** * The forward-slash spelling of `path`, with a drive letter in capitals: a - * Windows drive's letter is one letter whatever its case. + * Windows drive's letter is one letter whatever its case. Only Windows has a + * `\` separator: on POSIX a backslash is a name's character (a directory + * named `a\b` is one name), so the path is left as it is (review F5). */ export const posix = (path) => - path.replaceAll(`\\`, `/`).replace(/^[a-z]:/, (drive) => drive.toUpperCase()) + process.platform !== `win32` + ? path + : path + .replaceAll(`\\`, `/`) + .replace(/^[a-z]:/, (drive) => drive.toUpperCase()) /** The real path of `path`, in forward slashes. */ export const realPathOf = (path) => posix(realpathSync.native(path)) diff --git a/packages/oracles/oracle-gaps/report.mjs b/packages/oracles/oracle-gaps/report.mjs index f9b0390..34858c4 100644 --- a/packages/oracles/oracle-gaps/report.mjs +++ b/packages/oracles/oracle-gaps/report.mjs @@ -30,7 +30,7 @@ import { streamLine, verdictLine, } from './stream.mjs' -import { relativeWithin } from './paths.mjs' +import { posix, relativeWithin } from './paths.mjs' /** Outcomes that are gaps. Only a law verdict on a reproducible route kills. */ export const gapOutcomes = new Set([ @@ -468,7 +468,7 @@ export function renderStdout({ `left out, failing with no mutant active: ${excluded.join(`, `)}`, ), ) - lines.push(streamLine(`note`, `results ${paths.results}`)) + lines.push(streamLine(`note`, `results ${posix(paths.results)}`)) // The last line says the check ran to its end, how many gaps it found, // and where its full report is (G6, and the stream's done line): exit 0 // means the run completed, not that it found none. diff --git a/packages/oracles/oracle-gaps/stream.mjs b/packages/oracles/oracle-gaps/stream.mjs index 79db349..a2b07cd 100644 --- a/packages/oracles/oracle-gaps/stream.mjs +++ b/packages/oracles/oracle-gaps/stream.mjs @@ -77,21 +77,33 @@ export function testNamesOf(testCase) { // A word that a POSIX shell reads back as it is: plain, or in single quotes. const shellWord = (word) => /^[\w./-]+$/.test(word) ? word : `'${word.replaceAll(`'`, `'\\''`)}'` -// The same word for PowerShell (a single quote is doubled) and for cmd (a -// double-quoted word, which cmd reads as it is). +// PowerShell reads these as single quotes: the ASCII one and the typographic +// ones (U+2018 to U+201B). Inside a single-quoted word each is doubled. +const PS_SINGLE_QUOTES = /['\u2018\u2019\u201A\u201B]/g const powerShellWord = (word) => - /^[\w./-]+$/.test(word) ? word : `'${word.replaceAll(`'`, `''`)}'` + /^[\w./-]+$/.test(word) + ? word + : `'${word.replace(PS_SINGLE_QUOTES, (quote) => quote + quote)}'` +// cmd toggles its quoting on every double quote, and expands `%`, even inside +// quotes; a name with either has no cmd form (undefined), so the line says so. +// Inside double quotes cmd reads the other metacharacters literally. const cmdWord = (word) => - /^[\w./-]+$/.test(word) ? word : `"${word.replaceAll(`"`, `\\"`)}"` + /["%]/.test(word) ? undefined : /^[\w./-]+$/.test(word) ? word : `"${word}"` // The replay line of a command. A POSIX shell reads one line; on Windows // there are two, one for PowerShell and one for cmd, since the words differ. -const replayLines = (words) => - process.platform === `win32` - ? [ - ` replay (PowerShell): ${words.map(powerShellWord).join(` `)}`, - ` replay (cmd): ${words.map(cmdWord).join(` `)}`, - ] - : [` replay: ${words.map(shellWord).join(` `)}`] +// A cmd line with a word that has no cmd form says so instead of printing a +// line that would not replay (the PowerShell line still does). +export const replayLines = (words, platform = process.platform) => { + if (platform !== `win32`) + return [` replay: ${words.map(shellWord).join(` `)}`] + const cmd = words.map(cmdWord) + return [ + ` replay (PowerShell): ${words.map(powerShellWord).join(` `)}`, + cmd.every((word) => word !== undefined) + ? ` replay (cmd): ${cmd.join(` `)}` + : ` replay (cmd): use the PowerShell line for this name`, + ] +} // A control character, or a line or paragraph separator, as its `\uXXXX` // escape: the line stays one line, and the escape is regular-expression @@ -153,7 +165,7 @@ function violationLines(message) { * the config `--config` named, relative to the package: Vitest would * discover another config or none, so the replay names it too. */ -export function baselineLines(files, config) { +export function baselineLines(files, config, platform = process.platform) { const vitest = config === undefined ? [`vitest`, `run`] @@ -179,7 +191,7 @@ export function baselineLines(files, config) { `baseline`, `fail ${name}: ${tests.length === 0 ? `did not load` : `failed outside its tests`}: ${codeSpan(escapeControls(failure.split(`\n`)[0]))}`, ), - ...replayLines([...vitest, file]), + ...replayLines([...vitest, file], platform), ] const message = first.message ?? `` return [ @@ -189,13 +201,16 @@ export function baselineLines(files, config) { ), ` failing: ${codeSpan(escapeControls((first.names ?? [first.fullName]).join(` > `)))}: ${codeSpan(escapeControls(message.split(`\n`)[0]))}`, ...violationLines(message), - ...replayLines([ - ...vitest, - file, - ...(first.names === undefined - ? [] - : [`-t`, escapeControls(replayPattern(first.names))]), - ]), + ...replayLines( + [ + ...vitest, + file, + ...(first.names === undefined + ? [] + : [`-t`, escapeControls(replayPattern(first.names))]), + ], + platform, + ), ] }) } diff --git a/packages/oracles/scripts/windows-witness.mjs b/packages/oracles/scripts/windows-witness.mjs index d0d2441..7c45e79 100644 --- a/packages/oracles/scripts/windows-witness.mjs +++ b/packages/oracles/scripts/windows-witness.mjs @@ -63,6 +63,20 @@ function checkDone(stream, text, exitCode) { return ok ? undefined : `${stream}: done ${status} but exit code ${exitCode}` } +// The stop's own line (law #57): one done line, last, `done stopped (SIGINT)`, +// and exit 130. +function checkStop(stream, text, exitCode) { + const lines = text.split(/\r?\n/).filter((line) => line !== ``) + const done = doneLinesOf(text) + if (done.length !== 1) + return `${stream}: ${done.length} done lines, want exactly 1` + if (done[0] !== `oracle-gaps: done stopped (SIGINT)`) + return `${stream}: the done line is ${JSON.stringify(done[0])}, not a SIGINT stop` + if (lines.at(-1) !== done[0]) + return `${stream}: the done line is not the last line` + return exitCode === 130 ? undefined : `${stream}: exit ${exitCode}, want 130` +} + if (command === `done`) { const [stdoutFile, stderrFile, code] = args const exitCode = Number(code) @@ -73,19 +87,18 @@ if (command === `done`) { if (problems.length > 0) fail(problems.join('; ')) else console.log(`done line ok on both streams, exit ${exitCode}`) } else if (command === `replay`) { + // Every failing property in the output prints a PowerShell line and a cmd + // line (or says cmd has no form for its name). Each pair is run in its own + // shell, and each must reproduce the failure. The POSIX line is not run. const [outputFile, dir] = args const output = readFileSync(outputFile, `utf8`).replace(/\x1b\[[0-9;]*m/g, ``) - // The text after a label on the first line that carries it, wherever the - // error text indents it. - const lineOf = (label) => { - const line = output.split(/\r?\n/).find((text) => text.includes(label)) - return line === undefined - ? undefined - : line.slice(line.indexOf(label) + label.length) - } - const powershell = lineOf(`Replay (PowerShell): `) - const cmd = lineOf(`Replay (cmd): `) - const posix = lineOf(`Replay: `) + const lines = output.split(/\r?\n/) + const after = (label) => + lines + .filter((text) => text.includes(label)) + .map((text) => text.slice(text.indexOf(label) + label.length)) + const powershell = after(`Replay (PowerShell): `) + const cmd = after(`Replay (cmd): `) // The shells find the test runner in this package's own binaries. const env = { ...process.env, @@ -112,36 +125,29 @@ if (command === `done`) { const reproduces = (result) => result.status !== 0 && /violated|failed/.test(`${result.stdout}${result.stderr}`) - if (powershell === undefined || cmd === undefined) { - fail( - `the failure printed no per-shell replay lines (PowerShell: ${powershell === undefined ? `missing` : `ok`}, cmd: ${cmd === undefined ? `missing` : `ok`})`, - ) + if (powershell.length === 0) { + fail(`the failure printed no PowerShell replay line`) + } else if (powershell.length !== cmd.length) { + fail(`${powershell.length} PowerShell lines but ${cmd.length} cmd lines`) } else { - const psRun = runIn(`pwsh`, powershell) - const cmdRun = runIn(`cmd`, cmd) - if (!reproduces(psRun)) - fail( - `the PowerShell line did not reproduce the failure: ${psRun.stderr || psRun.stdout}`.slice( - 0, - 800, - ), - ) - else if (!reproduces(cmdRun)) - fail( - `the cmd line did not reproduce the failure: ${cmdRun.stderr || cmdRun.stdout}`.slice( - 0, - 800, - ), - ) - else console.log(`PowerShell and cmd lines each reproduce the failure`) - } - if (posix !== undefined) { - const posixRun = runIn(`pwsh`, posix) - if (reproduces(posixRun)) - console.log(`note: the POSIX line also reproduces in PowerShell`) + const failures = [] + powershell.forEach((psLine, i) => { + const psRun = runIn(`pwsh`, psLine) + if (!reproduces(psRun)) + failures.push(`PowerShell line ${i + 1} did not reproduce: ${psLine}`) + const cmdLine = cmd[i] + if (cmdLine === `use the PowerShell line for this name`) { + console.log(`cmd has no form for name ${i + 1}, as printed`) + return + } + const cmdRun = runIn(`cmd`, cmdLine) + if (!reproduces(cmdRun)) + failures.push(`cmd line ${i + 1} did not reproduce: ${cmdLine}`) + }) + if (failures.length > 0) fail(failures.join('; ').slice(0, 1500)) else console.log( - `RED: the POSIX line does not reproduce in PowerShell (exit ${posixRun.status})`, + `${powershell.length} printed name(s): PowerShell and cmd lines each reproduce`, ) } } else if (command === `stop`) { @@ -150,7 +156,9 @@ if (command === `done`) { // Ctrl+C from a non-console parent, and child.kill('SIGINT') there is // TerminateProcess, a limit like SIGKILL, so no line is owed for it. const [dir, ...cliArgs] = args - const stopFile = join(dir, `.stop-requested`) + // Absolute: the CLI runs with `dir` as its cwd, so a relative name is read + // from `dir/dir/...` and the stop never lands (review F1). + const stopFile = resolve(dir, `.stop-requested`) rmSync(stopFile, { force: true }) const child = spawn(process.execPath, [cli, ...cliArgs], { cwd: dir, @@ -159,28 +167,33 @@ if (command === `done`) { }) let stdout = `` let stderr = `` + // The stop lands 400 ms after the run starts: the fixture's whole run lasts + // about a second, and the CLI's output is written in one burst near its end, + // so the first output line is no clock. A run that ends first is a failure. let requested = false + const stopTimer = setTimeout(() => { + requested = true + writeFileSync(stopFile, ``) + }, 400) child.stdout.on(`data`, (chunk) => { stdout += chunk - // The run has begun once it prints its first line: the stop lands then. - if (!requested && stdout.length > 0) { - requested = true - writeFileSync(stopFile, ``) - } }) child.stderr.on(`data`, (chunk) => { stderr += chunk }) child.on(`close`, (code, signal) => { + clearTimeout(stopTimer) if (!requested) { fail( - `the run ended before its first line (exit ${code}, signal ${signal}), so no stop was sent`, + `the run ended before the stop landed (exit ${code}, signal ${signal})`, ) return } + // A stop must end with its stopped line on both streams, and exit 130 + // (SIGINT). A completed run is no stop, however its done line reads. const problems = [ - checkDone(`stdout`, stdout, code ?? -1), - checkDone(`stderr`, stderr, code ?? -1), + checkStop(`stdout`, stdout, code), + checkStop(`stderr`, stderr, code), ].filter(Boolean) if (problems.length > 0) fail( diff --git a/packages/oracles/src/oracle-property.ts b/packages/oracles/src/oracle-property.ts index 4054df5..cca614b 100644 --- a/packages/oracles/src/oracle-property.ts +++ b/packages/oracles/src/oracle-property.ts @@ -2266,7 +2266,7 @@ function rerunOf(task: NamedTask): Command { interface Command { readonly posix: string readonly powershell: string - readonly cmd: string + readonly cmd: string | undefined } /** @@ -2278,8 +2278,16 @@ function commandOf( command: string | undefined, pattern: string, ): Command { - const psQuoted = (text: string) => `'${text.replace(/'/g, `''`)}'` - const cmdQuoted = (text: string) => `"${text.replace(/"/g, `\\"`)}"` + // PowerShell reads the ASCII single quote and U+2018 to U+201B as quotes: + // inside a single-quoted word each is doubled. + const psQuoted = (text: string) => + `'${text.replace(/['\u2018\u2019\u201A\u201B]/g, (quote) => quote + quote)}'` + // cmd toggles quoting on every double quote and expands `%`: a text with + // either has no cmd form (undefined), and its line says so. + const cmdQuoted = (text: string) => + /["%]/.test(text) ? undefined : `"${text}"` + const cmdWords = (...parts: ReadonlyArray) => + parts.some((part) => part === undefined) ? undefined : words(parts) const words = (parts: ReadonlyArray) => parts.filter((part) => part !== undefined && part !== ``).join(` `) return { @@ -2296,11 +2304,14 @@ function commandOf( command, `-t ${psQuoted(pattern)}`, ]), - cmd: words([ - ...env.map(([name, value]) => `set "${name}=${value}" &&`), + cmd: cmdWords( + ...env.map(([name, value]) => { + const quoted = cmdQuoted(value) + return quoted === undefined ? undefined : `set "${name}=${value}" &&` + }), command, - `-t ${cmdQuoted(pattern)}`, - ]), + cmdQuoted(pattern) === undefined ? undefined : `-t ${cmdQuoted(pattern)}`, + ), } } @@ -2313,7 +2324,9 @@ function commandLines( return process.platform === `win32` ? [ `${label} (PowerShell): ${command.powershell}`, - `${label} (cmd): ${command.cmd}`, + command.cmd === undefined + ? `${label} (cmd): use the PowerShell line for this name` + : `${label} (cmd): ${command.cmd}`, ] : [`${label}: ${command.posix}`] } From b904d57a55da6b2f606ab419a250f471fcc5711b Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 17:38:50 -0700 Subject: [PATCH 23/53] fix(oracle-gaps): path-spelling oracle generates win32 separators and drives only on win32 The Linux run of 50b4ab5 failed gap-report-path-spelling: posix() acts only on win32 (review F5), so on POSIX a backslash is a name's character and a drive letter's case is a different name. The oracle now generates backslash joints and mixed drive letters only on win32; on POSIX it generates `/` joints and `C`. This narrows the law on POSIX (recorded): the POSIX law is the forward-slash spelling, and the separator and drive-case laws are judged on win32 (the Windows classify job runs the oracle there). Evidence block - Law: a path's separators and a drive's case change nothing the gap CLI decides on win32; on POSIX they are name characters (paths.mjs, F5). - Reach: the same 200 histories, with the separator and drive alphabets by platform. - RED: Linux shard 3 of 50b4ab5: "observed value differs" on the path-spelling oracle (2 of 5 tests). - GREEN: local POSIX 5/5. - Mutant: the win32 mutants run in the Windows classify job; not re-run on POSIX. - Limits: on POSIX the separator and drive laws are not generated. Co-authored-by: Isaac --- ...-report-path-spelling-oracle.property.test.mjs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-path-spelling-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-path-spelling-oracle.property.test.mjs index 89d1a8d..b1968f1 100644 --- a/packages/oracles/oracle-gaps/gap-report-path-spelling-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-path-spelling-oracle.property.test.mjs @@ -69,12 +69,21 @@ const NAMES = [`src`, `RUNNER~1`, `my dir`, `pkg`, `lib`, `a.js`, `b.ts`] const historyArbitrary = fc.record({ text: fc.integer({ min: 0, max: TEXTS.length - 1 }), // The directory's drive letter may differ in case from the file's. - drive: fc.constantFrom(`C`, `c`, `D`), - dirDrive: fc.constantFrom(`C`, `c`), + drive: fc.constantFrom( + ...(process.platform === `win32` ? [`C`, `c`, `D`] : [`C`]), + ), + dirDrive: fc.constantFrom( + ...(process.platform === `win32` ? [`C`, `c`] : [`C`]), + ), root: fc.array(fc.constantFrom(...NAMES), { minLength: 1, maxLength: 3 }), rest: fc.array(fc.constantFrom(...NAMES), { minLength: 1, maxLength: 2 }), // One separator for each joint: the drive to the first name, and each name. - joints: fc.array(fc.constantFrom(`/`, `\\`), { minLength: 8, maxLength: 8 }), + // On win32 a backslash separates; on POSIX it is a name's character, so only + // `/` joins there (a narrowing: the POSIX law is the forward-slash spelling). + joints: fc.array( + fc.constantFrom(...(process.platform === `win32` ? [`/`, `\\`] : [`/`])), + { minLength: 8, maxLength: 8 }, + ), }) /** The segments of a history's spelling: the drive, then the names. */ From de8a3733d8fdbbfde53e2ff10c4cd99254118a6b Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 18:05:34 -0700 Subject: [PATCH 24/53] fix(oracle-gaps): event-triggered stop, derived hostile names, and the POSIX F5 witness Coordinator rulings, applied: 1. Stop trigger: an event, not a clock. The witness writes the stop file on the CLI's first stderr line. Measured (2026-11-14): on a pipe the CLI streams its progress on stderr and prints the done summary (stdout) only at the end, so stdout is no start marker. No stream.log exists on this branch; the stderr line is the event. The 400 ms timer is gone. 2. Classify generator: the hostile names are derived from the test's own fields (HOSTILE[(ms+ti) % 7] for one test in four), not drawn. A drawn field would shift every later draw of the fixed seeds. Proof: `git diff origin/main` on the classify oracle changes no `fc.` line (zero), so the generator is unchanged. 3. Path spelling on POSIX: the oracle generates backslash joints and `x:` drives on every platform. The model splits the spelled strings per platform (win32: both separators, case-insensitive drive; POSIX: `/` only, a backslash is a name's character). On POSIX a disk witness creates the backslash name and asserts it exists, realPathOf keeps it native, and posix() leaves it unchanged (F5). This removes the earlier POSIX narrowing. Evidence block - Law: the stop is delivered at a run event and ends stopped (register: stop window on every platform); a backslash name on POSIX is kept native (review F5). - Reach: the path-spelling oracle's 200 histories, now with backslash and `x:` names on POSIX (disk-backed); the classify oracle's existing histories, with hostile names on one test in four. - RED: ebc39e6 with this oracle: 2 of 5 tests fail, "the observed value differs from the owed value" (the relation and disk checks). - GREEN: POSIX local 5/5 (path-spelling); stop witness local POSIX outcome (a), exit 130 (event trigger). Windows: pending CI on this head. - Mutant: the parent (ebc39e6) is the mutant for F5 (posix() rewrites backslashes on POSIX); its RED is above. - Limits: the stdout summary arrives only at the end on pipes (a finding, not a fix here); the stop trigger assumes the CLI prints a stderr line before its work completes (measured on the fixture only). Co-authored-by: Isaac --- ...p-report-classify-oracle.property.test.mjs | 35 ++++---- ...ort-path-spelling-oracle.property.test.mjs | 87 ++++++++++++++----- packages/oracles/scripts/windows-witness.mjs | 16 ++-- 3 files changed, 89 insertions(+), 49 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs index fb8af3b..6bcdd0a 100644 --- a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs @@ -510,21 +510,6 @@ const testRecord = fc.record({ }, ), ms: fc.nat({ max: 40 }), - // Names a Windows shell reads as quotes, metacharacters, or expansions (the - // replay line must replay in PowerShell and cmd, or say it cannot). Drawn - // last, so the fixed seeds keep their other draws. - hostile: fc.constantFrom( - ``, - ``, - ``, - `\u2019 `, - `\u201C\u201D `, - `& | < > `, - `( ) ^ `, - `% ! `, - `use "x & y `, - `back\\ `, - ), }) const records = fc.record({ files: fc.array( @@ -624,12 +609,29 @@ const fileName = (h, fi) => `/pkg/${relativeName(h, fi)}` // units, as JavaScript's `<` compares strings. const byPath = (h) => (a, b) => fileName(h, a) < fileName(h, b) ? -1 : fileName(h, a) > fileName(h, b) ? 1 : 0 +// Names a Windows shell reads as quotes, metacharacters, or expansions (the +// replay line must replay in PowerShell and cmd, or say it cannot). Derived +// from the test's own fields, not drawn: a draw would shift every later draw of +// the fixed seeds, so the generator stays as it was. +const HOSTILE = [ + `\u2019 `, + `\u201C\u201D `, + `& | < > `, + `( ) ^ `, + `% ! `, + `use "x & y `, + `back\\ `, +] +const hostileOf = (t, ti) => { + const pick = (t.ms + ti) % 4 === 0 ? (t.ms + ti) % HOSTILE.length : -1 + return pick < 0 ? `` : HOSTILE[pick] +} // A test's names, as the runner reports them: its suites' names, then its // own. Its own name holds its index, so the names within a file are // distinct. Its full name joins them with ` > `. const namesOf = (h, fi, ti) => { const t = h.files[fi].tests[ti] - return [...t.suites, `${t.label}${t.hostile ?? ``}t${ti}${t.tail}`] + return [...t.suites, `${t.label}${hostileOf(t, ti)}t${ti}${t.tail}`] } const nameOf = (h, fi, ti) => namesOf(h, fi, ti).join(` > `) const keyOf = (h, fi, ti) => `${fileName(h, fi)} > ${nameOf(h, fi, ti)}` @@ -3231,7 +3233,6 @@ const plainTest = (effect = {}, test = {}) => ({ suites: [], label: ``, tail: ``, - hostile: ``, fails: false, runOrder: 0, work: 10, diff --git a/packages/oracles/oracle-gaps/gap-report-path-spelling-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-path-spelling-oracle.property.test.mjs index b1968f1..5113cdb 100644 --- a/packages/oracles/oracle-gaps/gap-report-path-spelling-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-path-spelling-oracle.property.test.mjs @@ -43,12 +43,20 @@ * needs a real file (realpath, a short name that the OS expands) belongs to the * end-to-end oracle, which runs on Windows in CI. */ +import { + existsSync, + mkdirSync, + mkdtempSync, + realpathSync, + rmSync, +} from 'node:fs' +import { tmpdir } from 'node:os' import fc from 'fast-check' import { createRequire } from 'node:module' import { dirname, join } from 'node:path' import { fileURLToPath } from 'node:url' import { differs, oracleProperty, pass } from '../src/oracle-property.ts' -import { relativeWithin } from './paths.mjs' +import { posix, realPathOf, relativeWithin } from './paths.mjs' import { instrument } from './schemata.mjs' const here = dirname(fileURLToPath(import.meta.url)) @@ -69,21 +77,14 @@ const NAMES = [`src`, `RUNNER~1`, `my dir`, `pkg`, `lib`, `a.js`, `b.ts`] const historyArbitrary = fc.record({ text: fc.integer({ min: 0, max: TEXTS.length - 1 }), // The directory's drive letter may differ in case from the file's. - drive: fc.constantFrom( - ...(process.platform === `win32` ? [`C`, `c`, `D`] : [`C`]), - ), - dirDrive: fc.constantFrom( - ...(process.platform === `win32` ? [`C`, `c`] : [`C`]), - ), + drive: fc.constantFrom(`C`, `c`, `D`, `x`), + dirDrive: fc.constantFrom(`C`, `c`, `x`), root: fc.array(fc.constantFrom(...NAMES), { minLength: 1, maxLength: 3 }), rest: fc.array(fc.constantFrom(...NAMES), { minLength: 1, maxLength: 2 }), // One separator for each joint: the drive to the first name, and each name. // On win32 a backslash separates; on POSIX it is a name's character, so only // `/` joins there (a narrowing: the POSIX law is the forward-slash spelling). - joints: fc.array( - fc.constantFrom(...(process.platform === `win32` ? [`/`, `\\`] : [`/`])), - { minLength: 8, maxLength: 8 }, - ), + joints: fc.array(fc.constantFrom(`/`, `\\`), { minLength: 8, maxLength: 8 }), }) /** The segments of a history's spelling: the drive, then the names. */ @@ -109,17 +110,29 @@ function idsAt(path, text) { return mutants.map((m) => m.id).sort() } -// Law 2's answer, from the segments alone: the directory's segments start the -// file's (the drive compared case-insensitively), and the rest is the answer. -function relationModel({ root, rest, drive, dirDrive }) { - const dir = [dirDrive.toUpperCase(), ...root] - const file = [drive.toUpperCase(), ...root, ...rest] - const starts = dir.every((segment, i) => file[i] === segment) - return starts ? rest.join(`/`) : undefined +// Law 2's answer, from the spelled strings alone. On win32 both separators +// split and a drive's case is ignored; on POSIX only `/` splits, a backslash is +// a name's character, and `x:` is an ordinary name. The rest is the answer. +function relationModel(history) { + const win = process.platform === `win32` + const { directory, file } = segmentsOf(history) + const dirPath = spell( + directory, + history.joints.slice(0, directory.length - 1), + ) + const filePath = spell(file, history.joints.slice(0, file.length - 1)) + const split = (text) => text.split(win ? /[\\/]/ : /\//) + const same = (a, b) => (win ? a.toUpperCase() === b.toUpperCase() : a === b) + const dir = split(dirPath) + const all = split(filePath) + const starts = dir.every((segment, i) => same(segment, all[i] ?? ``)) + return starts ? all.slice(dir.length).join(`/`) : undefined } const model = (history) => ({ relation: relationModel(history), + // On POSIX a backslash spelling is a name the OS keeps: its real path exists. + disk: process.platform === `win32` ? undefined : true, }) const check = (history, trace, expected) => { @@ -130,6 +143,14 @@ const check = (history, trace, expected) => { return ( differs(`enumeration`, `the forward-slash spelling`, spelled, reference) ?? differs(`relation`, `segments`, find(`relation`), expected.relation) ?? + (expected.disk === undefined + ? undefined + : differs( + `disk`, + `a backslash name is kept native`, + find(`disk`), + true, + )) ?? pass ) } @@ -158,13 +179,31 @@ oracleProperty({ `spelled`, idsAt(spell(file, history.joints.slice(0, file.length - 1)), text), ) - observe( - `relation`, - relativeWithin( - spell(directory, history.joints.slice(0, directory.length - 1)), - spell(file, history.joints.slice(0, file.length - 1)), - ), + const dirPath = spell( + directory, + history.joints.slice(0, directory.length - 1), ) + const filePath = spell(file, history.joints.slice(0, file.length - 1)) + observe(`relation`, relativeWithin(dirPath, filePath)) + // The F5 witness on POSIX: the name is created with its backslashes, and + // its real path is that name, not a rewritten one. + if (process.platform !== `win32`) { + const base = realpathSync.native( + mkdtempSync(join(tmpdir(), `path-spelling-`)), + ) + try { + const made = join(base, filePath) + mkdirSync(made, { recursive: true }) + observe( + `disk`, + existsSync(made) && + realPathOf(made) === realpathSync.native(made) && + posix(made) === made, + ) + } finally { + rmSync(base, { recursive: true, force: true }) + } + } }, check, }) diff --git a/packages/oracles/scripts/windows-witness.mjs b/packages/oracles/scripts/windows-witness.mjs index 7c45e79..05ace4a 100644 --- a/packages/oracles/scripts/windows-witness.mjs +++ b/packages/oracles/scripts/windows-witness.mjs @@ -167,22 +167,22 @@ if (command === `done`) { }) let stdout = `` let stderr = `` - // The stop lands 400 ms after the run starts: the fixture's whole run lasts - // about a second, and the CLI's output is written in one burst near its end, - // so the first output line is no clock. A run that ends first is a failure. + // The stop lands on an event, not a clock: the CLI's first live line. Its + // progress lines stream on stderr as the run goes; stdout gets the done + // summary only at the end (measured, 2026-11-14), so stdout is no start + // marker. A run that ends before its first line is a failure. let requested = false - const stopTimer = setTimeout(() => { - requested = true - writeFileSync(stopFile, ``) - }, 400) child.stdout.on(`data`, (chunk) => { stdout += chunk }) child.stderr.on(`data`, (chunk) => { stderr += chunk + if (!requested) { + requested = true + writeFileSync(stopFile, ``) + } }) child.on(`close`, (code, signal) => { - clearTimeout(stopTimer) if (!requested) { fail( `the run ended before the stop landed (exit ${code}, signal ${signal})`, From 63bf49e938e532b557e593bdcadac922fe19a768 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 18:29:43 -0700 Subject: [PATCH 25/53] fix(oracle-gaps): constructed tests keep their names (classify reach) The derived hostile names reached constructed tests too (ms 0 at index 0), which broke the name-relationship reach of two constructed histories ("a failing baseline test whose name another test's contains" and "... ends another test's"). A hostile name now applies to generated tests only (ms >= 1). Evidence block - Law: the constructed histories reach their declared states (the oracle's own reach rule). - RED: de8a373 run: "constructed history ... did not enter a baseline test whose name another test's contains" (Linux shard 3 and the Windows classify job). - GREEN: local classify 276/276. - Mutant: not run. - Limits: as recorded in the previous commit. Co-authored-by: Isaac --- .../oracle-gaps/gap-report-classify-oracle.property.test.mjs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs index 6bcdd0a..ca3c395 100644 --- a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs @@ -623,7 +623,10 @@ const HOSTILE = [ `back\\ `, ] const hostileOf = (t, ti) => { - const pick = (t.ms + ti) % 4 === 0 ? (t.ms + ti) % HOSTILE.length : -1 + // A constructed test is plainTest, with ms 0: its names keep the reach of + // each constructed history, so only a generated test (ms >= 1) takes one. + const pick = + t.ms >= 1 && (t.ms + ti) % 4 === 0 ? (t.ms + ti) % HOSTILE.length : -1 return pick < 0 ? `` : HOSTILE[pick] } // A test's names, as the runner reports them: its suites' names, then its From 2b7b7f3f671e8a197eac197c178a7992d6c30c7c Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 19:54:11 -0700 Subject: [PATCH 26/53] fix(windows): Rerun cmd lines, ! as no-form, separators by platform, prefix rule, witness Round-2 review of #58 (NOT READY). Items addressed: 1. Rerun (cmd) fell back on win32 every time: an absent command was passed as undefined and read as "no cmd form". An absent command is now nothing; only a value that cannot be represented has no form (oracle-property.ts commandOf). The witness runs the Rerun lines (as `vitest run` plus the line) in pwsh, Windows PowerShell 5.1, and cmd. 2. `!` is in the cmd no-form set (framework and printer; DelayedExpansion expands it silently). The classify oracle's fallback test follows. 3. The path-spelling generator's original drive and joint domains are restored (the win32 form at b904d57); the POSIX spelling's backslash names and `x:` drives are an appended `wide` record, used on POSIX only. 4. The witness header, the stop failure text, and the replay comments describe the code (event trigger on the first stderr line; the no-form set). 5. isOutside, the owners' node_modules split, and the capture-setup machinery regex split on `\` only on win32; on POSIX a backslash is a name's character. 6. (see 4) 7. Windows PowerShell 5.1 (`powershell.exe`) runs each PowerShell line, as well as pwsh. 8. configureOracles rejects an envPrefix that is not `[A-Z][A-Z0-9_]*`, with a message (the prefix is printed in replay lines). The unit test follows. Evidence block - Law: the printed command replays in the shell it names, or says it has no form (register: deterministic replay across shells); a Rerun line replays too; `!` is a no-form character (coordinator ruling); a POSIX backslash is a name's character (register: Windows paths, as amended). - Reach: the same histories (the win32 domains are unchanged; POSIX uses the appended wide record). - RED (item 1): round-2 review: plain Rerun lines printed "use the PowerShell line" on win32. - GREEN: classify 276/276, path-spelling 5/5, framework 114/114 (locally, POSIX); Windows: CI on this head. - Mutant: the round-2 Rerun behaviour is the mutant (rerunOf passed an absent command as no form). - Limits: U+2019 is not reached by the seeds in the path-spelling or replay histories; the hostile names are exercised by the witness's fixture names. Co-authored-by: Isaac --- packages/oracles/oracle-gaps/args.mjs | 8 +- .../oracles/oracle-gaps/capture-setup.mjs | 7 +- ...p-report-classify-oracle.property.test.mjs | 2 +- ...ort-path-spelling-oracle.property.test.mjs | 33 ++++- packages/oracles/oracle-gaps/owners.mjs | 7 +- packages/oracles/oracle-gaps/stream.mjs | 5 +- packages/oracles/scripts/windows-witness.mjs | 125 ++++++++++-------- packages/oracles/src/oracle-config.ts | 8 +- packages/oracles/src/oracle-property.ts | 11 +- packages/oracles/tests/config.test.ts | 10 +- 10 files changed, 144 insertions(+), 72 deletions(-) diff --git a/packages/oracles/oracle-gaps/args.mjs b/packages/oracles/oracle-gaps/args.mjs index f3e6c69..d8f7d02 100644 --- a/packages/oracles/oracle-gaps/args.mjs +++ b/packages/oracles/oracle-gaps/args.mjs @@ -265,7 +265,13 @@ function runName() { * `..` (a name such as `..gen` is inside), or it is absolute. */ export function isOutside(path) { - return path === `..` || /^\.\.[\\/]/.test(path) || isAbsolute(path) + // On POSIX only `/` separates, so `..\\x` is a name, not a parent (review F5). + const separator = process.platform === `win32` ? `[\\\\/]` : `/` + return ( + path === `..` || + new RegExp(`^\\.\\.${separator}`).test(path) || + isAbsolute(path) + ) } /** diff --git a/packages/oracles/oracle-gaps/capture-setup.mjs b/packages/oracles/oracle-gaps/capture-setup.mjs index 8872333..5255684 100644 --- a/packages/oracles/oracle-gaps/capture-setup.mjs +++ b/packages/oracles/oracle-gaps/capture-setup.mjs @@ -783,8 +783,11 @@ globalThis.__oracleWorkBound = (ratio) => { // budget), Node's, or this file's does not, while a user's read through a // spy (tinyspy's frame first) still does. let clocked = false -const machinery = - /[\\/]node_modules[\\/](?:vitest|@vitest|fast-check|@fast-check|tinypool|tinyspy|chai|@tanstack[\\/]oracles)[\\/]|[\\/]packages[\\/]oracles[\\/]src[\\/]/ +// A backslash separates only on win32 (on POSIX it is a name's character). +const SEP = process.platform === `win32` ? `[\\\\/]` : `/` +const machinery = new RegExp( + `${SEP}node_modules${SEP}(?:vitest|@vitest|fast-check|@fast-check|tinypool|tinyspy|chai|@tanstack${SEP}oracles)${SEP}|${SEP}packages${SEP}oracles${SEP}src${SEP}`, +) function fromTest() { const limit = Error.stackTraceLimit Error.stackTraceLimit = 50 diff --git a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs index ca3c395..5c43e77 100644 --- a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs @@ -2756,7 +2756,7 @@ function printedBreach(h, lines, expected, platform) { const pattern = psLine ? powerShellWords(psLine.command)?.[4] : undefined - if (pattern === undefined || !/["%]/.test(pattern)) + if (pattern === undefined || !/["%!]/.test(pattern)) return breach( `the replay of ${file} says cmd cannot replay it, but no name in it needs that`, ) diff --git a/packages/oracles/oracle-gaps/gap-report-path-spelling-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-path-spelling-oracle.property.test.mjs index 5113cdb..951c5d8 100644 --- a/packages/oracles/oracle-gaps/gap-report-path-spelling-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-path-spelling-oracle.property.test.mjs @@ -77,16 +77,37 @@ const NAMES = [`src`, `RUNNER~1`, `my dir`, `pkg`, `lib`, `a.js`, `b.ts`] const historyArbitrary = fc.record({ text: fc.integer({ min: 0, max: TEXTS.length - 1 }), // The directory's drive letter may differ in case from the file's. - drive: fc.constantFrom(`C`, `c`, `D`, `x`), - dirDrive: fc.constantFrom(`C`, `c`, `x`), + drive: fc.constantFrom( + ...(process.platform === `win32` ? [`C`, `c`, `D`] : [`C`]), + ), + dirDrive: fc.constantFrom( + ...(process.platform === `win32` ? [`C`, `c`] : [`C`]), + ), root: fc.array(fc.constantFrom(...NAMES), { minLength: 1, maxLength: 3 }), rest: fc.array(fc.constantFrom(...NAMES), { minLength: 1, maxLength: 2 }), // One separator for each joint: the drive to the first name, and each name. // On win32 a backslash separates; on POSIX it is a name's character, so only // `/` joins there (a narrowing: the POSIX law is the forward-slash spelling). - joints: fc.array(fc.constantFrom(`/`, `\\`), { minLength: 8, maxLength: 8 }), + joints: fc.array( + fc.constantFrom(...(process.platform === `win32` ? [`/`, `\\`] : [`/`])), + { minLength: 8, maxLength: 8 }, + ), + // Appended last, so the fields above keep their draws: the drives and + // separators a POSIX spelling uses (a backslash name and an `x:` drive). + wide: fc.record({ + drive: fc.constantFrom(`C`, `c`, `D`, `x`), + dirDrive: fc.constantFrom(`C`, `c`, `x`), + joints: fc.array(fc.constantFrom(`/`, `\\`), { + minLength: 8, + maxLength: 8, + }), + }), }) +/** The spelling a platform uses: win32 the fields above, POSIX the wide ones. */ +const effective = (history) => + process.platform === `win32` ? history : { ...history, ...history.wide } + /** The segments of a history's spelling: the drive, then the names. */ const segmentsOf = ({ drive, dirDrive, root, rest }) => ({ directory: [`${dirDrive}:`, ...root], @@ -113,7 +134,8 @@ function idsAt(path, text) { // Law 2's answer, from the spelled strings alone. On win32 both separators // split and a drive's case is ignored; on POSIX only `/` splits, a backslash is // a name's character, and `x:` is an ordinary name. The rest is the answer. -function relationModel(history) { +function relationModel(raw) { + const history = effective(raw) const win = process.platform === `win32` const { directory, file } = segmentsOf(history) const dirPath = spell( @@ -171,7 +193,8 @@ oracleProperty({ }, timeout: 10_000, model, - driver: (history, observe) => { + driver: (raw, observe) => { + const history = effective(raw) const { directory, file } = segmentsOf(history) const text = TEXTS[history.text] observe(`reference`, idsAt(posixOf(file), text)) diff --git a/packages/oracles/oracle-gaps/owners.mjs b/packages/oracles/oracle-gaps/owners.mjs index f268c61..490f94a 100644 --- a/packages/oracles/oracle-gaps/owners.mjs +++ b/packages/oracles/oracle-gaps/owners.mjs @@ -301,7 +301,12 @@ export function ownership(ts, program, pkg, oracles) { const checker = program.getTypeChecker() const inPackage = (fileName) => { const path = relative(pkg, fileName) - return !isOutside(path) && !path.split(/[\\/]/).includes(`node_modules`) + return ( + !isOutside(path) && + !path + .split(process.platform === `win32` ? /[\\\\/]/ : `/`) + .includes(`node_modules`) + ) } const indexed = new Map() const functionsOf = (source) => { diff --git a/packages/oracles/oracle-gaps/stream.mjs b/packages/oracles/oracle-gaps/stream.mjs index a2b07cd..0430bb8 100644 --- a/packages/oracles/oracle-gaps/stream.mjs +++ b/packages/oracles/oracle-gaps/stream.mjs @@ -84,11 +84,12 @@ const powerShellWord = (word) => /^[\w./-]+$/.test(word) ? word : `'${word.replace(PS_SINGLE_QUOTES, (quote) => quote + quote)}'` -// cmd toggles its quoting on every double quote, and expands `%`, even inside +// cmd toggles its quoting on every double quote, expands `%`, and under +// DelayedExpansion expands `!` (a conservative no-form set, a coordinator ruling), even inside // quotes; a name with either has no cmd form (undefined), so the line says so. // Inside double quotes cmd reads the other metacharacters literally. const cmdWord = (word) => - /["%]/.test(word) ? undefined : /^[\w./-]+$/.test(word) ? word : `"${word}"` + /["%!]/.test(word) ? undefined : /^[\w./-]+$/.test(word) ? word : `"${word}"` // The replay line of a command. A POSIX shell reads one line; on Windows // there are two, one for PowerShell and one for cmd, since the words differ. // A cmd line with a word that has no cmd form says so instead of printing a diff --git a/packages/oracles/scripts/windows-witness.mjs b/packages/oracles/scripts/windows-witness.mjs index 05ace4a..e3266a2 100644 --- a/packages/oracles/scripts/windows-witness.mjs +++ b/packages/oracles/scripts/windows-witness.mjs @@ -8,19 +8,20 @@ // code (completed 0, refused 2, error 3, a stop 130 or 143). // // node scripts/windows-witness.mjs replay -// The replay lines of a failing property: the PowerShell line and the -// cmd line each reproduce the failure when run in their own shell. The -// POSIX line is run in PowerShell too, and must not reproduce (that is the -// RED of the framework's per-shell lines). A run whose output has only -// the POSIX line on win32 fails here, because the per-shell lines are -// missing. +// Every printed command in a failing property's output: each `Replay` and +// `Rerun` line for PowerShell and for cmd. Each PowerShell line runs in +// pwsh and in Windows PowerShell 5.1 (powershell.exe), each cmd line in +// cmd. A line must reproduce the failure, except a cmd line that says cmd +// has no form for its name (a name with a double quote, `%`, or `!`). A +// Rerun line is run as `vitest run` plus its arguments, since it names the +// test only. A run whose output has no per-shell line for a failure fails. // // node scripts/windows-witness.mjs stop -// One early stop: the CLI starts, the witness sends SIGINT once the CLI -// has printed its first line, and the run must end in one of the two -// outcomes of the all-or-nothing law (#57): (a) the stop's done line on -// both streams, matching the exit code (130 or 143); (b) no output at all. -// Anything else, including a partial output, fails. +// One stop: the CLI runs with the test-only hook ORACLE_GAPS_TEST_STOP, and +// the witness writes that file on the CLI's first stderr line (its progress +// streams there; stdout gets the summary only at the end). The run must end +// with its stopped line on both streams and exit 130 (SIGINT). A run that +// completes first, or ends with another line or code, fails. import { spawn, spawnSync } from 'node:child_process' import { readFileSync, rmSync, writeFileSync } from 'node:fs' import { delimiter, join, resolve } from 'node:path' @@ -87,9 +88,10 @@ if (command === `done`) { if (problems.length > 0) fail(problems.join('; ')) else console.log(`done line ok on both streams, exit ${exitCode}`) } else if (command === `replay`) { - // Every failing property in the output prints a PowerShell line and a cmd - // line (or says cmd has no form for its name). Each pair is run in its own - // shell, and each must reproduce the failure. The POSIX line is not run. + // Every printed command of a failing property: the Replay and Rerun lines, + // for PowerShell and for cmd. Each PowerShell line runs in pwsh and in + // Windows PowerShell 5.1; each cmd line runs in cmd. A cmd line that says it + // has no form (a name with `"`, `%`, or `!`) is the only line that may not run. const [outputFile, dir] = args const output = readFileSync(outputFile, `utf8`).replace(/\x1b\[[0-9;]*m/g, ``) const lines = output.split(/\r?\n/) @@ -97,8 +99,11 @@ if (command === `done`) { lines .filter((text) => text.includes(label)) .map((text) => text.slice(text.indexOf(label) + label.length)) - const powershell = after(`Replay (PowerShell): `) - const cmd = after(`Replay (cmd): `) + const replayPs = after(`Replay (PowerShell): `) + const replayCmd = after(`Replay (cmd): `) + const rerunPs = after(`Rerun (PowerShell): `) + const rerunCmd = after(`Rerun (cmd): `) + const noForm = `use the PowerShell line for this name` // The shells find the test runner in this package's own binaries. const env = { ...process.env, @@ -108,47 +113,63 @@ if (command === `done`) { process.env.PATH, ].join(delimiter), } - const runIn = (shell, line) => - spawnSync( - shell === `pwsh` ? `pwsh` : `cmd`, - shell === `pwsh` - ? [`-NoProfile`, `-Command`, line] - : [`/d`, `/s`, `/c`, line], + const runIn = (shell, line) => { + const argv = { + pwsh: [`pwsh`, [`-NoProfile`, `-Command`, line]], + powershell: [`powershell.exe`, [`-NoProfile`, `-Command`, line]], + cmd: [`cmd`, [`/d`, `/s`, `/c`, line]], + }[shell] + return spawnSync(argv[0], argv[1], { + cwd: dir, + env, + encoding: `utf8`, // cmd reads its line as it is; Node's own quoting would break it. - { - cwd: dir, - env, - encoding: `utf8`, - windowsVerbatimArguments: shell !== `pwsh`, - }, - ) + windowsVerbatimArguments: shell === `cmd`, + }) + } const reproduces = (result) => result.status !== 0 && /violated|failed/.test(`${result.stdout}${result.stderr}`) - if (powershell.length === 0) { - fail(`the failure printed no PowerShell replay line`) - } else if (powershell.length !== cmd.length) { - fail(`${powershell.length} PowerShell lines but ${cmd.length} cmd lines`) - } else { - const failures = [] - powershell.forEach((psLine, i) => { - const psRun = runIn(`pwsh`, psLine) - if (!reproduces(psRun)) - failures.push(`PowerShell line ${i + 1} did not reproduce: ${psLine}`) - const cmdLine = cmd[i] - if (cmdLine === `use the PowerShell line for this name`) { - console.log(`cmd has no form for name ${i + 1}, as printed`) - return + // A Rerun line names the test only: it runs as `vitest run `. + const asRerun = (line, prefix) => `vitest run ${line}` + const failures = [] + let checked = 0 + const check = (name, lines, shells, cmdLines, runOf) => { + lines.forEach((line, i) => { + for (const shell of shells) { + checked += 1 + const result = runIn(shell, runOf(line)) + if (!reproduces(result)) + failures.push( + `${name} ${i + 1} did not reproduce in ${shell}: ${line}`, + ) } - const cmdRun = runIn(`cmd`, cmdLine) - if (!reproduces(cmdRun)) - failures.push(`cmd line ${i + 1} did not reproduce: ${cmdLine}`) + if (cmdLines === undefined) return + const cmdLine = cmdLines[i] + if (cmdLine === noForm) return + checked += 1 + const result = runIn(`cmd`, runOf(cmdLine)) + if (!reproduces(result)) + failures.push(`${name} ${i + 1} did not reproduce in cmd: ${cmdLine}`) }) - if (failures.length > 0) fail(failures.join('; ').slice(0, 1500)) - else - console.log( - `${powershell.length} printed name(s): PowerShell and cmd lines each reproduce`, - ) + } + check(`Replay`, replayPs, [`pwsh`, `powershell`], replayCmd, (line) => line) + check(`Rerun`, rerunPs, [`pwsh`, `powershell`], rerunCmd, (line) => + asRerun(line), + ) + if (replayPs.length + rerunPs.length === 0) { + fail(`the failure printed no PowerShell replay or rerun line`) + } else if ( + replayPs.length !== replayCmd.length || + rerunPs.length !== rerunCmd.length + ) { + fail(`PowerShell and cmd lines differ in number`) + } else if (failures.length > 0) { + fail(failures.join(`; `).slice(0, 1500)) + } else { + console.log( + `${checked} printed commands reproduce the failure (PowerShell 7, Windows PowerShell 5.1, cmd)`, + ) } } else if (command === `stop`) { // The stop is the CLI's test-only hook (ORACLE_GAPS_TEST_STOP): a file that @@ -197,7 +218,7 @@ if (command === `done`) { ].filter(Boolean) if (problems.length > 0) fail( - `stop was not all-or-nothing (signal ${signal}, exit ${code}): ${problems.join('; ')}`, + `the stop did not end with its stopped line on both streams and exit 130 (signal ${signal}, exit ${code}): ${problems.join('; ')}`, ) else console.log( diff --git a/packages/oracles/src/oracle-config.ts b/packages/oracles/src/oracle-config.ts index 3da5a16..4aea6b8 100644 --- a/packages/oracles/src/oracle-config.ts +++ b/packages/oracles/src/oracle-config.ts @@ -74,8 +74,12 @@ export function configureOracles({ registry: propertyIds, retiredVariables: retired = [], }: OracleConfigurationOptions = {}): void { - if (envPrefix === ``) { - throw new Error(`configureOracles: envPrefix must be non-empty`) + // The prefix names environment variables that replay lines print, and a + // shell reads them as words only when they are plain (review item 8). + if (!/^[A-Z][A-Z0-9_]*$/.test(envPrefix)) { + throw new Error( + `configureOracles: envPrefix must be an upper-case name such as ORACLE (letters, digits, and underscores, starting with a letter), not ${JSON.stringify(envPrefix)}`, + ) } const candidate: OracleSettings = { prefix: envPrefix, diff --git a/packages/oracles/src/oracle-property.ts b/packages/oracles/src/oracle-property.ts index cca614b..5b7f4a9 100644 --- a/packages/oracles/src/oracle-property.ts +++ b/packages/oracles/src/oracle-property.ts @@ -2282,10 +2282,12 @@ function commandOf( // inside a single-quoted word each is doubled. const psQuoted = (text: string) => `'${text.replace(/['\u2018\u2019\u201A\u201B]/g, (quote) => quote + quote)}'` - // cmd toggles quoting on every double quote and expands `%`: a text with - // either has no cmd form (undefined), and its line says so. + // cmd toggles quoting on every double quote, expands `%`, and under + // DelayedExpansion expands `!`: a text with any of these has no cmd form + // (undefined), and its line says so. The set is conservative (a coordinator + // ruling). const cmdQuoted = (text: string) => - /["%]/.test(text) ? undefined : `"${text}"` + /["%!]/.test(text) ? undefined : `"${text}"` const cmdWords = (...parts: ReadonlyArray) => parts.some((part) => part === undefined) ? undefined : words(parts) const words = (parts: ReadonlyArray) => @@ -2309,7 +2311,8 @@ function commandOf( const quoted = cmdQuoted(value) return quoted === undefined ? undefined : `set "${name}=${value}" &&` }), - command, + // An absent command is nothing in cmd too, not a text without a form. + command ?? ``, cmdQuoted(pattern) === undefined ? undefined : `-t ${cmdQuoted(pattern)}`, ), } diff --git a/packages/oracles/tests/config.test.ts b/packages/oracles/tests/config.test.ts index 2d7a6fc..1294a4a 100644 --- a/packages/oracles/tests/config.test.ts +++ b/packages/oracles/tests/config.test.ts @@ -148,9 +148,15 @@ describe(`configureOracles`, () => { ) }) - it(`rejects an empty prefix`, () => { + it(`rejects a prefix that is not an upper-case name`, () => { expect(() => configureOracles({ envPrefix: `` })).toThrow( - `envPrefix must be non-empty`, + `envPrefix must be an upper-case name`, + ) + expect(() => configureOracles({ envPrefix: `oracle` })).toThrow( + `envPrefix must be an upper-case name`, + ) + expect(() => configureOracles({ envPrefix: `A B` })).toThrow( + `envPrefix must be an upper-case name`, ) }) From abc8b45f1b7ae9223fb0b3ea42fc0de7298f3ca7 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 20:20:22 -0700 Subject: [PATCH 27/53] fix(windows): the witness feeds PowerShell its line on stdin, not as an argument Run 43548823073 (head 2b7b7f3): the Windows CLI job failed on Windows PowerShell 5.1 ("Replay 3 did not reproduce in powershell", a parse exception on the fixture with a double-quoted name). The witness passed the line as a -Command argument, so Node's argument quoting turned the double quote into \" and 5.1 does not read that as an escape. A typed line never passes through an argv: the witness now pipes the line to `-Command -` on stdin, for pwsh and powershell.exe alike. Evidence block - Law: each printed line replays in the shell it names, as a user types it (the register's per-shell rule). - RED: run 43548823073, witness step: "Replay 3 did not reproduce in powershell" (the Windows PowerShell 5.1 parse exception). - GREEN: pending the next Windows run. - Mutant: the argument passing is the mutant (the run above). - Limits: a cmd line is still passed verbatim as an argument (cmd has no stdin command mode here). Co-authored-by: Isaac --- packages/oracles/scripts/windows-witness.mjs | 33 +++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/packages/oracles/scripts/windows-witness.mjs b/packages/oracles/scripts/windows-witness.mjs index e3266a2..b539270 100644 --- a/packages/oracles/scripts/windows-witness.mjs +++ b/packages/oracles/scripts/windows-witness.mjs @@ -114,18 +114,27 @@ if (command === `done`) { ].join(delimiter), } const runIn = (shell, line) => { - const argv = { - pwsh: [`pwsh`, [`-NoProfile`, `-Command`, line]], - powershell: [`powershell.exe`, [`-NoProfile`, `-Command`, line]], - cmd: [`cmd`, [`/d`, `/s`, `/c`, line]], - }[shell] - return spawnSync(argv[0], argv[1], { - cwd: dir, - env, - encoding: `utf8`, - // cmd reads its line as it is; Node's own quoting would break it. - windowsVerbatimArguments: shell === `cmd`, - }) + // PowerShell reads the line from stdin (`-Command -`), as a user's typed + // line arrives: an argument would pass through Node's quoting, which turns + // a double quote into `\"`, and Windows PowerShell 5.1 does not read that as + // an escape. cmd reads its line as it is (verbatim arguments). + if (shell === `cmd`) + return spawnSync(`cmd`, [`/d`, `/s`, `/c`, line], { + cwd: dir, + env, + encoding: `utf8`, + windowsVerbatimArguments: true, + }) + return spawnSync( + shell === `pwsh` ? `pwsh` : `powershell.exe`, + [`-NoProfile`, `-Command`, `-`], + { + cwd: dir, + env, + encoding: `utf8`, + input: line, + }, + ) } const reproduces = (result) => result.status !== 0 && From 69c82ebbafa455a115b3ea628a748bbbd4af4bd0 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 20:47:49 -0700 Subject: [PATCH 28/53] diagnostic: the replay witness reports the shell's own error for a line that does not reproduce Run 43551132190 (abc8b45): "Replay 3 did not reproduce in powershell" with no reason. The failure now carries the shell's exit code and output, so the Windows PowerShell 5.1 error for the double-quoted name is visible. Diagnostic; the witness's pass and fail rules are unchanged. Co-authored-by: Isaac --- packages/oracles/scripts/windows-witness.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/oracles/scripts/windows-witness.mjs b/packages/oracles/scripts/windows-witness.mjs index b539270..21e1796 100644 --- a/packages/oracles/scripts/windows-witness.mjs +++ b/packages/oracles/scripts/windows-witness.mjs @@ -150,7 +150,7 @@ if (command === `done`) { const result = runIn(shell, runOf(line)) if (!reproduces(result)) failures.push( - `${name} ${i + 1} did not reproduce in ${shell}: ${line}`, + `${name} ${i + 1} did not reproduce in ${shell} (exit ${result.status}: ${`${result.stderr}${result.stdout}`.trim().slice(0, 400)}): ${line}`, ) } if (cmdLines === undefined) return From 0772447aac52ef662459d75006e0651a3afda97d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 21:12:48 -0700 Subject: [PATCH 29/53] fix(windows): the -t argument never carries a shell-hostile character (option c) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PowerShell 5.1 passes an embedded double quote in a native .cmd argument wrongly (run 43553461458: a `"`-named fixture selected no test, exit 0). The coordinator's ruling (option c): `-t` is a coarse regular-expression filter, and the exact test is carried by ORACLE_UNIT where a line sets it. - The `-t` pattern (CLI baseline and framework Rerun) replaces each character a shell reads unsafely (`"`, `%`, `!`, the curly quotes) with `.`: still anchored, only slightly broader. - Framework environment values: PowerShell (5.1 and 7) doubles the single and curly quotes, which works for any name; cmd's `set "NAME=value"` tolerates a double quote, and `%` and `!` have no cmd form (environment values only). - Classify oracle: a CLI line has no environment values, so a cmd fallback line is a breach; the -t argument must carry no shell-hostile character. - Witness: each PowerShell Replay line's ORACLE_UNIT and ORACLE_PROPERTY decode to the printed text (round-trip), in addition to the reproduction in pwsh, Windows PowerShell 5.1, and cmd. Evidence block - Law: a printed command replays exactly the unit it names, in each shell (register: deterministic replay across shells; the coordinator's option c). - Reach: the classify oracle's 276 histories (the adversarial names, derived); the Windows fixtures (`’`, `"`, `%`). - RED: run 43553461458: "Replay 3 did not reproduce in powershell", exit 0, PowerShell 5.1 with an embedded double quote in the -t argument. - GREEN: classify 276/276 and framework 114/114 locally (POSIX); the Windows witness runs on this head. - Mutant: the pre-fix pattern (a raw double quote in -t) is the mutant; the RED above is its rejection. - Limits: the -t pattern is slightly broader for names with a hostile character (the wildcard may select a test whose name differs in that position, which the classify oracle's names cannot produce). Co-authored-by: Isaac --- ...p-report-classify-oracle.property.test.mjs | 28 +++++++++-------- packages/oracles/oracle-gaps/stream.mjs | 12 +++++++- packages/oracles/scripts/windows-witness.mjs | 30 +++++++++++++++++++ packages/oracles/src/oracle-property.ts | 17 +++++++---- 4 files changed, 68 insertions(+), 19 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs index 5c43e77..ee91b70 100644 --- a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs @@ -2750,24 +2750,28 @@ function printedBreach(h, lines, expected, platform) { for (const { command, read } of replays) { // cmd has no form for a name with a double quote or `%`: its line says so, // and only such a test may print that line. - if (command === CMD_FALLBACK) { - // The pattern the PowerShell line runs (the names carry the labels). - const psLine = replays.find((r) => r.read === powerShellWords) - const pattern = psLine - ? powerShellWords(psLine.command)?.[4] - : undefined - if (pattern === undefined || !/["%!]/.test(pattern)) - return breach( - `the replay of ${file} says cmd cannot replay it, but no name in it needs that`, - ) - continue - } + // A CLI line has no environment values, so cmd always has its form: the + // `-t` argument carries no shell-hostile character (review option c). No + // fallback line may print for it. + if (command === CMD_FALLBACK) + return breach( + `the replay of ${file} says cmd cannot replay it, but its -t argument has no shell-hostile character to need that`, + ) const words = read(command) if (!words) return breach( `the replay of ${file} is ${JSON.stringify(command)}, which the shell does not read as plain words`, ) const [vitest, run, target, flag, pattern, ...more] = words + // The -t argument is safe in every shell: no double quote, `%`, `!`, or + // curly quote (the regular expression's `.` stands in for each). + if ( + pattern !== undefined && + /["%!\u2018\u2019\u201A\u201B]/.test(pattern) + ) + return breach( + `the replay of ${file} puts ${JSON.stringify(pattern)} in its -t argument, which a shell reads unsafely`, + ) const path = relativeName(h, owed.fi) if (vitest !== `vitest` || run !== `run` || target !== path) return breach(`the replay of ${file} is not \`vitest run ${path}\``) diff --git a/packages/oracles/oracle-gaps/stream.mjs b/packages/oracles/oracle-gaps/stream.mjs index 0430bb8..bae8a95 100644 --- a/packages/oracles/oracle-gaps/stream.mjs +++ b/packages/oracles/oracle-gaps/stream.mjs @@ -119,8 +119,18 @@ export const escapeControls = (text) => // regular-expression syntax escaped, joined by what either Vitest puts // between them (a space in Vitest 4, ` > ` in Vitest 5), anchored at both // ends. The framework's own rerun command (oracle-property.ts) is built so. +// The characters no shell reads safely inside a `-t` argument: a double quote, +// `%` and `!` (cmd), and the curly quotes (PowerShell). Each becomes the regular +// expression's `.`: the pattern stays anchored and only slightly broader, and +// the exact test is carried by ORACLE_UNIT where a line sets it (review option +// c, a coordinator ruling). +const SHELL_HOSTILE = /["%!\u2018\u2019\u201A\u201B]/g const replayPattern = (names) => - `^${names.map((name) => name.replace(/[.*+?^${}()|[\]\\]/g, `\\$&`)).join(`(?: | > )`)}$` + `^${names + .map((name) => + name.replace(/[.*+?^${}()|[\]\\]/g, `\\$&`).replace(SHELL_HOSTILE, `.`), + ) + .join(`(?: | > )`)}$` // A count of things. const counted = (n, thing) => `${n} ${thing}${n === 1 ? `` : `s`}` diff --git a/packages/oracles/scripts/windows-witness.mjs b/packages/oracles/scripts/windows-witness.mjs index 21e1796..d7382ca 100644 --- a/packages/oracles/scripts/windows-witness.mjs +++ b/packages/oracles/scripts/windows-witness.mjs @@ -162,6 +162,36 @@ if (command === `done`) { failures.push(`${name} ${i + 1} did not reproduce in cmd: ${cmdLine}`) }) } + // ORACLE_UNIT and ORACLE_PROPERTY carry the exact test (review option c): each + // must decode to the text the failure printed, and must not be lost to the + // `-t` pattern's wildcards. A PowerShell single-quoted value doubles its quotes. + const decodeEnv = (line, name) => { + const at = line.indexOf(`$env:${name}='`) + if (at < 0) return undefined + let value = `` + for (let i = at + name.length + 8; i < line.length; i += 1) { + const c = line[i] + if (/['\u2018\u2019\u201A\u201B]/.test(c)) { + if (line[i + 1] === c) { + value += c + i += 1 + continue + } + return value + } + value += c + } + return undefined + } + for (const line of replayPs) { + for (const name of [`ORACLE_UNIT`, `ORACLE_PROPERTY`]) { + const value = decodeEnv(line, name) + if (value === undefined || !output.includes(value)) + failures.push( + `${name} does not round-trip to the printed failure: ${JSON.stringify(value)}`, + ) + } + } check(`Replay`, replayPs, [`pwsh`, `powershell`], replayCmd, (line) => line) check(`Rerun`, rerunPs, [`pwsh`, `powershell`], rerunCmd, (line) => asRerun(line), diff --git a/packages/oracles/src/oracle-property.ts b/packages/oracles/src/oracle-property.ts index 5b7f4a9..7166408 100644 --- a/packages/oracles/src/oracle-property.ts +++ b/packages/oracles/src/oracle-property.ts @@ -2250,7 +2250,12 @@ function rerunPatternOf(task: NamedTask): string { const names: Array = [] for (let at: NamedTask | undefined = task; at; at = at.suite) names.unshift(at.name) - const escape = (text: string) => text.replace(/[.*+?^${}()|[\]\\]/g, `\\$&`) + // The characters no shell reads safely in a `-t` argument become `.`: the + // pattern stays anchored and only slightly broader (review option c). + const escape = (text: string) => + text + .replace(/[.*+?^${}()|[\]\\]/g, `\\$&`) + .replace(/["%!\u2018\u2019\u201A\u201B]/g, `.`) return `^${names.map(escape).join(`(?: | > )`)}$` } @@ -2282,12 +2287,12 @@ function commandOf( // inside a single-quoted word each is doubled. const psQuoted = (text: string) => `'${text.replace(/['\u2018\u2019\u201A\u201B]/g, (quote) => quote + quote)}'` - // cmd toggles quoting on every double quote, expands `%`, and under - // DelayedExpansion expands `!`: a text with any of these has no cmd form - // (undefined), and its line says so. The set is conservative (a coordinator - // ruling). + // An environment value in cmd: `set "NAME=value"` takes everything up to the + // last quote, so a double quote inside is fine. `%` expands, and `!` expands + // under DelayedExpansion, so a value with either has no cmd form (a + // coordinator ruling). The `-t` argument carries no hostile character (above). const cmdQuoted = (text: string) => - /["%!]/.test(text) ? undefined : `"${text}"` + /[%!]/.test(text) ? undefined : `"${text}"` const cmdWords = (...parts: ReadonlyArray) => parts.some((part) => part === undefined) ? undefined : words(parts) const words = (parts: ReadonlyArray) => From ae44184cf2a1799b0322699169c3936cf936f426 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 21:37:14 -0700 Subject: [PATCH 30/53] Revert "diagnostic: the replay witness reports the shell's own error for a line that does not reproduce" This reverts commit 69c82ebbafa455a115b3ea628a748bbbd4af4bd0. --- packages/oracles/scripts/windows-witness.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/oracles/scripts/windows-witness.mjs b/packages/oracles/scripts/windows-witness.mjs index d7382ca..0f0c744 100644 --- a/packages/oracles/scripts/windows-witness.mjs +++ b/packages/oracles/scripts/windows-witness.mjs @@ -150,7 +150,7 @@ if (command === `done`) { const result = runIn(shell, runOf(line)) if (!reproduces(result)) failures.push( - `${name} ${i + 1} did not reproduce in ${shell} (exit ${result.status}: ${`${result.stderr}${result.stdout}`.trim().slice(0, 400)}): ${line}`, + `${name} ${i + 1} did not reproduce in ${shell}: ${line}`, ) } if (cmdLines === undefined) return From 96989220f523690188e3ba108b4108951e647a5b Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 22:23:37 -0700 Subject: [PATCH 31/53] fix(windows): cmd env rule, exact witness matching, set-line readers, fixtures (round-3 review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - F1: a cmd environment value containing `"`, `%`, or `!` has no form (the register's rule); `set "VAR=value" &&` is only emitted for a plain value (oracle-property.ts commandOf). Fixtures: one with a curly-quote property and no `"%!` (a cmd set line runs); one with a single `"` in its property (its cmd line points to PowerShell). - F2: the PowerShell decoder's offset is the value's own position, and decoded values are compared exactly with a failing unit's property and unit, as vitest prints them (`FAIL > > `). cmd `set` lines decode too and must agree with the PowerShell line. - F3 (partial): Rerun lines carry no ORACLE_UNIT: the framework reads ORACLE_UNIT only for a random unit during a path replay, so a fixed-seed Rerun cannot be selected exactly. Reported to the coordinator. - F6: the path-spelling comment states what it means (no history to keep). - F7: the classify readers reject `!` and judge `set … &&` prefixes; the fallback is allowed wherever the printer uses it (a file path with a hostile character). - F9: control characters in rerunPatternOf are escaped as stream.mjs does (by code point, not a control-character regex). - The witness fails when a failing unit prints no per-shell line (FAIL lines are counted per unit). Evidence block - Law: a printed command replays the failing unit it names, in each shell (register: deterministic replay across shells; the set rule). - Reach: the classify oracle's 276 histories; the Windows fixtures. - RED: run 43555531142 and the coordinator's round-3 review at ae44184 (a `"` in a set value broke `&&`). - GREEN: classify 276/276, framework 114/114, locally (POSIX). Windows: this head. - Mutant: the pre-fix env rule (a `"` in a set value) is the mutant. - Limits: Rerun lines are not exact (see F3). Co-authored-by: Isaac --- .github/workflows/pr.yml | 4 +- ...p-report-classify-oracle.property.test.mjs | 24 ++- ...ort-path-spelling-oracle.property.test.mjs | 4 +- packages/oracles/scripts/windows-witness.mjs | 172 +++++++++++------- packages/oracles/src/oracle-property.ts | 28 ++- 5 files changed, 144 insertions(+), 88 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 2a81a5c..c757a1c 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -363,7 +363,7 @@ jobs: import { differs, oracleProperty, pass } from '../../packages/oracles/dist/esm/index.js' oracleProperty({ - id: `fixture ’ & | < > ( ) ^ ! replay`, + id: `fixture ’ & | < > ( ) ^ replay`, law: { statement: `a fixture that always fails, so its replay lines are printed.`, authority: `the CI witness for the per-shell replay lines`, @@ -390,7 +390,7 @@ jobs: import { differs, oracleProperty, pass } from '../../packages/oracles/dist/esm/index.js' oracleProperty({ - id: `fixture "quoted" 100% replay`, + id: `fixture Q"1 replay`, law: { statement: `a fixture that always fails, so its replay lines are printed.`, authority: `the CI witness for the per-shell replay lines`, diff --git a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs index ee91b70..618b04b 100644 --- a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs @@ -2748,15 +2748,19 @@ function printedBreach(h, lines, expected, platform) { `${file}'s replay lines are not the platform's: ${JSON.stringify(block.filter((line) => line.startsWith(` replay`)))}`, ) for (const { command, read } of replays) { - // cmd has no form for a name with a double quote or `%`: its line says so, + // cmd has no form for a word with a double quote, `%`, or `!`: its line says so, // and only such a test may print that line. // A CLI line has no environment values, so cmd always has its form: the // `-t` argument carries no shell-hostile character (review option c). No // fallback line may print for it. - if (command === CMD_FALLBACK) + if (command === CMD_FALLBACK) { + // The printer's cmd line has no form when a word has `"`, `%`, or `!`: + // the file path can carry one (the -t argument never does). + if (/["%!]/.test(relativeName(h, owed.fi))) continue return breach( - `the replay of ${file} says cmd cannot replay it, but its -t argument has no shell-hostile character to need that`, + `the replay of ${file} says cmd cannot replay it, but no word of it has a hostile character`, ) + } const words = read(command) if (!words) return breach( @@ -2878,13 +2882,23 @@ function powerShellWords(line) { // a double quote toggles quoting; outside quotes `^` escapes the next // character, and `& | < > ( )` separate or redirect; `%` expands even inside // quotes. `!` is literal, since delayed expansion is off. Undefined otherwise. -function cmdWords(line) { +function cmdWords(raw) { + // A `set "NAME=value" &&` prefix: each value must be plain (no `"`, `%`, `!`), + // and the words are those after the last `&&`. + let line = raw + for (;;) { + const set = line.match(/^set "([^"]*)" && /) + if (set === null) break + if (/[%!]/.test(set[1])) return undefined + line = line.slice(set[0].length) + } + if (/&&/.test(line)) return undefined const words = [] let word let quoted = false for (let i = 0; i < line.length;) { const c = line[i] - if (c === `%`) return undefined + if (c === `%` || c === `!`) return undefined if (quoted) { if (c === `"`) quoted = false else word = (word ?? ``) + c diff --git a/packages/oracles/oracle-gaps/gap-report-path-spelling-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-path-spelling-oracle.property.test.mjs index 951c5d8..645423d 100644 --- a/packages/oracles/oracle-gaps/gap-report-path-spelling-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-path-spelling-oracle.property.test.mjs @@ -92,8 +92,8 @@ const historyArbitrary = fc.record({ fc.constantFrom(...(process.platform === `win32` ? [`/`, `\\`] : [`/`])), { minLength: 8, maxLength: 8 }, ), - // Appended last, so the fields above keep their draws: the drives and - // separators a POSIX spelling uses (a backslash name and an `x:` drive). + // The drives and separators a POSIX spelling uses (a backslash name and an + // `x:` drive). The oracle is new in #58, so no earlier history is kept. wide: fc.record({ drive: fc.constantFrom(`C`, `c`, `D`, `x`), dirDrive: fc.constantFrom(`C`, `c`, `x`), diff --git a/packages/oracles/scripts/windows-witness.mjs b/packages/oracles/scripts/windows-witness.mjs index 0f0c744..9451c5f 100644 --- a/packages/oracles/scripts/windows-witness.mjs +++ b/packages/oracles/scripts/windows-witness.mjs @@ -12,7 +12,7 @@ // `Rerun` line for PowerShell and for cmd. Each PowerShell line runs in // pwsh and in Windows PowerShell 5.1 (powershell.exe), each cmd line in // cmd. A line must reproduce the failure, except a cmd line that says cmd -// has no form for its name (a name with a double quote, `%`, or `!`). A +// has no form for its name (a word with a double quote, `%`, or `!`). A // Rerun line is run as `vitest run` plus its arguments, since it names the // test only. A run whose output has no per-shell line for a failure fails. // @@ -88,10 +88,12 @@ if (command === `done`) { if (problems.length > 0) fail(problems.join('; ')) else console.log(`done line ok on both streams, exit ${exitCode}`) } else if (command === `replay`) { - // Every printed command of a failing property: the Replay and Rerun lines, - // for PowerShell and for cmd. Each PowerShell line runs in pwsh and in - // Windows PowerShell 5.1; each cmd line runs in cmd. A cmd line that says it - // has no form (a name with `"`, `%`, or `!`) is the only line that may not run. + // Every printed command of a failing property: the Replay and Rerun lines for + // PowerShell and for cmd. Each PowerShell line runs in pwsh and in Windows + // PowerShell 5.1; each cmd line runs in cmd. A cmd line that says cmd has no + // form (an environment value with `"`, `%`, or `!`) is the only line that may + // not run. Every failing unit must print its line, and each Replay line's + // ORACLE_PROPERTY and ORACLE_UNIT must be exactly a failing unit's names. const [outputFile, dir] = args const output = readFileSync(outputFile, `utf8`).replace(/\x1b\[[0-9;]*m/g, ``) const lines = output.split(/\r?\n/) @@ -104,24 +106,59 @@ if (command === `done`) { const rerunPs = after(`Rerun (PowerShell): `) const rerunCmd = after(`Rerun (cmd): `) const noForm = `use the PowerShell line for this name` - // The shells find the test runner in this package's own binaries. - const env = { - ...process.env, - PATH: [ - resolve(here, `../node_modules/.bin`), - join(dir, `node_modules`, `.bin`), - process.env.PATH, - ].join(delimiter), + // The failing units, as vitest prints them: `FAIL > > `. + const failing = new Map() + for (const text of lines) { + const match = text.match( + /FAIL\s+\S+ > (.*?) > ((?:random|fixed) seed: .*)$/, + ) + if (match) + failing.set(`${match[1]}\0${match[2]}`, { + property: match[1], + unit: match[2], + }) + } + const failingRandom = [...failing.values()].filter((f) => + f.unit.startsWith(`random seed: `), + ) + const failingFixed = [...failing.values()].filter((f) => + f.unit.startsWith(`fixed seed `), + ) + // The PowerShell value of $env:NAME='...', decoded: a doubled quote (of any + // PowerShell single-quote character) is that character. + const decodePs = (line, name) => { + const head = `$env:${name}='` + const at = line.indexOf(head) + if (at < 0) return undefined + let value = `` + for (let i = at + head.length; i < line.length; i += 1) { + const c = line[i] + if (/['\u2018\u2019\u201A\u201B]/.test(c)) { + if (line[i + 1] === c) { + value += c + i += 1 + continue + } + return value + } + value += c + } + return undefined + } + // The cmd value of set "NAME=value". A value with a double quote, `%`, or `!` + // has no cmd form, so its line says so and no set line is decoded. + const decodeSet = (line, name) => { + const match = line.match(new RegExp(`set "${name}=([^"]*)" &&`)) + return match === null ? undefined : match[1] } const runIn = (shell, line) => { - // PowerShell reads the line from stdin (`-Command -`), as a user's typed - // line arrives: an argument would pass through Node's quoting, which turns - // a double quote into `\"`, and Windows PowerShell 5.1 does not read that as - // an escape. cmd reads its line as it is (verbatim arguments). + // PowerShell reads the line from stdin (`-Command -`), as a typed line + // arrives: an argument would pass through Node's quoting. cmd reads its line + // as it is (verbatim arguments). if (shell === `cmd`) return spawnSync(`cmd`, [`/d`, `/s`, `/c`, line], { cwd: dir, - env, + env: process.env, encoding: `utf8`, windowsVerbatimArguments: true, }) @@ -130,7 +167,7 @@ if (command === `done`) { [`-NoProfile`, `-Command`, `-`], { cwd: dir, - env, + env: process.env, encoding: `utf8`, input: line, }, @@ -140,75 +177,70 @@ if (command === `done`) { result.status !== 0 && /violated|failed/.test(`${result.stdout}${result.stderr}`) // A Rerun line names the test only: it runs as `vitest run `. - const asRerun = (line, prefix) => `vitest run ${line}` const failures = [] let checked = 0 - const check = (name, lines, shells, cmdLines, runOf) => { - lines.forEach((line, i) => { - for (const shell of shells) { + const runEach = (name, psLines, cmdLines, prefix) => { + psLines.forEach((line, i) => { + for (const shell of [`pwsh`, `powershell`]) { checked += 1 - const result = runIn(shell, runOf(line)) + const result = runIn(shell, `${prefix}${line}`) if (!reproduces(result)) failures.push( - `${name} ${i + 1} did not reproduce in ${shell}: ${line}`, + `${name} ${i + 1} did not reproduce in ${shell} (exit ${result.status}: ${`${result.stderr}${result.stdout}`.trim().slice(0, 400)}): ${line}`, ) } - if (cmdLines === undefined) return const cmdLine = cmdLines[i] + if (cmdLine === undefined) { + failures.push(`${name} ${i + 1} has no cmd line`) + return + } if (cmdLine === noForm) return checked += 1 - const result = runIn(`cmd`, runOf(cmdLine)) + const result = runIn(`cmd`, `${prefix}${cmdLine}`) if (!reproduces(result)) failures.push(`${name} ${i + 1} did not reproduce in cmd: ${cmdLine}`) }) } - // ORACLE_UNIT and ORACLE_PROPERTY carry the exact test (review option c): each - // must decode to the text the failure printed, and must not be lost to the - // `-t` pattern's wildcards. A PowerShell single-quoted value doubles its quotes. - const decodeEnv = (line, name) => { - const at = line.indexOf(`$env:${name}='`) - if (at < 0) return undefined - let value = `` - for (let i = at + name.length + 8; i < line.length; i += 1) { - const c = line[i] - if (/['\u2018\u2019\u201A\u201B]/.test(c)) { - if (line[i + 1] === c) { - value += c - i += 1 - continue - } - return value - } - value += c - } - return undefined - } - for (const line of replayPs) { - for (const name of [`ORACLE_UNIT`, `ORACLE_PROPERTY`]) { - const value = decodeEnv(line, name) - if (value === undefined || !output.includes(value)) + // Every failing unit prints its line: a random unit a Replay line, a fixed + // unit a Rerun line (review: a failure with no per-shell line fails here). + if (replayPs.length !== failingRandom.length) + failures.push( + `${failingRandom.length} random units failed, but ${replayPs.length} Replay lines printed`, + ) + if (rerunPs.length !== failingFixed.length) + failures.push( + `${failingFixed.length} fixed units failed, but ${rerunPs.length} Rerun lines printed`, + ) + // A Replay line's ORACLE_PROPERTY and ORACLE_UNIT must be exactly a failing + // unit's names: decoded, and equal (not contained). + const names = new Set(failingRandom.map((f) => `${f.property}\0${f.unit}`)) + replayPs.forEach((line, i) => { + const property = decodePs(line, `ORACLE_PROPERTY`) + const unit = decodePs(line, `ORACLE_UNIT`) + if (!names.has(`${property}\0${unit}`)) + failures.push( + `Replay ${i + 1} names ${JSON.stringify([property, unit])}, not a failing unit's names`, + ) + const cmdLine = replayCmd[i] + if (cmdLine !== undefined && cmdLine !== noForm) { + const cmdProperty = decodeSet(cmdLine, `ORACLE_PROPERTY`) + const cmdUnit = decodeSet(cmdLine, `ORACLE_UNIT`) + if (cmdProperty !== property || cmdUnit !== unit) failures.push( - `${name} does not round-trip to the printed failure: ${JSON.stringify(value)}`, + `Replay ${i + 1}: cmd names ${JSON.stringify([cmdProperty, cmdUnit])}, PowerShell ${JSON.stringify([property, unit])}`, ) } - } - check(`Replay`, replayPs, [`pwsh`, `powershell`], replayCmd, (line) => line) - check(`Rerun`, rerunPs, [`pwsh`, `powershell`], rerunCmd, (line) => - asRerun(line), - ) - if (replayPs.length + rerunPs.length === 0) { + }) + if (replayPs.length + rerunPs.length === 0) fail(`the failure printed no PowerShell replay or rerun line`) - } else if ( - replayPs.length !== replayCmd.length || - rerunPs.length !== rerunCmd.length - ) { - fail(`PowerShell and cmd lines differ in number`) - } else if (failures.length > 0) { - fail(failures.join(`; `).slice(0, 1500)) - } else { - console.log( - `${checked} printed commands reproduce the failure (PowerShell 7, Windows PowerShell 5.1, cmd)`, - ) + else { + runEach(`Replay`, replayPs, replayCmd, ``) + runEach(`Rerun`, rerunPs, rerunCmd, `vitest run `) + if (failures.length > 0) fail(failures.join(`; `).slice(0, 1500)) + else + console.log( + `${checked} printed commands reproduce the failure (PowerShell 7, Windows PowerShell 5.1, cmd)`, + ) } } else if (command === `stop`) { // The stop is the CLI's test-only hook (ORACLE_GAPS_TEST_STOP): a file that diff --git a/packages/oracles/src/oracle-property.ts b/packages/oracles/src/oracle-property.ts index 7166408..010fb8a 100644 --- a/packages/oracles/src/oracle-property.ts +++ b/packages/oracles/src/oracle-property.ts @@ -2256,6 +2256,16 @@ function rerunPatternOf(task: NamedTask): string { text .replace(/[.*+?^${}()|[\]\\]/g, `\\$&`) .replace(/["%!\u2018\u2019\u201A\u201B]/g, `.`) + // A control character (a line break, a tab) stays one line: as stream.mjs + // escapes it, the regular expression names the same character (F9). + .split(``) + .map((c) => { + const code = c.charCodeAt(0) + const control = + code < 0x20 || code === 0x7f || code === 0x2028 || code === 0x2029 + return control ? `\\u${code.toString(16).padStart(4, `0`)}` : c + }) + .join(``) return `^${names.map(escape).join(`(?: | > )`)}$` } @@ -2287,12 +2297,12 @@ function commandOf( // inside a single-quoted word each is doubled. const psQuoted = (text: string) => `'${text.replace(/['\u2018\u2019\u201A\u201B]/g, (quote) => quote + quote)}'` - // An environment value in cmd: `set "NAME=value"` takes everything up to the - // last quote, so a double quote inside is fine. `%` expands, and `!` expands - // under DelayedExpansion, so a value with either has no cmd form (a - // coordinator ruling). The `-t` argument carries no hostile character (above). - const cmdQuoted = (text: string) => - /[%!]/.test(text) ? undefined : `"${text}"` + // cmd: a double quote in `set "NAME=value"` closes the quote early, so an odd + // number of them breaks the `&&` (F1); `%` expands, and `!` expands under + // DelayedExpansion. An environment value with any of these has no cmd form + // (the register's rule). The `-t` argument carries none (rerunPatternOf). + const cmdEnv = (value: string) => (/["%!]/.test(value) ? undefined : value) + const cmdQuoted = (text: string) => `"${text}"` const cmdWords = (...parts: ReadonlyArray) => parts.some((part) => part === undefined) ? undefined : words(parts) const words = (parts: ReadonlyArray) => @@ -2313,12 +2323,12 @@ function commandOf( ]), cmd: cmdWords( ...env.map(([name, value]) => { - const quoted = cmdQuoted(value) - return quoted === undefined ? undefined : `set "${name}=${value}" &&` + const quoted = cmdEnv(value) + return quoted === undefined ? undefined : `set "${name}=${quoted}" &&` }), // An absent command is nothing in cmd too, not a text without a form. command ?? ``, - cmdQuoted(pattern) === undefined ? undefined : `-t ${cmdQuoted(pattern)}`, + `-t ${cmdQuoted(pattern)}`, ), } } From 257a5b02e054f7f51972ae35be2f7f15e22e691d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 22:44:49 -0700 Subject: [PATCH 32/53] fix(windows): the replay witness counts each printed line once, and runs on the package's PATH Run 43561333340 (9698922), Windows CLI job: "2 random units failed, but 4 Replay lines printed" (vitest repeats a failing unit's error text, so its replay lines repeat), and every line failed with "'vitest' is not recognized" (the rewritten runIn dropped the PATH that adds the package's node_modules/.bin). - Each distinct printed line counts once, in first-occurrence order. - Every shell runs with the package's binaries on PATH. Evidence block - Law: a printed command replays in each shell it names (the register's per-shell rule); each failing unit prints one line (witness rule). - RED: run 43561333340, the witness: "4 Replay lines printed" for 2 random units; "'vitest' is not recognized". - GREEN: pending the next Windows run. - Mutant: not run. - Limits: as in the previous commit (Rerun lines are not exact). Co-authored-by: Isaac --- packages/oracles/scripts/windows-witness.mjs | 24 +++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/packages/oracles/scripts/windows-witness.mjs b/packages/oracles/scripts/windows-witness.mjs index 9451c5f..a1dd59b 100644 --- a/packages/oracles/scripts/windows-witness.mjs +++ b/packages/oracles/scripts/windows-witness.mjs @@ -101,10 +101,13 @@ if (command === `done`) { lines .filter((text) => text.includes(label)) .map((text) => text.slice(text.indexOf(label) + label.length)) - const replayPs = after(`Replay (PowerShell): `) - const replayCmd = after(`Replay (cmd): `) - const rerunPs = after(`Rerun (PowerShell): `) - const rerunCmd = after(`Rerun (cmd): `) + // Vitest repeats a failing unit's error text, so its replay lines repeat too: + // each distinct line counts once, in the order it first printed. + const once = (xs) => [...new Set(xs)] + const replayPs = once(after(`Replay (PowerShell): `)) + const replayCmd = once(after(`Replay (cmd): `)) + const rerunPs = once(after(`Rerun (PowerShell): `)) + const rerunCmd = once(after(`Rerun (cmd): `)) const noForm = `use the PowerShell line for this name` // The failing units, as vitest prints them: `FAIL > > `. const failing = new Map() @@ -151,6 +154,15 @@ if (command === `done`) { const match = line.match(new RegExp(`set "${name}=([^"]*)" &&`)) return match === null ? undefined : match[1] } + // The package's own binaries are on the PATH, so `vitest` runs as the line says. + const env = { + ...process.env, + PATH: [ + resolve(here, `../node_modules/.bin`), + join(dir, `node_modules`, `.bin`), + process.env.PATH, + ].join(delimiter), + } const runIn = (shell, line) => { // PowerShell reads the line from stdin (`-Command -`), as a typed line // arrives: an argument would pass through Node's quoting. cmd reads its line @@ -158,7 +170,7 @@ if (command === `done`) { if (shell === `cmd`) return spawnSync(`cmd`, [`/d`, `/s`, `/c`, line], { cwd: dir, - env: process.env, + env, encoding: `utf8`, windowsVerbatimArguments: true, }) @@ -167,7 +179,7 @@ if (command === `done`) { [`-NoProfile`, `-Command`, `-`], { cwd: dir, - env: process.env, + env, encoding: `utf8`, input: line, }, From 0c42c33ac0e45bc820bd990f5a199e210501d45f Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 23:09:26 -0700 Subject: [PATCH 33/53] fix(windows): the replay witness pairs each printed line with its cmd line and checks coverage Run 43563021693 (257a5b0): "2 random units failed, but 4 Replay lines printed" and "Replay 4 has no cmd line": a failing fixed unit prints its replay under the random unit's name, so one failing unit can have more than one replay line, and index pairing breaks. A random unit's violation also prints its replay under the Rerun label, so a Rerun line can carry ORACLE_ variables, and running it with `vitest run` in front broke it. - Each distinct PowerShell line pairs with the cmd line printed after it. - A line that carries ORACLE_ variables is a replay and runs as printed; a bare -t line is a rerun and runs as `vitest run` plus its arguments. - Coverage: every failing unit is matched by a replay whose ORACLE_PROPERTY and ORACLE_UNIT decode to its names (a fixed unit by the random unit it replays). Evidence block - Law: every failing unit prints a replay that names it exactly, and each printed command replays in each shell it names (register: deterministic replay; the witness rule). - RED: run 43563021693: "2 random units failed, but 4 Replay lines printed"; "Replay 4 has no cmd line"; "Rerun 2 did not reproduce in cmd". - GREEN: pending the next Windows run. - Mutant: not run. - Limits: a bare Rerun line is not exact (see F3). Co-authored-by: Isaac --- packages/oracles/scripts/windows-witness.mjs | 117 ++++++++----------- 1 file changed, 50 insertions(+), 67 deletions(-) diff --git a/packages/oracles/scripts/windows-witness.mjs b/packages/oracles/scripts/windows-witness.mjs index a1dd59b..0b7a55c 100644 --- a/packages/oracles/scripts/windows-witness.mjs +++ b/packages/oracles/scripts/windows-witness.mjs @@ -103,57 +103,38 @@ if (command === `done`) { .map((text) => text.slice(text.indexOf(label) + label.length)) // Vitest repeats a failing unit's error text, so its replay lines repeat too: // each distinct line counts once, in the order it first printed. - const once = (xs) => [...new Set(xs)] - const replayPs = once(after(`Replay (PowerShell): `)) - const replayCmd = once(after(`Replay (cmd): `)) - const rerunPs = once(after(`Rerun (PowerShell): `)) - const rerunCmd = once(after(`Rerun (cmd): `)) - const noForm = `use the PowerShell line for this name` - // The failing units, as vitest prints them: `FAIL > > `. - const failing = new Map() - for (const text of lines) { - const match = text.match( - /FAIL\s+\S+ > (.*?) > ((?:random|fixed) seed: .*)$/, + // Each printed PowerShell line pairs with the cmd line that follows it (the + // framework prints the two together). A random unit's violation prints its + // replay under the Rerun label too, so a line is a replay when it carries the + // ORACLE_ variables, and runs as printed; a bare -t line is a rerun and runs as + // `vitest run` plus its arguments. Each distinct PowerShell line counts once. + const pairs = [] + const seen = new Set() + lines.forEach((text, at) => { + const head = text.match(/(Replay|Rerun) \(PowerShell\): (.*)$/) + if (head === null || seen.has(head[2])) return + seen.add(head[2]) + const cmdAt = lines.findIndex( + (other, j) => + j > at && + j <= at + 6 && + new RegExp(`${head[1]} \\(cmd\\): `).test(other), ) - if (match) - failing.set(`${match[1]}\0${match[2]}`, { - property: match[1], - unit: match[2], - }) - } - const failingRandom = [...failing.values()].filter((f) => - f.unit.startsWith(`random seed: `), - ) - const failingFixed = [...failing.values()].filter((f) => - f.unit.startsWith(`fixed seed `), - ) - // The PowerShell value of $env:NAME='...', decoded: a doubled quote (of any - // PowerShell single-quote character) is that character. - const decodePs = (line, name) => { - const head = `$env:${name}='` - const at = line.indexOf(head) - if (at < 0) return undefined - let value = `` - for (let i = at + head.length; i < line.length; i += 1) { - const c = line[i] - if (/['\u2018\u2019\u201A\u201B]/.test(c)) { - if (line[i + 1] === c) { - value += c - i += 1 - continue - } - return value - } - value += c - } - return undefined - } - // The cmd value of set "NAME=value". A value with a double quote, `%`, or `!` - // has no cmd form, so its line says so and no set line is decoded. - const decodeSet = (line, name) => { - const match = line.match(new RegExp(`set "${name}=([^"]*)" &&`)) - return match === null ? undefined : match[1] - } + const cmd = + cmdAt < 0 + ? undefined + : lines[cmdAt].replace(new RegExp(`^.*${head[1]} \\(cmd\\): `), ``) + pairs.push({ + kind: head[1], + ps: head[2], + cmd, + replay: head[2].includes(`$env:`), + }) + }) + const replayPs = pairs.filter((pair) => pair.replay).map((pair) => pair.ps) + const replayCmd = pairs.filter((pair) => pair.replay).map((pair) => pair.cmd) + const rerunPs = pairs.filter((pair) => !pair.replay).map((pair) => pair.ps) + const rerunCmd = pairs.filter((pair) => !pair.replay).map((pair) => pair.cmd) // The package's own binaries are on the PATH, so `vitest` runs as the line says. const env = { ...process.env, @@ -213,26 +194,28 @@ if (command === `done`) { failures.push(`${name} ${i + 1} did not reproduce in cmd: ${cmdLine}`) }) } - // Every failing unit prints its line: a random unit a Replay line, a fixed - // unit a Rerun line (review: a failure with no per-shell line fails here). - if (replayPs.length !== failingRandom.length) - failures.push( - `${failingRandom.length} random units failed, but ${replayPs.length} Replay lines printed`, - ) - if (rerunPs.length !== failingFixed.length) - failures.push( - `${failingFixed.length} fixed units failed, but ${rerunPs.length} Rerun lines printed`, - ) - // A Replay line's ORACLE_PROPERTY and ORACLE_UNIT must be exactly a failing - // unit's names: decoded, and equal (not contained). - const names = new Set(failingRandom.map((f) => `${f.property}\0${f.unit}`)) + // Every failing unit is covered by a printed replay whose ORACLE_PROPERTY and + // ORACLE_UNIT decode to exactly its names: a random unit by its own unit name, + // a fixed unit by the random unit it replays (`random seed: `). + const replayNames = new Set( + replayPs.map( + (line) => + `${decodePs(line, `ORACLE_PROPERTY`)}\0${decodePs(line, `ORACLE_UNIT`)}`, + ), + ) + for (const f of failing.values()) { + const unit = f.unit.startsWith(`random seed: `) + ? f.unit + : `random seed: ${f.unit.replace(/^fixed seed \d+: /, ``)}` + if (!replayNames.has(`${f.property}\0${unit}`)) + failures.push( + `no printed replay names the failing unit ${JSON.stringify([f.property, f.unit])}`, + ) + } + // A Replay line's cmd line sets the same names. replayPs.forEach((line, i) => { const property = decodePs(line, `ORACLE_PROPERTY`) const unit = decodePs(line, `ORACLE_UNIT`) - if (!names.has(`${property}\0${unit}`)) - failures.push( - `Replay ${i + 1} names ${JSON.stringify([property, unit])}, not a failing unit's names`, - ) const cmdLine = replayCmd[i] if (cmdLine !== undefined && cmdLine !== noForm) { const cmdProperty = decodeSet(cmdLine, `ORACLE_PROPERTY`) From 08f028db2f5e14b75dcd0e8c279d7d0dfa91d538 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 23:36:12 -0700 Subject: [PATCH 34/53] fix(windows): restore the replay witness's helpers that the pairing rewrite removed Run 0c42c33 (the Windows CLI job): "ReferenceError: decodePs is not defined". The pairing rewrite replaced the block that held the FAIL map, the PowerShell and cmd decoders, and the no-form constant. They are restored, and a local smoke run of the replay branch on a fabricated output reaches the report (pwsh is absent on this Mac, so its runs fail as expected; no reference error). Evidence block - Law: the replay witness reports a failing unit's printed commands (witness rule). - RED: run 0c42c33: ReferenceError in the replay step. - GREEN: pending the next Windows run. - Mutant: not run. - Limits: as before. Co-authored-by: Isaac --- packages/oracles/scripts/windows-witness.mjs | 41 ++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/packages/oracles/scripts/windows-witness.mjs b/packages/oracles/scripts/windows-witness.mjs index 0b7a55c..b23d121 100644 --- a/packages/oracles/scripts/windows-witness.mjs +++ b/packages/oracles/scripts/windows-witness.mjs @@ -135,6 +135,47 @@ if (command === `done`) { const replayCmd = pairs.filter((pair) => pair.replay).map((pair) => pair.cmd) const rerunPs = pairs.filter((pair) => !pair.replay).map((pair) => pair.ps) const rerunCmd = pairs.filter((pair) => !pair.replay).map((pair) => pair.cmd) + // The failing units, as vitest prints them: `FAIL > > `. + const failing = new Map() + for (const text of lines) { + const match = text.match( + /FAIL\s+\S+ > (.*?) > ((?:random|fixed) seed: .*)$/, + ) + if (match) + failing.set(`${match[1]}\0${match[2]}`, { + property: match[1], + unit: match[2], + }) + } + // The PowerShell value of $env:NAME='...', decoded: a doubled quote (of any + // PowerShell single-quote character) is that character. + const decodePs = (line, name) => { + const head = `$env:${name}='` + const at = line.indexOf(head) + if (at < 0) return undefined + let value = `` + for (let i = at + head.length; i < line.length; i += 1) { + const c = line[i] + if (/['\u2018\u2019\u201A\u201B]/.test(c)) { + if (line[i + 1] === c) { + value += c + i += 1 + continue + } + return value + } + value += c + } + return undefined + } + // The cmd value of set "NAME=value". A value with a double quote, `%`, or `!` + // has no cmd form, so its line says so and no set line is decoded. + const decodeSet = (line, name) => { + const match = line.match(new RegExp(`set "${name}=([^"]*)" &&`)) + return match === null ? undefined : match[1] + } + // The line a cmd line prints when its words have no cmd form. + const noForm = `use the PowerShell line for this name` // The package's own binaries are on the PATH, so `vitest` runs as the line says. const env = { ...process.env, From bd208ac49ae982ff760421fb9dcbe3259f1de09a Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 00:07:29 -0700 Subject: [PATCH 35/53] fix(windows): a Rerun line counts only for its own failing fixed unit (F3 ruling b) The coordinator's ruling: Rerun lines stay coarse -t filters (a sibling whose name the anchored pattern matches also runs). The witness now counts a Rerun line only when the failing fixed unit whose exact name it selects shows its own FAIL line in its output, so a sibling's failure cannot satisfy it. An exact selector is tracked in #78; Replay lines select exactly, through ORACLE_PROPERTY and ORACLE_UNIT. Evidence block - Law: a Rerun line runs the failing unit it names, and only it counts (register amendment: Replay lines select exactly; Rerun lines are anchored, coarse filters). - Reach: the Windows fixtures (the `Q"1` unit has no Rerun line in CI; the sibling case is the Limit's example). - RED: none in CI yet; the sibling case is not exercised by the fixtures. - GREEN: pending the next Windows run. - Mutant: not run. - Limits: a sibling case is not in the CI fixtures (recorded in the PR body). Co-authored-by: Isaac --- packages/oracles/scripts/windows-witness.mjs | 40 +++++++++++++++++++- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/packages/oracles/scripts/windows-witness.mjs b/packages/oracles/scripts/windows-witness.mjs index b23d121..086ffa9 100644 --- a/packages/oracles/scripts/windows-witness.mjs +++ b/packages/oracles/scripts/windows-witness.mjs @@ -213,12 +213,48 @@ if (command === `done`) { // A Rerun line names the test only: it runs as `vitest run `. const failures = [] let checked = 0 + // The -t argument of a PowerShell line, decoded (a PowerShell single-quoted + // value): the coarse filter a Rerun line applies to test names. + const tArgument = (line) => { + const at = line.indexOf(`-t '`) + if (at < 0) return undefined + return decodePs(`$env:X='${line.slice(at + 4)}`, `X`) + } + // The failing fixed units a Rerun line's filter selects, by exact full name, + // and whose own FAIL line its output must show. + const rerunTargets = (line) => { + const pattern = tArgument(line) + if (pattern === undefined) return undefined + let filter + try { + filter = new RegExp(pattern) + } catch { + return undefined + } + return [...failing.values()].filter( + (f) => + f.unit.startsWith(`fixed seed `) && + filter.test(`${f.property} > ${f.unit}`), + ) + } const runEach = (name, psLines, cmdLines, prefix) => { psLines.forEach((line, i) => { + const targets = name === `Rerun` ? rerunTargets(line) : undefined + if (name === `Rerun` && (targets === undefined || targets.length === 0)) { + failures.push(`Rerun ${i + 1} selects no failing fixed unit: ${line}`) + return + } + const shownBy = (result) => + targets === undefined || + targets.every((f) => + `${result.stdout}${result.stderr}`.includes( + `${f.property} > ${f.unit}`, + ), + ) for (const shell of [`pwsh`, `powershell`]) { checked += 1 const result = runIn(shell, `${prefix}${line}`) - if (!reproduces(result)) + if (!reproduces(result) || !shownBy(result)) failures.push( `${name} ${i + 1} did not reproduce in ${shell} (exit ${result.status}: ${`${result.stderr}${result.stdout}`.trim().slice(0, 400)}): ${line}`, ) @@ -231,7 +267,7 @@ if (command === `done`) { if (cmdLine === noForm) return checked += 1 const result = runIn(`cmd`, `${prefix}${cmdLine}`) - if (!reproduces(result)) + if (!reproduces(result) || !shownBy(result)) failures.push(`${name} ${i + 1} did not reproduce in cmd: ${cmdLine}`) }) } From 3b97795df311ebc201b4eb18e99992f706c2350b Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 00:32:18 -0700 Subject: [PATCH 36/53] fix(windows): the witness reads a fixed unit's FAIL line (`fixed seed 1: `) Run 43569602268 (bd208ac): "Rerun 1 selects no failing fixed unit" for both fixtures. The FAIL regex expected `fixed seed: ` with no number, but vitest prints `fixed seed 1: `, so no fixed unit was recorded. (The earlier fix attempt did not apply; this commit applies it and checks the match.) Evidence block - Law: a Rerun line counts only for the failing fixed unit whose exact name it selects (witness rule; register: Rerun lines are anchored coarse filters). - RED: run 43569602268: "Rerun 1 selects no failing fixed unit". - GREEN: pending the next Windows run. - Mutant: not run. - Limits: as recorded in the PR body (sibling case, #78). Co-authored-by: Isaac --- packages/oracles/scripts/windows-witness.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/oracles/scripts/windows-witness.mjs b/packages/oracles/scripts/windows-witness.mjs index 086ffa9..d154375 100644 --- a/packages/oracles/scripts/windows-witness.mjs +++ b/packages/oracles/scripts/windows-witness.mjs @@ -139,7 +139,7 @@ if (command === `done`) { const failing = new Map() for (const text of lines) { const match = text.match( - /FAIL\s+\S+ > (.*?) > ((?:random|fixed) seed: .*)$/, + /FAIL\s+\S+ > (.*?) > ((?:random seed: |fixed seed \d+: ).*)$/, ) if (match) failing.set(`${match[1]}\0${match[2]}`, { From 7d46701096ddf4399997dba357f60f6a529988a5 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 00:55:43 -0700 Subject: [PATCH 37/53] fix(windows): the Rerun name check ignores vitest's colors Run 43571581118 (3b97795): each Rerun line reproduced (exit 1, "Failed Tests 1"), but the check that the failing unit's exact name appears in its output failed: vitest colors the output, so the name is split by color codes. The witness now strips the color codes before it matches a unit's name. Evidence block - Law: a Rerun line counts for the failing fixed unit whose exact name it selects and whose FAIL line its output shows (witness rule). - RED: run 43571581118: "Rerun 1 did not reproduce in pwsh (exit 1 ... Failed Tests 1)" with the name check failing. - GREEN: pending the next Windows run. - Mutant: not run. - Limits: as recorded in the PR body (sibling case, #78). Co-authored-by: Isaac --- packages/oracles/scripts/windows-witness.mjs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/oracles/scripts/windows-witness.mjs b/packages/oracles/scripts/windows-witness.mjs index d154375..aab5b0f 100644 --- a/packages/oracles/scripts/windows-witness.mjs +++ b/packages/oracles/scripts/windows-witness.mjs @@ -244,13 +244,15 @@ if (command === `done`) { failures.push(`Rerun ${i + 1} selects no failing fixed unit: ${line}`) return } - const shownBy = (result) => - targets === undefined || - targets.every((f) => - `${result.stdout}${result.stderr}`.includes( - `${f.property} > ${f.unit}`, - ), + // Vitest colors its output: the names are matched with the colors removed. + const shownBy = (result) => { + if (targets === undefined) return true + const text = `${result.stdout}${result.stderr}`.replace( + /\x1b\[[0-9;]*m/g, + ``, ) + return targets.every((f) => text.includes(`${f.property} > ${f.unit}`)) + } for (const shell of [`pwsh`, `powershell`]) { checked += 1 const result = runIn(shell, `${prefix}${line}`) From f4bc85c7e0ae6900e1a931fee539d7da0251fcee Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 02:36:03 -0700 Subject: [PATCH 38/53] fix(oracle-gaps): the e2e replay check reads and runs the PowerShell line on win32 Class 1 of the Windows static and e2e failures (merged head 5186cd5): the e2e oracle's replay check read the POSIX ` replay: ` line and ran it with `sh`. On Windows the CLI prints ` replay (PowerShell): ` (and a cmd line), so the replay command was undefined and the ownership law failed ("the replay command undefined ran undefined, owed only ..."; 3 violations). The check now reads the PowerShell line on win32 and runs it through pwsh on stdin, as a typed line runs; the comparison of the tests it ran is unchanged. Evidence block - Law: a baseline's replay command, pasted into the shell it names, runs the owed test (the ownership law; the register's per-shell rule). - Reach: the e2e histories with a failing owner (the baseline replays). - RED: run 43575409066 / 5186cd5: "the replay command undefined ran undefined, owed only [...]" (3 e2e violations, law ownership). - GREEN: pending the next Windows run. - Mutant: the POSIX-only reader is the mutant (the RED above). - Limits: the stdout gap-line count class (greppable stream) is not addressed here. Co-authored-by: Isaac --- .../gap-report-oracle.property.test.mjs | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 6d5a748..2a47ad8 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -483,7 +483,7 @@ import { writeFileSync, } from 'node:fs' import { tmpdir } from 'node:os' -import { dirname, join, relative, resolve } from 'node:path' +import { delimiter, dirname, join, relative, resolve } from 'node:path' import { fileURLToPath, pathToFileURL } from 'node:url' import fc from 'fast-check' import { socketOf, stateDirOf } from './daemon.mjs' @@ -663,10 +663,13 @@ function runCli(dir, files, how = {}) { child.stdout, ) // The command the baseline printed to replay a failing owner (law 3). + // On win32 the CLI prints the PowerShell line (its POSIX form is not printed). + const label = + process.platform === `win32` ? ` replay (PowerShell): ` : ` replay: ` const replay = `${child.stderr ?? ``}` .split(`\n`) - .find((line) => line.startsWith(` replay: `)) - ?.slice(` replay: `.length) + .find((line) => line.startsWith(label)) + ?.slice(label.length) return { ...run, strayWrites: strays(dir, before), replay } } @@ -852,10 +855,22 @@ function replayed(dir, command) { const scratch = mkdtempSync(join(realpathSync(tmpdir()), `gap-replay-`)) try { const report = join(scratch, `report.json`) - spawnSync( - `sh`, - [`-c`, `${command} --reporter=json --outputFile='${report}'`], - { + const full = `${command} --reporter=json --outputFile='${report}'` + // On win32 the line runs in PowerShell, as a user's typed line does: on stdin. + if (process.platform === `win32`) + spawnSync(`pwsh`, [`-NoProfile`, `-Command`, `-`], { + cwd: dir, + encoding: `utf8`, + input: full, + env: { + ...process.env, + PATH: [join(here, `../node_modules/.bin`), process.env.PATH].join( + delimiter, + ), + }, + }) + else + spawnSync(`sh`, [`-c`, full], { cwd: dir, encoding: `utf8`, timeout: 120_000, @@ -863,8 +878,7 @@ function replayed(dir, command) { ...cleanEnv(), PATH: `${join(here, `..`, `node_modules`, `.bin`)}:${process.env.PATH}`, }, - }, - ) + }) if (!existsSync(report)) return [] return JSON.parse(readFileSync(report, `utf8`)) .testResults.flatMap((file) => file.assertionResults) From dd3347a2a73ddfd850f051d1587266ca6a42447e Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 03:04:50 -0700 Subject: [PATCH 39/53] test(oracle-gaps): the greppable-stream breach names the stdout's gap lines A failure reported only the count of the stdout's gap lines, so a Windows run's extra lines could not be read from CI. The breach now carries the lines themselves. Co-authored-by: Isaac --- .../oracle-gaps/gap-report-oracle.property.test.mjs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 2a47ad8..7e1bd23 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -1094,7 +1094,9 @@ function readRun(dir, out, status, stderr, cli, stdout) { lastLine: printed.at(-1) && namedScratch(printed.at(-1), scratch), // The stdout's own lines of the stream's kinds: the gap lines under // their functions, as the stream prints them (law 15). - stdoutGaps: printed.filter((line) => streamedGap.test(line)).length, + stdoutGaps: printed + .filter((line) => streamedGap.test(line)) + .map((line) => namedScratch(line, scratch)), baselineFiles: lines.flatMap((line) => { const file = line.match( /^oracle-gaps: baseline (?:pass|fail) (\S+)(?: \(ordinary\))?: /, @@ -2575,9 +2577,9 @@ function streamBreach(checkpoint, run, expected, plan) { `the stream's decisive baseline lines are ${JSON.stringify(run.stream.verdicts)}, owed ${JSON.stringify([verdict])}`, { actual: run.stream.verdicts, expected: [verdict] }, ) - if (run.stdoutGaps !== expected.gaps.length) + if (run.stdoutGaps.length !== expected.gaps.length) return breach( - `the stdout has ${run.stdoutGaps} gap lines for ${expected.gaps.length} gaps`, + `the stdout has ${run.stdoutGaps.length} gap lines for ${expected.gaps.length} gaps: ${JSON.stringify(run.stdoutGaps)}`, ) return undefined } From 872194623782cd9f7a4349655dfe9e4bda99b62c Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 03:24:17 -0700 Subject: [PATCH 40/53] test(oracle-gaps): the greppable-stream breach names the stderr's gap lines too Co-authored-by: Isaac --- .../oracles/oracle-gaps/gap-report-oracle.property.test.mjs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 7e1bd23..d6b5de0 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -1097,6 +1097,9 @@ function readRun(dir, out, status, stderr, cli, stdout) { stdoutGaps: printed .filter((line) => streamedGap.test(line)) .map((line) => namedScratch(line, scratch)), + stderrGaps: lines + .filter((line) => streamedGap.test(line)) + .map((line) => namedScratch(line, scratch)), baselineFiles: lines.flatMap((line) => { const file = line.match( /^oracle-gaps: baseline (?:pass|fail) (\S+)(?: \(ordinary\))?: /, @@ -2579,7 +2582,7 @@ function streamBreach(checkpoint, run, expected, plan) { ) if (run.stdoutGaps.length !== expected.gaps.length) return breach( - `the stdout has ${run.stdoutGaps.length} gap lines for ${expected.gaps.length} gaps: ${JSON.stringify(run.stdoutGaps)}`, + `the stdout has ${run.stdoutGaps.length} gap lines for ${expected.gaps.length} gaps: ${JSON.stringify(run.stdoutGaps)}; the stderr has ${JSON.stringify(run.stderrGaps)}`, ) return undefined } From f00c2c77410573ae998d01ddc7f11aa55596954b Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 03:43:41 -0700 Subject: [PATCH 41/53] fix(oracle-gaps): invalidate a patched file by its forward-slash path on win32 An authored mutant is served by patching a file in a warm context, which invalidates the file first. Vite keys its module map by forward-slash paths, so invalidateFile given a backslash path found no module and the context kept the original module. The authored mutant then survived as not-observed on Windows, where the model expected it killed (the e2e shards' 'greppable stream' breaches: authored:a1 and authored:a2 on sut-less fixtures, in run 43585144306). posix() is a no-op off win32. Co-authored-by: Isaac --- packages/oracles/oracle-gaps/check.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/oracles/oracle-gaps/check.mjs b/packages/oracles/oracle-gaps/check.mjs index 9ca8b5b..af9e62a 100644 --- a/packages/oracles/oracle-gaps/check.mjs +++ b/packages/oracles/oracle-gaps/check.mjs @@ -778,7 +778,7 @@ export default async (env) => { for (const path of paths) { const hash = servedHash(path, index) if (served.get(path) === hash) continue - context.invalidateFile(path) + context.invalidateFile(posix(path)) served.set(path, hash) } } @@ -1457,7 +1457,7 @@ export default async (env) => { const hash = started.get(path) ?? now const read = served.get(path) ?? hash // The context may hold other text than the path's text now. - if (read !== now || hash !== now) context.invalidateFile(path) + if (read !== now || hash !== now) context.invalidateFile(posix(path)) if (hash !== now) stable = false served.set(path, now) return [path, hash] From b6085b8f22820e1019f09d6de32d77c5aec2ff3a Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 04:04:29 -0700 Subject: [PATCH 42/53] fix(oracle-gaps): the CLI-directory reach state compares paths in one spelling The refused run whose package lies inside the CLI's directory was tested with a string prefix of the CLI's directory plus a forward slash. On win32 the package's path is joined with backslashes, so the prefix never matched and the state was never reached (RED: run 43586666931, e2e shard 6/6, 'did not enter a refused run whose package lies inside the CLI's directory'). isWithin compares both paths in forward slashes. Co-authored-by: Isaac --- .../oracles/oracle-gaps/gap-report-oracle.property.test.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index d6b5de0..10687fc 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -487,6 +487,7 @@ import { delimiter, dirname, join, relative, resolve } from 'node:path' import { fileURLToPath, pathToFileURL } from 'node:url' import fc from 'fast-check' import { socketOf, stateDirOf } from './daemon.mjs' +import { isWithin } from './paths.mjs' import { oracleProperty, pass, violation } from '../src/oracle-property.ts' import { @@ -1724,7 +1725,7 @@ async function driver(plan, observe) { // A refused run's output is recorded with its scratch paths named, and // this package lies inside the CLI's directory: the case in which the // naming order decides whether a random name stays in the trace. - if (cli.exit !== 0 && dir.startsWith(`${how.cli}/`)) + if (cli.exit !== 0 && dir !== how.cli && isWithin(how.cli, dir)) states.add(`a refused run whose package lies inside the CLI's directory`) // A failing owner's replay printed in a package whose config only // `--config` names, beside a discovered config that includes no test. From 0f9c1a5d3961f39fba58e8af81c270dc05c39821 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 04:05:51 -0700 Subject: [PATCH 43/53] fix(oracle-gaps): the static driver names the package in both path spellings The CLI prints its paths in forward slashes, but the static driver named only the package's join-spelled path. The scratch directory's random name stayed in the forward-slash lines, so two runs of one constructed history recorded different arguments traces on Windows (RED: run 43586666931, static shards, 'two runs ... recorded different traces at observation arguments'). The e2e driver already names both spellings. Co-authored-by: Isaac --- .../oracle-gaps/gap-report-static-oracle.property.test.mjs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs index 99251a4..5ccab44 100644 --- a/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs @@ -135,6 +135,7 @@ import { written, } from './gap-report-model-oracle.mjs' import { parseArgs, usage } from './args.mjs' +import { posix } from './paths.mjs' import { keyAndSelect } from './report.mjs' const here = dirname(fileURLToPath(import.meta.url)) @@ -505,7 +506,10 @@ function argumentsRunOf(dir, plan) { run.status, run.stderr, out, - (text) => text.replaceAll(dir, ``), + // The CLI names its paths in forward slashes (the Windows paths rule), so + // the package is named in both spellings. + (text) => + text.replaceAll(posix(dir), ``).replaceAll(dir, ``), run.stdout, ) } From 957a060c99eda6e097fd91c7c1f2acd19703b7c7 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 04:25:39 -0700 Subject: [PATCH 44/53] ci: the Windows static oracle job blocks the preview All six shards of the Windows static oracle are green (run 43588402680), each under 3 minutes, so the job no longer continues on error and the preview waits for it. The end-to-end job stays non-blocking: two of its shards cannot pass on Windows yet (a held daemon cannot be paused, and a stop by SIGTERM or SIGINT cannot be delivered), which needs a maintainer decision on the declared reach. Co-authored-by: Isaac --- .github/workflows/pr.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 8cbab1b..ca16020 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -168,10 +168,11 @@ jobs: - test-oracle-gaps-fast - test-oracle-gaps-end-to-end - test-vitest-5 - # The blocking Windows jobs (the non-blocking static and e2e jobs are not - # a dependency until they are green). + # The blocking Windows jobs (the non-blocking end-to-end job is not a + # dependency until it is green). - windows-unit - windows-gap-classify + - windows-gap-oracles - windows-gap-cli runs-on: ubuntu-latest steps: @@ -223,10 +224,6 @@ jobs: windows-gap-oracles: name: Windows gap report static oracle (${{ matrix.shard }}/6) runs-on: windows-latest - # Not blocking yet: these suites fail on Windows, tracked by class in - # https://github.com/TanStack/oracles/issues/72. Remove this line when they - # are green. - continue-on-error: true timeout-minutes: 20 strategy: fail-fast: false From 8fb0b00a1ff4512ed64ac96598eac945a935e8cf Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 04:50:05 -0700 Subject: [PATCH 45/53] fix(oracle-gaps): the daemon pause and win32 stops do not need signals that win32 lacks A held daemon is paused by a test-only pause at the daemon's reply (ORACLE_GAPS_TEST_PAUSE_AT=daemon-reply), which writes its pid to the check's output directory, instead of SIGSTOP (no signal pauses a process on win32). The pause fires once per check, so a daemon the CLI restarts after the death answers. On win32 a stop goes through the CLI's test stop hook (ORACLE_GAPS_TEST_STOP) and is observed as SIGINT. The SIGTERM stop state is required on POSIX only: win32 cannot deliver SIGTERM (its kill is TerminateProcess). Limits states the reason. RED (Windows): e2e shard 1/6 run 43588402680, 'a check whose daemon held it past the CLI's wait, then died' not reached; shard 4/6, 'every outcome class' sent SIGTERM and stderr had no done line. GREEN (macOS, local): 'an edit sequence through the daemon' and 'every outcome class' pass (fixed seed 20261105). Co-authored-by: Isaac --- packages/oracles/oracle-gaps/daemon.mjs | 17 +++++ .../gap-report-oracle.property.test.mjs | 70 +++++++++++++------ 2 files changed, 64 insertions(+), 23 deletions(-) diff --git a/packages/oracles/oracle-gaps/daemon.mjs b/packages/oracles/oracle-gaps/daemon.mjs index 6ea0215..a5f59e6 100644 --- a/packages/oracles/oracle-gaps/daemon.mjs +++ b/packages/oracles/oracle-gaps/daemon.mjs @@ -29,6 +29,7 @@ import { spawn } from 'node:child_process' import { randomUUID } from 'node:crypto' import { + existsSync, linkSync, mkdirSync, openSync, @@ -370,6 +371,22 @@ async function serve(pkg) { stderr.push(`${line}\n`), ) await store.flush() + // A test-only pause (ORACLE_GAPS_TEST_PAUSE_AT=daemon-reply, as + // ORACLE_GAPS_TEST_STOP): the check is done and its reply is held, + // so the CLI waits as it would for a daemon that is slow and dies. + // The pause is marked in the check's output directory, with this + // daemon's pid, for the oracle to kill it by (no signal pauses a + // process on win32). + // Once: a daemon that the CLI restarts after this one died inherits the + // variable, and must answer, or the CLI waits as long as the run allows. + const mark = join(options.out, `paused`) + if ( + process.env.ORACLE_GAPS_TEST_PAUSE_AT === `daemon-reply` && + !existsSync(mark) + ) { + writeFileSync(mark, `${process.pid}`) + await new Promise(() => {}) + } } catch (error) { failed = sent(error) } diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 10687fc..200622f 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -723,6 +723,11 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { writeFixture(dir, files) const before = new Set(filesUnder(dir)) const pids = mkdtempSync(join(realpathSync(tmpdir()), `gap-stop-`)) + // win32 cannot deliver a signal to a Node process: kill ends it at once, with + // no stopped line. There the stop is the CLI's test stop hook (the file it + // names, as a SIGINT), so the run is observed as SIGINT (see below). + const win = process.platform === `win32` + const stopFile = `${pids}-stop` const { args } = cliArgs(dir, files, { ...how, outName: `out-stop`, @@ -730,15 +735,17 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { only: ids, pool: abort ? `threads` : pool, }) + const env = after + ? { ...cleanEnv(), ORACLE_GAPS_TEST_HOLD_EXIT_MS: `500` } + : warm + ? { ...cleanEnv(), ORACLE_GAPS_TEST_WARM_PROBE: join(pids, `warming`) } + : abort + ? { ...cleanEnv(), GAP_ABORT: `1` } + : { ...cleanEnv(), GAP_STOP_DIR: pids } + if (win) env.ORACLE_GAPS_TEST_STOP = stopFile const child = spawn(process.execPath, args, { cwd: repo, - env: after - ? { ...cleanEnv(), ORACLE_GAPS_TEST_HOLD_EXIT_MS: `500` } - : warm - ? { ...cleanEnv(), ORACLE_GAPS_TEST_WARM_PROBE: join(pids, `warming`) } - : abort - ? { ...cleanEnv(), GAP_ABORT: `1` } - : { ...cleanEnv(), GAP_STOP_DIR: pids }, + env, stdio: [`ignore`, `pipe`, `pipe`], }) let stdout = `` @@ -785,7 +792,10 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { ? existsSync(join(pids, `warming`)) && !exited : readdirSync(pids).length >= 2 const strayWhileRunning = strays(dir, before, true) - if (!abort) child.kill(signal) + if (!abort) { + if (win) writeFileSync(stopFile, ``) + else child.kill(signal) + } const ended = await Promise.race([exit, sleep(30_000).then(() => false)]) await sleep(1000) const alive = readdirSync(pids) @@ -809,8 +819,10 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { } if (!ended) child.kill(`SIGKILL`) rmSync(pids, { recursive: true, force: true }) + rmSync(stopFile, { force: true }) return { - signal, + // The signal the CLI took: the hook's SIGINT on win32. + signal: win ? `SIGINT` : signal, pool, spun, ended, @@ -1490,35 +1502,39 @@ async function incremental(dir, plan, cli, copy, observe, states) { { ...how, daemon: true, - env: step === `dieMidCheck` ? { ORACLE_GAPS_WAIT_MS: `3000` } : {}, + env: + step === `dieMidCheck` + ? { + ORACLE_GAPS_WAIT_MS: `3000`, + ORACLE_GAPS_TEST_PAUSE_AT: `daemon-reply`, + } + : {}, }, ]) - let paused = false + let held = false if (step === `resetBeforeRead`) { await new Promise((r) => setTimeout(r, 500)) if (pid) process.kill(pid, `SIGKILL`) } else { // Mid-check: the daemon has written this check's state file. A check // that ends first (a refusal runs nothing) leaves its daemon alone. - const started = join(dir, `out-daemon`, `state.json`) + const pauseMark = join(dir, `out-daemon`, `paused`) let finished = false running.then(() => { finished = true }) for (let waited = 0; !finished && waited < 60_000; waited += 50) { - const [running] = daemonsOf(dir) - if (running && existsSync(started)) { - process.kill(running, `SIGSTOP`) + if (existsSync(pauseMark)) { await new Promise((r) => setTimeout(r, 4000)) - process.kill(running, `SIGKILL`) - paused = !finished + process.kill(Number(readFileSync(pauseMark, `utf8`)), `SIGKILL`) + held = !finished break } await new Promise((r) => setTimeout(r, 50)) } } const runs = await running - if (paused && runs[0].reported) + if (held && runs[0].reported) states.add(`a check whose daemon held it past the CLI's wait, then died`) return runs } @@ -1706,7 +1722,7 @@ async function driver(plan, observe) { states.add(`a run whose process aborted`) continue } - states.add(`a run stopped by ${stop.signal} while a worker spun`) + states.add(`a run stopped by ${stopped.signal} while a worker spun`) states.add( `a stopped run whose workers are ${stop.pool === `forks` ? `child processes` : `threads`}`, ) @@ -3196,7 +3212,6 @@ const required = [ `a hang in one owner and a survivor in another timed out`, `confirmations past a mutant's budget timed out`, `a heartbeat named a busy mutant`, - `a run stopped by SIGTERM while a worker spun`, `a run stopped by SIGINT while a worker spun`, `a stop signal after the completed line`, `a stop signal while the config warms`, @@ -3338,7 +3353,16 @@ const separate = [ `edit lazy changed the cold result`, `a lazy edit changed the cold result after a batch was cut off`, ] -const witnessed = required.filter((s) => !separate.includes(s)) +// A stop by SIGTERM exists only where Node can deliver it: on win32 kill is +// TerminateProcess, so there the stop law is judged by SIGINT (the test stop +// hook), and the SIGTERM state is not required (a declared platform bound). +const posixStops = + process.platform === `win32` + ? [] + : [`a run stopped by SIGTERM while a worker spun`] +const witnessed = [...required, ...posixStops].filter( + (s) => !separate.includes(s), +) // A constructed history builds from a generated base whose first function is // owned by file a (one owned function, not a whole file), so a random @@ -3363,7 +3387,7 @@ oracleProperty({ law: { statement: `The gap report enumerates the promised operator mutants, leaves out exactly the probe-only code, discovers every oracle that owns a function's call tree and refuses broken ownership, judges each mutant only by its owners' tests on their registered routes and counts only law verdicts on reproducible routes as kills, which outrank a run that never ends, anchors each mutant by its function and normalized statement, lists every gap and unowned mutant once with consistent counts, keeps keys stable under unrelated edits, reports through one daemon per package what a cold run reports at every state of a sequence of edits, and bounds the runs that confirm a mutant's cut-off copies by one budget per mutant and names each busy mutant in a heartbeat while it runs, leaves nothing in the package and writes nothing where Vitest's include could find it while it runs, leaves no process running once it is stopped, runs the package as Vitest does from any install path and any --source spelling: one instance of each workspace dependency, no refusal over an unbuildable devDependency, child processes for a native addon anywhere in the tree, a capture that a test's builtin spies cannot see, and a refusal of test.projects; leads its report with every test file's baseline, blocks only the mutants a failing oracle file owns and judges the rest, prints every line of its stream with the prefix oracle-gaps: and its kind, one decisive baseline line before any mutant, and a done line last, and every run that does not complete ends, cold or through the daemon, with a refusal that exits 2, prints a refused: block and one fix: directive with no stack trace, and writes the refusal to the run's output directory.`, authority: `The maintainer's decisions in design runs v11, v14, and v19 (2026-11-04 and 2026-11-05), not in this package's records, including that a kill by reach agreement alone is a gap and that an incremental run through the daemon equals a cold one (v19.3); the project's oracle guide on quiet exclusions; the oracle rubric's outcome kinds (ORC-006); the CLI's documented operator list in schemata.mjs; the maintainer's dispositions G1, G2, G4, G5, G6, and G13 of the framework papercut ledger (2026-11-11) and the v20 freeze's namespacing, one-loop, and provisional-report decisions (2026-11-10); the maintainer's request for a greppable stream (2026-11-12, relayed by the coordinator with its decisions of that day: the same prefixes on stdout and stderr, a done line for a stopped run, and no --max-gaps).`, - limits: `Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come only between checks, so a file saved during a check is not generated (the runner invalidates a known input that changed while a run ran and does not store that run, which only inspection fixes; a witness needs a step that edits a file between a unit's start and its end), and a module that a run first loads and that changes during that same run is stored under its later hash; law 9's edits leave out files read through fs, other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated.`, + limits: `On win32 Node cannot deliver SIGTERM (its kill is TerminateProcess, which ends the process with no stopped line), so the stop states that name SIGTERM are required on POSIX only, and a stop there goes through the CLI's test stop hook and is observed as SIGINT; every SIGINT state stays required on win32. Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come only between checks, so a file saved during a check is not generated (the runner invalidates a known input that changed while a run ran and does not store that run, which only inspection fixes; a witness needs a step that edits a file between a unit's start and its end), and a module that a run first loads and that changes during that same run is stored under its later hash; law 9's edits leave out files read through fs, other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated.`, }, histories: { // A failing history shrinks through at most six smaller ones. Each runs @@ -3373,7 +3397,7 @@ oracleProperty({ // verdict its first failure holds. The counterexample is the smallest // of those seven histories, its law the first failure's. arbitrary: fc.limitShrink(histories, 6), - states: required, + states: [...required, ...posixStops], measured: [ ...staticStates, `extra work in a generated function`, From 7efe17503bd49a043b959349bbe5dba7dd3d71dd Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 05:11:41 -0700 Subject: [PATCH 46/53] fix(oracle-gaps): the abort run is POSIX only; the Windows end-to-end job prints test durations A run that aborts by SIGABRT needs a signal win32 cannot deliver to a process, so its state and run are required on POSIX only, with the reason in Limits (RED: e2e shard 4/6, run 43591904999, 'the stopped run\x27s last line is done completed ... not done error, the run\x27s process exited with SIGABRT'). The Windows end-to-end step uses the verbose reporter, so each test\x27s duration is in the log for the re-shard (shard 1/6 hit the 900 s test timeout in run 43591904999). Co-authored-by: Isaac --- .github/workflows/pr.yml | 3 ++- .../oracle-gaps/gap-report-oracle.property.test.mjs | 10 +++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index ca16020..7fe3edc 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -284,7 +284,8 @@ jobs: working-directory: packages/oracles env: SHARD: ${{ matrix.shard }} - run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-fixed-seed-windows "$SHARD")" + # The verbose reporter prints each test with its duration, which sizes the shards. + run: pnpm exec vitest --run --reporter=verbose --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-fixed-seed-windows "$SHARD")" windows-gap-cli: name: Windows gap CLI on a fixture runs-on: windows-latest diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 200622f..f2a3b30 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -1696,6 +1696,8 @@ async function driver(plan, observe) { // of them the previous stopped run had left. if (cli.exit === 0 && stopIds.length > 0) for (const stop of plan.stopRuns) { + // win32 cannot deliver SIGABRT, so the abort run is POSIX only (Limits). + if (process.platform === `win32` && stop.when === `abort`) continue const run = await stoppedRun(dir, files, how, stopIds, stop) const stopped = { ...run, @@ -3215,7 +3217,6 @@ const required = [ `a run stopped by SIGINT while a worker spun`, `a stop signal after the completed line`, `a stop signal while the config warms`, - `a run whose process aborted`, `a config refusal through the daemon`, `a completed run whose tests wrote nothing`, `a stopped run whose workers are child processes`, @@ -3359,7 +3360,10 @@ const separate = [ const posixStops = process.platform === `win32` ? [] - : [`a run stopped by SIGTERM while a worker spun`] + : [ + `a run stopped by SIGTERM while a worker spun`, + `a run whose process aborted`, + ] const witnessed = [...required, ...posixStops].filter( (s) => !separate.includes(s), ) @@ -3387,7 +3391,7 @@ oracleProperty({ law: { statement: `The gap report enumerates the promised operator mutants, leaves out exactly the probe-only code, discovers every oracle that owns a function's call tree and refuses broken ownership, judges each mutant only by its owners' tests on their registered routes and counts only law verdicts on reproducible routes as kills, which outrank a run that never ends, anchors each mutant by its function and normalized statement, lists every gap and unowned mutant once with consistent counts, keeps keys stable under unrelated edits, reports through one daemon per package what a cold run reports at every state of a sequence of edits, and bounds the runs that confirm a mutant's cut-off copies by one budget per mutant and names each busy mutant in a heartbeat while it runs, leaves nothing in the package and writes nothing where Vitest's include could find it while it runs, leaves no process running once it is stopped, runs the package as Vitest does from any install path and any --source spelling: one instance of each workspace dependency, no refusal over an unbuildable devDependency, child processes for a native addon anywhere in the tree, a capture that a test's builtin spies cannot see, and a refusal of test.projects; leads its report with every test file's baseline, blocks only the mutants a failing oracle file owns and judges the rest, prints every line of its stream with the prefix oracle-gaps: and its kind, one decisive baseline line before any mutant, and a done line last, and every run that does not complete ends, cold or through the daemon, with a refusal that exits 2, prints a refused: block and one fix: directive with no stack trace, and writes the refusal to the run's output directory.`, authority: `The maintainer's decisions in design runs v11, v14, and v19 (2026-11-04 and 2026-11-05), not in this package's records, including that a kill by reach agreement alone is a gap and that an incremental run through the daemon equals a cold one (v19.3); the project's oracle guide on quiet exclusions; the oracle rubric's outcome kinds (ORC-006); the CLI's documented operator list in schemata.mjs; the maintainer's dispositions G1, G2, G4, G5, G6, and G13 of the framework papercut ledger (2026-11-11) and the v20 freeze's namespacing, one-loop, and provisional-report decisions (2026-11-10); the maintainer's request for a greppable stream (2026-11-12, relayed by the coordinator with its decisions of that day: the same prefixes on stdout and stderr, a done line for a stopped run, and no --max-gaps).`, - limits: `On win32 Node cannot deliver SIGTERM (its kill is TerminateProcess, which ends the process with no stopped line), so the stop states that name SIGTERM are required on POSIX only, and a stop there goes through the CLI's test stop hook and is observed as SIGINT; every SIGINT state stays required on win32. Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come only between checks, so a file saved during a check is not generated (the runner invalidates a known input that changed while a run ran and does not store that run, which only inspection fixes; a witness needs a step that edits a file between a unit's start and its end), and a module that a run first loads and that changes during that same run is stored under its later hash; law 9's edits leave out files read through fs, other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated.`, + limits: `On win32 Node cannot deliver SIGTERM (its kill is TerminateProcess, which ends the process with no stopped line), so the stop states that name SIGTERM, and the run whose process aborts by SIGABRT, are required on POSIX only (win32 cannot deliver either signal to a process; the abort witness exists only where it can), and a stop there goes through the CLI's test stop hook and is observed as SIGINT; every SIGINT state stays required on win32. Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come only between checks, so a file saved during a check is not generated (the runner invalidates a known input that changed while a run ran and does not store that run, which only inspection fixes; a witness needs a step that edits a file between a unit's start and its end), and a module that a run first loads and that changes during that same run is stored under its later hash; law 9's edits leave out files read through fs, other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated.`, }, histories: { // A failing history shrinks through at most six smaller ones. Each runs From 9b22c714e69301ae24abf534b9859391f8fb779b Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 05:40:55 -0700 Subject: [PATCH 47/53] test(oracle-gaps): the idle profile records which condition blocked the idle clock A hang cut off for idleness is not reached on win32 (e2e shard 4/6, run 43593613669). The profile (ORACLE_GAPS_IDLE) now counts, per test, the samples that each condition blocked: late, io with its kinds, busy, a timer due, timers listed. The Windows end-to-end job writes it and prints it. The driver passes it through GAP_IDLE_PROFILE, since cleanEnv drops the ORACLE_GAPS_ prefix. Diagnostic only; no behavior change. Co-authored-by: Isaac --- .github/workflows/pr.yml | 7 ++++++ .../oracles/oracle-gaps/capture-setup.mjs | 25 +++++++++++++------ .../gap-report-oracle.property.test.mjs | 5 ++++ 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 7fe3edc..e1a9a07 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -284,8 +284,15 @@ jobs: working-directory: packages/oracles env: SHARD: ${{ matrix.shard }} + GAP_IDLE_PROFILE: ${{ runner.temp }}/idle-${{ matrix.shard }}.jsonl # The verbose reporter prints each test with its duration, which sizes the shards. run: pnpm exec vitest --run --reporter=verbose --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-fixed-seed-windows "$SHARD")" + # Diagnostic (idle clock): which condition blocked each test's idle stretch. + - if: always() + shell: bash + run: cat "$RUNNER_TEMP/idle-$SHARD.jsonl" 2>/dev/null || echo no-idle-profile + env: + SHARD: ${{ matrix.shard }} windows-gap-cli: name: Windows gap CLI on a fixture runs-on: windows-latest diff --git a/packages/oracles/oracle-gaps/capture-setup.mjs b/packages/oracles/oracle-gaps/capture-setup.mjs index 5255684..2645ca7 100644 --- a/packages/oracles/oracle-gaps/capture-setup.mjs +++ b/packages/oracles/oracle-gaps/capture-setup.mjs @@ -953,6 +953,8 @@ if (state.capture || idleProfile) { let since let last let longest = 0 + // Profiling only: how many samples each condition blocked the idle clock. + let blocked = new Map() let use = realUse() let reported = false beforeEach(() => { @@ -960,6 +962,7 @@ if (state.capture || idleProfile) { baseTimers = timersListed(process.getActiveResourcesInfo()) since = undefined longest = 0 + blocked = new Map() sawIo = false }) beforeEach((context) => { @@ -969,7 +972,7 @@ if (state.capture || idleProfile) { if (idleProfile) appendFileSync( idleProfile, - `${JSON.stringify({ key: testKey(context.task), longest })}\n`, + `${JSON.stringify({ key: testKey(context.task), longest, blocked: Object.fromEntries(blocked) })}\n`, ) }) }) @@ -988,14 +991,20 @@ if (state.capture || idleProfile) { // a timeout at the limit anyway (a collection's five-minute GC timer). const horizon = now + (state.idleHorizonMs ?? Infinity) const timerDue = [...pending.values()].some((due) => due <= horizon) - if ( - late || - io || - used.utilization > 0.05 || - timerDue || - timersListed(kinds) > baseTimers - ) { + const blocker = late + ? `late` + : io + ? `io ${kinds.filter((kind) => !quiet.has(kind)).join(`,`)}` + : used.utilization > 0.05 + ? `busy` + : timerDue + ? `timer due` + : timersListed(kinds) > baseTimers + ? `timers listed` + : undefined + if (blocker !== undefined) { since = undefined + if (idleProfile) blocked.set(blocker, (blocked.get(blocker) ?? 0) + 1) return } since ??= now diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index f2a3b30..db9543f 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -563,6 +563,11 @@ const cleanEnv = () => ({ name !== `NODE_ENV`, ), ), + // Diagnostic (CI): GAP_IDLE_PROFILE names a file that the idle clock's + // profile is appended to (ORACLE_GAPS_IDLE, which cleanEnv would drop). + ...(process.env.GAP_IDLE_PROFILE + ? { ORACLE_GAPS_IDLE: process.env.GAP_IDLE_PROFILE } + : {}), ORACLE_GAPS_HEARTBEAT_MS: String(HEARTBEAT_MS), }) From 495b28124940bbf7a020070023237ec1c9bdd4c0 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 06:06:48 -0700 Subject: [PATCH 48/53] fix(oracle-gaps): the abort state is reachable on every platform again Reverts the POSIX-only scoping of the run whose process aborted: win32 has the mechanism (a V8 out-of-memory abort happens there too), so the state stays required everywhere. The witness still needs a win32 way to report the abort, which the CLI's supervisor (POSIX only) does today; that is open. The idle profile writes each sample of a cuttable copy as it happens, so a copy the watchdog kills keeps its samples (diagnostic). Co-authored-by: Isaac --- packages/oracles/oracle-gaps/capture-setup.mjs | 10 ++++++++++ .../oracle-gaps/gap-report-oracle.property.test.mjs | 9 ++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/packages/oracles/oracle-gaps/capture-setup.mjs b/packages/oracles/oracle-gaps/capture-setup.mjs index 2645ca7..18fd410 100644 --- a/packages/oracles/oracle-gaps/capture-setup.mjs +++ b/packages/oracles/oracle-gaps/capture-setup.mjs @@ -955,6 +955,8 @@ if (state.capture || idleProfile) { let longest = 0 // Profiling only: how many samples each condition blocked the idle clock. let blocked = new Map() + // The test whose samples are being recorded (profiling). + let key let use = realUse() let reported = false beforeEach(() => { @@ -967,6 +969,7 @@ if (state.capture || idleProfile) { }) beforeEach((context) => { if (continuing) return + key = testKey(context.task) context.onTestFinished(() => { baseTimers = undefined if (idleProfile) @@ -1002,6 +1005,13 @@ if (state.capture || idleProfile) { : timersListed(kinds) > baseTimers ? `timers listed` : undefined + // Each sample of a copy that can be cut off is written as it happens, so + // a copy the watchdog kills keeps its samples (profiling). + if (idleProfile && state.idleMs) + appendFileSync( + idleProfile, + `${JSON.stringify({ sample: blocker ?? `idle`, key, at: Math.round(now) })}\n`, + ) if (blocker !== undefined) { since = undefined if (idleProfile) blocked.set(blocker, (blocked.get(blocker) ?? 0) + 1) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index db9543f..1a25d29 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -1701,8 +1701,6 @@ async function driver(plan, observe) { // of them the previous stopped run had left. if (cli.exit === 0 && stopIds.length > 0) for (const stop of plan.stopRuns) { - // win32 cannot deliver SIGABRT, so the abort run is POSIX only (Limits). - if (process.platform === `win32` && stop.when === `abort`) continue const run = await stoppedRun(dir, files, how, stopIds, stop) const stopped = { ...run, @@ -3365,10 +3363,7 @@ const separate = [ const posixStops = process.platform === `win32` ? [] - : [ - `a run stopped by SIGTERM while a worker spun`, - `a run whose process aborted`, - ] + : [`a run stopped by SIGTERM while a worker spun`] const witnessed = [...required, ...posixStops].filter( (s) => !separate.includes(s), ) @@ -3396,7 +3391,7 @@ oracleProperty({ law: { statement: `The gap report enumerates the promised operator mutants, leaves out exactly the probe-only code, discovers every oracle that owns a function's call tree and refuses broken ownership, judges each mutant only by its owners' tests on their registered routes and counts only law verdicts on reproducible routes as kills, which outrank a run that never ends, anchors each mutant by its function and normalized statement, lists every gap and unowned mutant once with consistent counts, keeps keys stable under unrelated edits, reports through one daemon per package what a cold run reports at every state of a sequence of edits, and bounds the runs that confirm a mutant's cut-off copies by one budget per mutant and names each busy mutant in a heartbeat while it runs, leaves nothing in the package and writes nothing where Vitest's include could find it while it runs, leaves no process running once it is stopped, runs the package as Vitest does from any install path and any --source spelling: one instance of each workspace dependency, no refusal over an unbuildable devDependency, child processes for a native addon anywhere in the tree, a capture that a test's builtin spies cannot see, and a refusal of test.projects; leads its report with every test file's baseline, blocks only the mutants a failing oracle file owns and judges the rest, prints every line of its stream with the prefix oracle-gaps: and its kind, one decisive baseline line before any mutant, and a done line last, and every run that does not complete ends, cold or through the daemon, with a refusal that exits 2, prints a refused: block and one fix: directive with no stack trace, and writes the refusal to the run's output directory.`, authority: `The maintainer's decisions in design runs v11, v14, and v19 (2026-11-04 and 2026-11-05), not in this package's records, including that a kill by reach agreement alone is a gap and that an incremental run through the daemon equals a cold one (v19.3); the project's oracle guide on quiet exclusions; the oracle rubric's outcome kinds (ORC-006); the CLI's documented operator list in schemata.mjs; the maintainer's dispositions G1, G2, G4, G5, G6, and G13 of the framework papercut ledger (2026-11-11) and the v20 freeze's namespacing, one-loop, and provisional-report decisions (2026-11-10); the maintainer's request for a greppable stream (2026-11-12, relayed by the coordinator with its decisions of that day: the same prefixes on stdout and stderr, a done line for a stopped run, and no --max-gaps).`, - limits: `On win32 Node cannot deliver SIGTERM (its kill is TerminateProcess, which ends the process with no stopped line), so the stop states that name SIGTERM, and the run whose process aborts by SIGABRT, are required on POSIX only (win32 cannot deliver either signal to a process; the abort witness exists only where it can), and a stop there goes through the CLI's test stop hook and is observed as SIGINT; every SIGINT state stays required on win32. Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come only between checks, so a file saved during a check is not generated (the runner invalidates a known input that changed while a run ran and does not store that run, which only inspection fixes; a witness needs a step that edits a file between a unit's start and its end), and a module that a run first loads and that changes during that same run is stored under its later hash; law 9's edits leave out files read through fs, other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated.`, + limits: `On win32 Node cannot deliver SIGTERM (its kill is TerminateProcess, which ends the process with no stopped line), so the stop states that name SIGTERM are required on POSIX only, and a stop there goes through the CLI's test stop hook and is observed as SIGINT; every SIGINT state stays required on win32. Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come only between checks, so a file saved during a check is not generated (the runner invalidates a known input that changed while a run ran and does not store that run, which only inspection fixes; a witness needs a step that edits a file between a unit's start and its end), and a module that a run first loads and that changes during that same run is stored under its later hash; law 9's edits leave out files read through fs, other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated.`, }, histories: { // A failing history shrinks through at most six smaller ones. Each runs From 5d97ae889c01feb57d1915f8cde2a3768d480f1c Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 06:41:24 -0700 Subject: [PATCH 49/53] chore(oracle-gaps): temporary daemon-history profile on Linux and Windows Removes the GAP_IDLE_PROFILE pass-through and the idle print step. Adds a temporary daemon-history-profile job that runs the one fixed-seed history on ubuntu and windows with GAP_PROFILE_ALL (a temporary route of the CLI's profile spans to one file) and prints per-kind totals. Diagnostic only; remove before merge, with the GAP_PROFILE_ALL route in check.mjs. Co-authored-by: Isaac --- .github/workflows/pr.yml | 50 +++++++++++++++++-- packages/oracles/oracle-gaps/check.mjs | 5 +- .../gap-report-oracle.property.test.mjs | 5 -- 3 files changed, 48 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index e1a9a07..07997ad 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -284,15 +284,55 @@ jobs: working-directory: packages/oracles env: SHARD: ${{ matrix.shard }} - GAP_IDLE_PROFILE: ${{ runner.temp }}/idle-${{ matrix.shard }}.jsonl # The verbose reporter prints each test with its duration, which sizes the shards. run: pnpm exec vitest --run --reporter=verbose --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-fixed-seed-windows "$SHARD")" - # Diagnostic (idle clock): which condition blocked each test's idle stretch. - - if: always() + daemon-history-profile: + # Temporary diagnostic (CI numbers for the daemon history's cost); remove before merge. + name: Daemon history profile (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + continue-on-error: true + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Setup Tools + uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 + - run: pnpm --filter @tanstack/oracles run build + - name: Daemon history, profiled shell: bash - run: cat "$RUNNER_TEMP/idle-$SHARD.jsonl" 2>/dev/null || echo no-idle-profile + working-directory: packages/oracles env: - SHARD: ${{ matrix.shard }} + GAP_PROFILE_ALL: ${{ runner.temp }}/profile.jsonl + run: | + start=$(date +%s) + pnpm exec vitest --run --testTimeout=2700000 --reporter=verbose --config oracle-gaps/vitest.config.mjs gap-report-oracle -t 'fixed seed 20261105: an edit sequence through the daemon$' || true + echo "wall $(( $(date +%s) - start )) s" + cat > "$RUNNER_TEMP/summ.mjs" <<'EOF' + import { readFileSync } from 'node:fs' + const rows = readFileSync(process.argv[2], 'utf8').trim().split('\n').map((l) => JSON.parse(l)) + const checks = rows.filter((r) => r.kind === 'check') + const runs = rows.filter((r) => r.kind === 'run') + const phases = rows.filter((r) => r.kind === 'phase') + const mutants = rows.filter((r) => r.kind === 'mutant') + const sum = (a) => a.reduce((s, r) => s + (r.ms ?? 0), 0) + console.log(`checks: ${checks.length}, total ${(sum(checks) / 1000).toFixed(1)} s`) + console.log(`runs (vitest batches/baseline): ${runs.length}, total ${(sum(runs) / 1000).toFixed(1)} s`) + console.log(`mutant evaluations: ${mutants.length}, total ${(sum(mutants) / 1000).toFixed(1)} s`) + const byPhase = {} + for (const p of phases) { byPhase[p.name] ??= { n: 0, ms: 0 }; byPhase[p.name].n++; byPhase[p.name].ms += p.ms } + for (const [name, v] of Object.entries(byPhase).sort((a, b) => b[1].ms - a[1].ms)) console.log(` phase ${name}: n=${v.n} total ${(v.ms / 1000).toFixed(1)} s`) + const byRunKind = {} + for (const r of runs) { const k = r.file === 'batch' ? 'batch' : 'single'; byRunKind[k] ??= { n: 0, ms: 0 }; byRunKind[k].n++; byRunKind[k].ms += r.ms } + for (const [k, v] of Object.entries(byRunKind)) console.log(` run ${k}: n=${v.n} total ${(v.ms / 1000).toFixed(1)} s`) + + EOF + node "$RUNNER_TEMP/summ.mjs" "$GAP_PROFILE_ALL" windows-gap-cli: name: Windows gap CLI on a fixture runs-on: windows-latest diff --git a/packages/oracles/oracle-gaps/check.mjs b/packages/oracles/oracle-gaps/check.mjs index af9e62a..90aff0e 100644 --- a/packages/oracles/oracle-gaps/check.mjs +++ b/packages/oracles/oracle-gaps/check.mjs @@ -1299,8 +1299,9 @@ export default async (env) => { rmSync(testOutput, { force: true }) profileFile = process.env.ORACLE_GAPS_PROFILE ? join(options.out, `profile.jsonl`) - : undefined - if (profileFile) rmSync(profileFile, { force: true }) + : process.env.GAP_PROFILE_ALL + if (process.env.ORACLE_GAPS_PROFILE && profileFile) + rmSync(profileFile, { force: true }) const checkStarted = Date.now() let phase = Date.now() const endPhase = (name, fields) => { diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 1a25d29..667c736 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -563,11 +563,6 @@ const cleanEnv = () => ({ name !== `NODE_ENV`, ), ), - // Diagnostic (CI): GAP_IDLE_PROFILE names a file that the idle clock's - // profile is appended to (ORACLE_GAPS_IDLE, which cleanEnv would drop). - ...(process.env.GAP_IDLE_PROFILE - ? { ORACLE_GAPS_IDLE: process.env.GAP_IDLE_PROFILE } - : {}), ORACLE_GAPS_HEARTBEAT_MS: String(HEARTBEAT_MS), }) From 887dfe328e93794fa98ff8c1d53f4019aefdcb3f Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 07:02:02 -0700 Subject: [PATCH 50/53] chore(oracle-gaps): probe the daemon-history job's startup and gap timings (temporary) Co-authored-by: Isaac --- .github/workflows/pr.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 07997ad..e7148eb 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -310,6 +310,11 @@ jobs: env: GAP_PROFILE_ALL: ${{ runner.temp }}/profile.jsonl run: | + probe() { local name=$1; shift; local t0=$(date +%s%N); "$@" >/dev/null 2>&1; echo "probe $name: $(( ($(date +%s%N) - t0) / 1000000 )) ms"; } + probe node-startup node -e 0 + probe cli-help node oracle-gaps/index.mjs --help + probe ps-list ps -A -o pid=,command= + probe stop-daemon node oracle-gaps/index.mjs --stop-daemon --package "$PWD/oracle-gaps/fixtures-none" start=$(date +%s) pnpm exec vitest --run --testTimeout=2700000 --reporter=verbose --config oracle-gaps/vitest.config.mjs gap-report-oracle -t 'fixed seed 20261105: an edit sequence through the daemon$' || true echo "wall $(( $(date +%s) - start )) s" @@ -320,6 +325,15 @@ jobs: const runs = rows.filter((r) => r.kind === 'run') const phases = rows.filter((r) => r.kind === 'phase') const mutants = rows.filter((r) => r.kind === 'mutant') + const cs = [...checks].sort((a, b) => a.at - b.at) + let gap = 0 + const big = [] + for (let i = 1; i < cs.length; i++) { + const g = cs[i].at - (cs[i - 1].at + cs[i - 1].ms) + if (g > 0) gap += g + if (g > 5000) big.push(g) + } + console.log(`gaps between checks: total ${(gap / 1000).toFixed(1)} s, ${big.length} gaps over 5 s totalling ${(big.reduce((a, b) => a + b, 0) / 1000).toFixed(1)} s`) const sum = (a) => a.reduce((s, r) => s + (r.ms ?? 0), 0) console.log(`checks: ${checks.length}, total ${(sum(checks) / 1000).toFixed(1)} s`) console.log(`runs (vitest batches/baseline): ${runs.length}, total ${(sum(runs) / 1000).toFixed(1)} s`) From 144bc06696498118c4753cb70903b0fc889a6d14 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 07:27:34 -0700 Subject: [PATCH 51/53] chore(oracle-gaps): time ps in the temporary profile job Co-authored-by: Isaac --- .github/workflows/pr.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index e7148eb..50a9109 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -313,7 +313,8 @@ jobs: probe() { local name=$1; shift; local t0=$(date +%s%N); "$@" >/dev/null 2>&1; echo "probe $name: $(( ($(date +%s%N) - t0) / 1000000 )) ms"; } probe node-startup node -e 0 probe cli-help node oracle-gaps/index.mjs --help - probe ps-list ps -A -o pid=,command= + for k in 1 2 3; do probe ps-list-$k timeout 150 ps -A -o pid=,command=; done + timeout 150 ps -A -o pid=,command= | wc -l probe stop-daemon node oracle-gaps/index.mjs --stop-daemon --package "$PWD/oracle-gaps/fixtures-none" start=$(date +%s) pnpm exec vitest --run --testTimeout=2700000 --reporter=verbose --config oracle-gaps/vitest.config.mjs gap-report-oracle -t 'fixed seed 20261105: an edit sequence through the daemon$' || true From d0cd4fe5f05efc6d491240a6d5dcac9d6ac9665e Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 07:48:16 -0700 Subject: [PATCH 52/53] chore(oracle-gaps): probe ps without timeout and tolerate probe failures Co-authored-by: Isaac --- .github/workflows/pr.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 50a9109..5c0da6f 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -310,11 +310,11 @@ jobs: env: GAP_PROFILE_ALL: ${{ runner.temp }}/profile.jsonl run: | - probe() { local name=$1; shift; local t0=$(date +%s%N); "$@" >/dev/null 2>&1; echo "probe $name: $(( ($(date +%s%N) - t0) / 1000000 )) ms"; } + probe() { local name=$1; shift; local t0=$(date +%s%N); "$@" >/dev/null 2>&1 || true; echo "probe $name: $(( ($(date +%s%N) - t0) / 1000000 )) ms"; } probe node-startup node -e 0 probe cli-help node oracle-gaps/index.mjs --help - for k in 1 2 3; do probe ps-list-$k timeout 150 ps -A -o pid=,command=; done - timeout 150 ps -A -o pid=,command= | wc -l + for k in 1 2 3; do probe ps-list-$k ps -A -o pid=,command=; done + ps -A -o pid=,command= | wc -l probe stop-daemon node oracle-gaps/index.mjs --stop-daemon --package "$PWD/oracle-gaps/fixtures-none" start=$(date +%s) pnpm exec vitest --run --testTimeout=2700000 --reporter=verbose --config oracle-gaps/vitest.config.mjs gap-report-oracle -t 'fixed seed 20261105: an edit sequence through the daemon$' || true From 79f421e13630a1a2d2627d28257edc96a0256ea0 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 08:12:03 -0700 Subject: [PATCH 53/53] chore(oracle-gaps): remove the temporary daemon-history profile The profile job, its probes, and the GAP_PROFILE_ALL route were diagnostics for the daemon history's cost; the numbers are in the handback. Co-authored-by: Isaac --- .github/workflows/pr.yml | 62 -------------------------- packages/oracles/oracle-gaps/check.mjs | 5 +-- 2 files changed, 2 insertions(+), 65 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 5c0da6f..7fe3edc 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -286,68 +286,6 @@ jobs: SHARD: ${{ matrix.shard }} # The verbose reporter prints each test with its duration, which sizes the shards. run: pnpm exec vitest --run --reporter=verbose --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-fixed-seed-windows "$SHARD")" - daemon-history-profile: - # Temporary diagnostic (CI numbers for the daemon history's cost); remove before merge. - name: Daemon history profile (${{ matrix.os }}) - runs-on: ${{ matrix.os }} - timeout-minutes: 45 - continue-on-error: true - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, windows-latest] - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Setup Tools - uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - - run: pnpm --filter @tanstack/oracles run build - - name: Daemon history, profiled - shell: bash - working-directory: packages/oracles - env: - GAP_PROFILE_ALL: ${{ runner.temp }}/profile.jsonl - run: | - probe() { local name=$1; shift; local t0=$(date +%s%N); "$@" >/dev/null 2>&1 || true; echo "probe $name: $(( ($(date +%s%N) - t0) / 1000000 )) ms"; } - probe node-startup node -e 0 - probe cli-help node oracle-gaps/index.mjs --help - for k in 1 2 3; do probe ps-list-$k ps -A -o pid=,command=; done - ps -A -o pid=,command= | wc -l - probe stop-daemon node oracle-gaps/index.mjs --stop-daemon --package "$PWD/oracle-gaps/fixtures-none" - start=$(date +%s) - pnpm exec vitest --run --testTimeout=2700000 --reporter=verbose --config oracle-gaps/vitest.config.mjs gap-report-oracle -t 'fixed seed 20261105: an edit sequence through the daemon$' || true - echo "wall $(( $(date +%s) - start )) s" - cat > "$RUNNER_TEMP/summ.mjs" <<'EOF' - import { readFileSync } from 'node:fs' - const rows = readFileSync(process.argv[2], 'utf8').trim().split('\n').map((l) => JSON.parse(l)) - const checks = rows.filter((r) => r.kind === 'check') - const runs = rows.filter((r) => r.kind === 'run') - const phases = rows.filter((r) => r.kind === 'phase') - const mutants = rows.filter((r) => r.kind === 'mutant') - const cs = [...checks].sort((a, b) => a.at - b.at) - let gap = 0 - const big = [] - for (let i = 1; i < cs.length; i++) { - const g = cs[i].at - (cs[i - 1].at + cs[i - 1].ms) - if (g > 0) gap += g - if (g > 5000) big.push(g) - } - console.log(`gaps between checks: total ${(gap / 1000).toFixed(1)} s, ${big.length} gaps over 5 s totalling ${(big.reduce((a, b) => a + b, 0) / 1000).toFixed(1)} s`) - const sum = (a) => a.reduce((s, r) => s + (r.ms ?? 0), 0) - console.log(`checks: ${checks.length}, total ${(sum(checks) / 1000).toFixed(1)} s`) - console.log(`runs (vitest batches/baseline): ${runs.length}, total ${(sum(runs) / 1000).toFixed(1)} s`) - console.log(`mutant evaluations: ${mutants.length}, total ${(sum(mutants) / 1000).toFixed(1)} s`) - const byPhase = {} - for (const p of phases) { byPhase[p.name] ??= { n: 0, ms: 0 }; byPhase[p.name].n++; byPhase[p.name].ms += p.ms } - for (const [name, v] of Object.entries(byPhase).sort((a, b) => b[1].ms - a[1].ms)) console.log(` phase ${name}: n=${v.n} total ${(v.ms / 1000).toFixed(1)} s`) - const byRunKind = {} - for (const r of runs) { const k = r.file === 'batch' ? 'batch' : 'single'; byRunKind[k] ??= { n: 0, ms: 0 }; byRunKind[k].n++; byRunKind[k].ms += r.ms } - for (const [k, v] of Object.entries(byRunKind)) console.log(` run ${k}: n=${v.n} total ${(v.ms / 1000).toFixed(1)} s`) - - EOF - node "$RUNNER_TEMP/summ.mjs" "$GAP_PROFILE_ALL" windows-gap-cli: name: Windows gap CLI on a fixture runs-on: windows-latest diff --git a/packages/oracles/oracle-gaps/check.mjs b/packages/oracles/oracle-gaps/check.mjs index 90aff0e..af9e62a 100644 --- a/packages/oracles/oracle-gaps/check.mjs +++ b/packages/oracles/oracle-gaps/check.mjs @@ -1299,9 +1299,8 @@ export default async (env) => { rmSync(testOutput, { force: true }) profileFile = process.env.ORACLE_GAPS_PROFILE ? join(options.out, `profile.jsonl`) - : process.env.GAP_PROFILE_ALL - if (process.env.ORACLE_GAPS_PROFILE && profileFile) - rmSync(profileFile, { force: true }) + : undefined + if (profileFile) rmSync(profileFile, { force: true }) const checkStarted = Date.now() let phase = Date.now() const endPhase = (name, fields) => {