diff --git a/.changeset/gap-cli-stream-log.md b/.changeset/gap-cli-stream-log.md new file mode 100644 index 0000000..1c8c225 --- /dev/null +++ b/.changeset/gap-cli-stream-log.md @@ -0,0 +1,5 @@ +--- +'@tanstack/oracles': patch +--- + +The `oracle-gaps` CLI now writes the lines it streams on stderr to `stream.log` in the output directory too, in order, cold and through `--daemon`, and the first line of a check on both streams names that file: `oracle-gaps: note log /stream.log`. An agent that starts a cold check in the background can read the run's progress and its done line there while it runs. A check through `--daemon` prints its lines at the end of its check, so its progress is readable there only after it ends. diff --git a/README.md b/README.md index fd67e25..de7f4ed 100644 --- a/README.md +++ b/README.md @@ -123,13 +123,25 @@ agent. to the process's stdout and stderr goes to `test-output.log` in the output directory, which a note names; `summary.md` holds the report for reading. A run through the daemon prints the same lines, at the end of its check. + The first line of a check, on both streams, is + `oracle-gaps: note log /stream.log`: the run writes its stderr lines + to that file too, in order, so a check started in the background can be + read there while it runs and after it ends. A cold run's lines reach the + file as they are printed; a run through `--daemon` prints its lines at the + end of its check, so the file fills then. `--help` is not a check, so its usage text has no prefix. - For contributors: two environment variables in the `ORACLE_GAPS_TEST_` - namespace are test-only hooks for the CLI's own oracle, not settings for - users. `ORACLE_GAPS_TEST_HOLD_EXIT_MS` holds the run's exit after its done - line, and `ORACLE_GAPS_TEST_WARM_PROBE` makes the first config warm-up - wait where a stop signal can reach it. + For contributors: the `ORACLE_GAPS_TEST_` environment variables are test-only + hooks for the CLI's own oracle, not settings for users. Each is inert unless + set; a hook may pause at one named point and never changes an outcome. + `ORACLE_GAPS_TEST_STARTED` names a file that the CLI writes as its first project statement, before any project module loads, so a test can tell a stop that came before the CLI started from one that came during its loads. + `ORACLE_GAPS_TEST_PAUSE_AFTER_STARTED` (milliseconds) blocks the CLI for that long right after the started file is written, so a stop sent on seeing the file lands inside the pause. + `ORACLE_GAPS_TEST_PAUSE_AT` names one point at which `ORACLE_GAPS_TEST_PAUSE_MS` (milliseconds) blocks the CLI: `takeover` (inside a stale-claim takeover, after the stale pid is read) and `mutex-create` (inside the takeover mutex's creation). + `ORACLE_GAPS_TEST_STOP` names a file whose appearance makes the CLI take a stop as a SIGINT would. + `ORACLE_GAPS_TEST_HOLD_UNTIL` names a file: after its done line the run keeps + its exit, and its output claim, until the file exists, so a test can send a + stop or start a second run before the run ends. `ORACLE_GAPS_TEST_WARM_PROBE` makes the first config warm-up wait where a + stop signal can reach it. A few endings print no done line, or one more line: a SIGKILL of the CLI, or a signal that comes while Node is still starting the CLI, before its diff --git a/packages/oracles/oracle-gaps/ending.mjs b/packages/oracles/oracle-gaps/ending.mjs index ff58f9b..e9658a8 100644 --- a/packages/oracles/oracle-gaps/ending.mjs +++ b/packages/oracles/oracle-gaps/ending.mjs @@ -38,8 +38,10 @@ export function endRun(error, out = error?.out) { // The last line says the run refused, and where its report is once the // run knows its output directory (the stream's done line, stream.mjs); // it ends stdout too (index.mjs). + // A reportless refusal (another run uses the directory) writes nothing + // there: the directory's files are the other run's. const done = write( - out, + error.reportless ? undefined : out, results, `Refused: ${message}\n\nFix: ${error.fix}\n`, ) diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index 6ebe9f3..1f83580 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -769,6 +769,73 @@ const drawn = fc.record({ }), // Who names the importer's function in an owns entry, if any. importer: fc.constantFrom(`none`, `a`, `b`), + // Where the run writes: the output directory `--out` names (`explicit`), or + // the one the CLI picks when `--out` is left out (`default`, args.mjs), whose + // stream log, results, and done line must still agree (law 15). + outMode: fc.constantFrom(`explicit`, `explicit`, `explicit`, `default`), + // The run's `/stream.log` is a directory, so the log cannot be written + // (a problem the author can fix: the run refuses before it checks). + logBlocked: fc.constantFrom( + false, + false, + false, + false, + false, + false, + false, + false, + true, + ), + // A second run starts on the first run's output directory while the first + // is live (the coordinator's decision of 2026-11-13 on F4): the second + // refuses, and the first's stream log stays its own. + overlap: fc.constantFrom( + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + ), + // The stop witness alone (`w_stop`), which a run the driver stops needs. A + // field of its own, so the witness options above keep their weights. + stopWitness: fc.constantFrom( + false, + false, + false, + false, + false, + false, + false, + false, + true, + ), + // A stop sent as soon as the CLI spawns, before its own run has started (P1: + // the note is written before the child exists, so the stopped line follows + // it). The driver adds this stop to the run's stops. + spawnStop: fc.constantFrom( + false, + false, + false, + false, + false, + false, + false, + false, + true, + ), + // The signal of a spawn stop and when it lands, 0 to 150 ms after the CLI + // spawns (the spawn-delay law, review of #57). + spawnSignal: fc.constantFrom(`SIGINT`, `SIGINT`, `SIGTERM`), + spawnDelay: fc.integer({ min: 0, max: 150 }), + // The spawn stop waits for the CLI's started marker, then signals at once (the + // deterministic witness, event-driven rather than timed). + spawnOnStart: fc.constantFrom(false, false, false, true), }) // The edit sequence over the witness module, the one list of steps that a @@ -780,6 +847,10 @@ const drawn = fc.record({ // nothing changed, which reuses the confirmations that the first check ran. const witnessSteps = [`test`, `helper`, `revert`] const budgetSteps = [`recheck`] +// The stop witness alone takes no steps, so it counts only where the history +// has none; a history with steps and the field is a legal history without it. +const stopsOnly = (history) => + history.stopWitness === true && history.steps === null const keepsSteps = (history) => { if (history.witness === false || history.steps === null) return true const kept = history.witness === `budget` ? budgetSteps : witnessSteps @@ -2469,7 +2540,7 @@ function resolveHistory(history) { ...exported, ...importer, ...dotGen, - ...(history.witness + ...(history.witness || stopsOnly(history) ? witness .filter( (w) => @@ -2477,6 +2548,7 @@ function resolveHistory(history) { // so only histories of the `budget` mode carry it. (history.witness === true && w.id !== `w_slowpair`) || (history.witness === `capture` && captureWitness.has(w.id)) || + (stopsOnly(history) && w.id === `w_stop`) || (history.witness === `budget` && w.id === `w_slowpair`), ) .map((w) => ({ @@ -2837,7 +2909,26 @@ function resolveHistory(history) { depSubpath: history.depSubpath === true, unbuiltDep: history.unbuiltDep === true, budget: history.budget ?? `two`, + outMode: history.outMode ?? `explicit`, + // Both name the run's `--out` directory before it starts, so they need + // an explicit one (a default directory is chosen by the CLI). + logBlocked: + history.logBlocked === true && outModeOf(history) === `explicit`, + // A second run needs a live first run, which has its note and stream log. + // A blocked log refuses the first run before it is live, and a refusing + // fault (an argument, declaration, or config fault) ends it before its note + // too, so the overlap is generated only for a run that reaches its check + // (decision of 2026-11-13, review of #57: O1 and O2). + overlap: + history.overlap === true && + outModeOf(history) === `explicit` && + history.logBlocked !== true && + (history.fault ?? `none`) === `none`, stopRuns: history.stopRuns ?? [], + spawnStop: history.spawnStop === true, + spawnSignal: history.spawnSignal ?? `SIGINT`, + spawnDelay: history.spawnDelay ?? 0, + spawnOnStart: history.spawnOnStart === true, onlyRun: history.onlyRun === true, pragma: history.pragma === true, cliPath: history.cliPath ?? `plain`, @@ -4083,10 +4174,23 @@ function refusal(plan) { if (plan.fault === `brokenVitest`) return { cause: `vitest`, runtime: true } if (plan.fault === `projects`) return { cause: `test.projects`, runtime: true } + // A stream log that cannot be written is refused before the check starts + // (the CLI opens the log before its first line, index.mjs), so its stdout + // owes its done line alone. + if (plan.logBlocked === true) + return { cause: `the stream log`, arguments: true } if (plan.fault === `badEntry`) return { cause: `${SOURCE}#nope` } + // An owns entry that names a missing file (a test file's declaration) is + // found by the analysis, after the check has started, so this refusal + // owes the stream log's note. The argument-stage case, a `--source` that + // is not a file of the package, is the static oracle's `absentSource`. if (plan.fault === `missingSource`) return { cause: `src/missing.ts` } if (plan.fault === `nonLiteral`) return { cause: `b.test.ts` } - if (plan.fault === `outsideSource`) return { cause: `outside the package` } + // A `--source` outside the package is an argument the CLI refuses before + // it starts a check (the argument rules that the static oracle judges), + // so its stdout has no line that names a stream log. + if (plan.fault === `outsideSource`) + return { cause: `outside the package`, arguments: true } // The analysis reads the test files that the package's Vitest config // includes (index.mjs, step 2), so a config that does not load is named // before anything the analysis would find. The cause names the config @@ -4183,6 +4287,7 @@ const declarationFaults = [ `nonLiteral`, `outsideSource`, ] +const outModeOf = (history) => history.outMode ?? `explicit` const reachesRun = (history) => !declarationFaults.includes(history.fault ?? `none`) @@ -4235,21 +4340,22 @@ function model(plan) { // fault, a Vitest that does not load, a config that does not load, and a // config that defines test.projects, which a loaded context shows. Every // other run names it once, refused or not. - const configLines = [ - `outsideSource`, - `brokenVitest`, - `configThrows`, - `configNoDefault`, - `configNull`, - `configSyntax`, - `projects`, - ].includes(plan.fault) - ? [] - : [ - plan.configPath === undefined - ? `oracle-gaps: config none, Vitest's default discovery` - : `oracle-gaps: config ${plan.configPath}`, - ] + const configLines = + [ + `outsideSource`, + `brokenVitest`, + `configThrows`, + `configNoDefault`, + `configNull`, + `configSyntax`, + `projects`, + ].includes(plan.fault) || plan.logBlocked === true + ? [] + : [ + plan.configPath === undefined + ? `oracle-gaps: config none, Vitest's default discovery` + : `oracle-gaps: config ${plan.configPath}`, + ] if (refused) return { refused, configLines } const sites = sitesOf(plan) const owners = ownersOf(plan) @@ -5493,8 +5599,13 @@ function endingOf(exit, stderr, out, name = (text) => text, stdout = ``) { * The first way a run's ending breaks the refusal law, or undefined. The * cause must name `cause`; with `outKnown`, the run knew its output * directory, so the block names it and the directory holds the refusal. + * `logNote` is the line that names the run's stream log, owed first on + * stdout when the run refused after it started a check, and undefined for + * a refusal of its arguments, before any check (the maintainer's decision + * of 2026-11-12 that every run that starts a check names its stream log + * first). */ -function refusalBreach(ending, { cause, outKnown }) { +function refusalBreach(ending, { cause, outKnown, logNote }) { if (ending.exit !== 2) return `the refusal exited ${ending.exit}, not 2` if (ending.stackFrames) return `the refusal printed a stack trace` if (ending.cause === undefined) @@ -5513,12 +5624,14 @@ function refusalBreach(ending, { cause, outKnown }) { !/^refused(?:, report .+)?$/.test(ending.done) ) return `the refusal's last line is not \`oracle-gaps: done refused\`` - // The done line ends stdout too, as its only line. - if ( - ending.stdout.length !== 1 || - ending.stdout[0] !== `oracle-gaps: done ${ending.done}` - ) - return `the refusal's stdout is ${JSON.stringify(ending.stdout)}, not its done line` + // The done line ends stdout too, as its only line after the line that + // names the stream log, if the run started a check. + const owedStdout = [ + ...(logNote === undefined ? [] : [logNote]), + `oracle-gaps: done ${ending.done}`, + ] + if (JSON.stringify(ending.stdout) !== JSON.stringify(owedStdout)) + return `the refusal's stdout is ${JSON.stringify(ending.stdout)}, not ${JSON.stringify(owedStdout)}` if (!outKnown) return undefined if (!ending.namesOutput) return `the refusal does not name its report in its output directory: done ${ending.done}` diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 9f0fe4c..8dc3edb 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -350,7 +350,15 @@ * The run's last line is its done line: `completed, gaps[ * (provisional)], report ` on stdout, `refused[, report * ]` after a refusal's block, or `stopped ()` - * after a stop. A check through the daemon streams the same lines. + * after a stop. A check through the daemon streams the same lines. The + * first line, on both streams, of a run that starts a check names its + * stream log, `oracle-gaps: note log /stream.log`, and that file + * holds the stderr's lines, in order, cold and through the daemon (the + * maintainer's decision of 2026-11-12: an agent that runs a check in the + * background learns from the run itself where to read it). A refusal + * after the check started has that line and then its done line on + * stdout; a refusal of the run's arguments comes before any check, and + * its stdout is its done line alone. * * Authority: the maintainer's decisions in the gap-check design runs v11, v14, * and v19 (2026-11-04 and 2026-11-05; the design notes are kept by the project @@ -580,14 +588,17 @@ function cliArgs( sources = `${SOURCE},${LIB}`, budget = `two`, outName = daemon ? `out-daemon` : `out`, + // `default` leaves `--out` out, so the CLI picks its own output + // directory (args.mjs): the run's first stdout line names it. + outMode = `explicit`, config, contexts = 4, only, pool, } = {}, ) { - const out = join(dir, outName) - rmSync(out, { recursive: true, force: true }) + const out = outMode === `default` ? undefined : join(dir, outName) + if (out !== undefined) rmSync(out, { recursive: true, force: true }) // The CLI runs from the package directory. A daemon check names its paths // relative to there, as an author types them, and the daemon must read // them as the CLI's caller meant them. @@ -620,9 +631,8 @@ function cliArgs( // one copy: one slow owner's confirmation fits it, two do not (law 4). `--mutant-limit`, String(MUTANT_LIMIT_MS), - `--out`, - path(out), ] + if (out !== undefined) args.push(`--out`, path(out)) if (only !== undefined) args.push(`--only`, only.join(`,`)) if (pool !== undefined) args.push(`--pool`, pool) if (ordinary) args.push(`--ordinary`) @@ -647,6 +657,7 @@ function runCli(dir, files, how = {}) { writeFixture(dir, files) const before = new Set(filesUnder(dir)) const { args, out } = cliArgs(dir, files, how) + if (how.logBlocked) mkdirSync(join(out, `stream.log`), { recursive: true }) const child = spawnSync(process.execPath, args, { cwd: repo, encoding: `utf8`, @@ -697,7 +708,7 @@ const strays = (dir, before, running = false) => * * With `when` `after`, the run is not stopped while it runs: the witness * does not spin, the run's process holds its exit for 500 ms once it has - * printed its done line (`ORACLE_GAPS_TEST_HOLD_EXIT_MS`, a hook for this + * printed its done line (`ORACLE_GAPS_TEST_HOLD_UNTIL`, a hook for this * driver), and the driver sends the signal once that line is on stdout * (review of #46, F3). The run then ends as it completed, with exactly one * done line. With `warm`, the driver sends the signal while the run warms @@ -707,7 +718,16 @@ const strays = (dir, before, running = false) => * Vitest's does; re-review N1). The run then ends stopped, with exactly * one done line and no refusal. */ -async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { +async function stoppedRun( + dir, + files, + how, + ids, + { signal, pool, when, delay = 0, onStart = false }, +) { + // `spawn`: the signal comes as the CLI starts, once its own code runs (a + // signal during Node's own start has no handler yet; Limits of #57). + const atSpawn = when === `spawn` const after = when === `after` const warm = when === `warm` // With `abort`, no signal comes: a test of file a aborts the run's @@ -719,22 +739,38 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { writeFixture(dir, files) const before = new Set(filesUnder(dir)) const pids = mkdtempSync(join(realpathSync(tmpdir()), `gap-stop-`)) - const { args } = cliArgs(dir, files, { + const { args, out: given } = cliArgs(dir, files, { ...how, outName: `out-stop`, contexts: 1, only: ids, pool: abort ? `threads` : pool, }) + // The started marker: the CLI writes it as its first statement (index.mjs). + const startedPath = join(pids, `started`) const child = spawn(process.execPath, args, { cwd: repo, - env: after - ? { ...cleanEnv(), ORACLE_GAPS_TEST_HOLD_EXIT_MS: `500` } - : warm - ? { ...cleanEnv(), ORACLE_GAPS_TEST_WARM_PROBE: join(pids, `warming`) } - : abort - ? { ...cleanEnv(), GAP_ABORT: `1` } - : { ...cleanEnv(), GAP_STOP_DIR: pids }, + env: { + ...(atSpawn ? { ORACLE_GAPS_TEST_STARTED: startedPath } : {}), + // The constructed witness pauses after the marker, so the signal it sends + // on seeing the marker lands inside the pause (review F2). + ...(atSpawn && onStart + ? { ORACLE_GAPS_TEST_PAUSE_AFTER_STARTED: `200` } + : {}), + ...(after + ? { + ...cleanEnv(), + ORACLE_GAPS_TEST_HOLD_UNTIL: join(pids, `after-release`), + } + : warm + ? { + ...cleanEnv(), + ORACLE_GAPS_TEST_WARM_PROBE: join(pids, `warming`), + } + : abort + ? { ...cleanEnv(), GAP_ABORT: `1` } + : { ...cleanEnv(), GAP_STOP_DIR: pids }), + }, stdio: [`ignore`, `pipe`, `pipe`], }) let stdout = `` @@ -759,6 +795,7 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { let waited = 0; !exited && !abort && + !atSpawn && (after ? !stdout.includes(`oracle-gaps: done completed`) : warm @@ -775,13 +812,27 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { if (abort) await Promise.race([exit, sleep(120_000)]) const spun = abort ? exited - : after - ? stdout.includes(`oracle-gaps: done completed`) && !exited - : warm - ? existsSync(join(pids, `warming`)) && !exited - : readdirSync(pids).length >= 2 + : atSpawn + ? !exited + : after + ? stdout.includes(`oracle-gaps: done completed`) && !exited + : warm + ? existsSync(join(pids, `warming`)) && !exited + : readdirSync(pids).length >= 2 const strayWhileRunning = strays(dir, before, true) + // A spawn stop signals after its drawn delay, or, when it waits for the + // started marker, at once once the marker exists (event-driven, 30 s cap). + if (atSpawn && onStart) + for ( + let waited = 0; + !existsSync(startedPath) && waited < 30_000; + waited += 1 + ) + await sleep(1) + else if (atSpawn) await sleep(delay) if (!abort) child.kill(signal) + // The `after` stop is sent while the exit is held; the hold is released now. + if (after) writeFileSync(join(pids, `after-release`), ``) const ended = await Promise.race([exit, sleep(30_000).then(() => false)]) await sleep(1000) const alive = readdirSync(pids) @@ -805,8 +856,22 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { } if (!ended) child.kill(`SIGKILL`) rmSync(pids, { recursive: true, force: true }) - return { + const out = + given ?? + `${stdout}` + .split(`\n`)[0] + .match(/^oracle-gaps: note log (.*)\/stream\.log$/)?.[1] ?? + join(dir, `no-output`) + const scratchNames = [ + [how.cli ?? here, ``], + [dir, ``], + ...(given === undefined ? [[out, ``]] : []), + ] + const result = { signal, + // Whether the CLI wrote its started marker before the signal (outcomes a and b). + started: atSpawn ? existsSync(startedPath) : undefined, + stdoutLength: stdout.length, pool, spun, ended, @@ -818,16 +883,14 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { exit: exitCode, // Both streams' lines (law 15), with the scratch paths named. stream: streamOf(stderr.split(`\n`), (text) => - namedScratch(text, [ - [how.cli ?? here, ``], - [dir, ``], - ]), + namedScratch(text, scratchNames), ), stdoutStream: streamOf(stdout.split(`\n`), (text) => - namedScratch(text, [ - [how.cli ?? here, ``], - [dir, ``], - ]), + namedScratch(text, scratchNames), + ), + outDir: namedScratch(out, scratchNames), + log: streamLogOf(out, stderr, stdout, (text) => + namedScratch(text, scratchNames), ), // The session directories in the package once it has ended: a stopped // run leaves its own, and the next run removes it. @@ -839,6 +902,9 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { (file) => !/^\.oracle-gaps-\d+\//.test(file), ), } + // A default output directory is the run's own scratch. + if (given === undefined) rmSync(out, { recursive: true, force: true }) + return result } /** @@ -887,6 +953,107 @@ function filesUnder(root, within = ``) { ) } +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)) + +/** + * Runs the CLI twice on one output directory, the second while the first is + * live (the first has printed its note, so it has its stream log). The first + * run is the history's `cli`, and the second's ending is the `overlap` + * observation (the coordinator's decision of 2026-11-13 on F4: a second run + * refuses, and the first's stream log stays its own). + */ +async function overlappingRuns(dir, files, how) { + writeFixture(dir, files) + const before = new Set(filesUnder(dir)) + const { args, out } = cliArgs(dir, files, how) + const start = (holdUntilFile) => { + const child = spawn(process.execPath, args, { + cwd: repo, + // The first run holds its exit (and its output claim) until the driver + // releases it after the second run has been refused: the overlap is + // deterministic (ORACLE_GAPS_TEST_HOLD_UNTIL, index.mjs). + env: { + ...cleanEnv(), + ...(holdUntilFile === undefined + ? {} + : { ORACLE_GAPS_TEST_HOLD_UNTIL: holdUntilFile }), + }, + stdio: [`ignore`, `pipe`, `pipe`], + }) + const got = { stdout: ``, stderr: `` } + child.stdout.on(`data`, (chunk) => { + got.stdout += chunk + }) + child.stderr.on(`data`, (chunk) => { + got.stderr += chunk + }) + const status = new Promise((resolvePromise) => { + const timer = setTimeout(() => child.kill(), 300_000) + child.on(`close`, (code) => { + clearTimeout(timer) + resolvePromise(code) + }) + }) + return { got, status } + } + // The release file lives outside the package: a stray write there is judged. + const releaseDir = mkdtempSync(join(realpathSync(tmpdir()), `gap-release-`)) + const releaseFile = join(releaseDir, `overlap-release`) + const first = start(releaseFile) + let firstEnded = false + first.status.then(() => { + firstEnded = true + }) + // The first run's note is its first stdout line: its stream log exists. A + // first run that ends first never had a live output directory, so the + // overlap cannot be observed: the driver says so, rather than waiting. + for ( + let waited = 0; + !first.got.stdout.includes(`oracle-gaps: note log`) && + !firstEnded && + waited < 120_000; + waited += 20 + ) + await sleep(20) + if (!first.got.stdout.includes(`oracle-gaps: note log`)) + throw new Error( + `the first run ended before its stream log existed; an overlap needs a live first run`, + ) + const second = start() + const secondStatus = await second.status + writeFileSync(releaseFile, ``) + const firstStatus = await first.status + const ended = readRun( + dir, + out, + firstStatus, + first.got.stderr, + how.cli ?? here, + first.got.stdout, + ) + const refused = readRun( + dir, + out, + secondStatus, + second.got.stderr, + how.cli ?? here, + second.got.stdout, + ) + const replay = first.got.stderr + .split(`\n`) + .find((line) => line.startsWith(` replay: `)) + ?.slice(` replay: `.length) + return { + cli: { ...ended, replay, strayWrites: strays(dir, before) }, + overlap: { + exit: refused.exit, + ending: refused.ending, + stream: refused.stream, + stdoutStream: refused.stdoutStream, + }, + } +} + // Where a run may write in the package: only the output directories the // driver names with `--out` (`out`, `out-daemon`, `out-stop`, and // `out-only`). The daemon's state, the workspace dependencies' one-file @@ -970,6 +1137,33 @@ const streamedGap = * or a heartbeat that names a mutant came before the first verdict; * - `last`: the last line, which ends a run that did not complete. */ +/** + * What the run's stream log says (the maintainer's decision of 2026-11-12: + * the CLI writes its stream to `/stream.log`, and its first line on + * both streams names that log), as relations: the first line of each + * stream, with scratch paths named, whether the log exists, and whether its + * lines are the stderr's lines, in order. + */ +function streamLogOf(out, stderr, stdout, name) { + // The lines of a text, its final newline ending the last line. A blank line + // is a line: the log must equal the stderr's lines exactly (F10). + const lines = (text) => `${text ?? ``}`.replace(/\n$/, ``).split(`\n`) + let log + try { + log = readFileSync(join(out, `stream.log`), `utf8`) + } catch { + // No log: the run wrote none. + } + return { + stderrFirst: name(lines(stderr)[0] ?? ``), + stdoutFirst: name(lines(stdout)[0] ?? ``), + exists: log !== undefined, + equal: + log !== undefined && + JSON.stringify(lines(log)) === JSON.stringify(lines(stderr)), + } +} + function streamOf(lines, name) { const kinds = new RegExp(`^oracle-gaps: (?:${streamKinds.join(`|`)}) `) const strays = new Set() @@ -1017,20 +1211,55 @@ function streamOf(lines, name) { } function readRun(dir, out, status, stderr, cli, stdout) { + // A run without `--out` writes to the directory its stream log is in, which + // its first stdout line names (law 15); that directory is removed after the + // run is read, as the run's own scratch. + const chosen = + out ?? + `${stdout ?? ``}` + .split(`\n`)[0] + .match(/^oracle-gaps: note log (.*)\/stream\.log$/)?.[1] + try { + return readRunIn( + dir, + chosen ?? join(dir, `no-output`), + status, + stderr, + cli, + stdout, + out === undefined ? chosen : undefined, + ) + } finally { + if (out === undefined && chosen !== undefined) + rmSync(chosen, { recursive: true, force: true }) + } +} + +// `derived` is the output directory the CLI chose (no `--out`), which is named +// `` so that two runs of one history, each with its own directory, record +// the same trace (law 9's determinism). +function readRunIn(dir, out, status, stderr, cli, stdout, derived) { // The output names the CLI directory that ran and the package, as scratch // directories do, before it is cut to its last characters. const scratch = [ [cli, ``], [dir, ``], + ...(derived === undefined ? [] : [[derived, ``]]), ] const named = status === 0 ? undefined : namedScratch(`${stderr}`, scratch) let results let report let summary = `` + // The state of `results.json` in the output directory: missing, unparseable, + // or read (review of #57, batch 3: the message names which). + let reportState = `missing` try { - report = JSON.parse(readFileSync(join(out, `results.json`), `utf8`)) + const text = readFileSync(join(out, `results.json`), `utf8`) + reportState = `unparseable` + report = JSON.parse(text) results = report.results summary = readFileSync(join(out, `summary.md`), `utf8`) + reportState = `read` } catch { // A run that wrote no report is recorded by its exit status and output. } @@ -1101,6 +1330,10 @@ function readRun(dir, out, status, stderr, cli, stdout) { // The output directory the run was given, as the trace names it: the // final line names its report there. outDir: namedScratch(out, scratch), + // The run's stream log, and the first line that names it. + log: streamLogOf(out, stderr, stdout, (text) => + namedScratch(text, scratch), + ), // Whether the run captured its tests' own console output there. captured: (() => { try { @@ -1130,6 +1363,7 @@ function readRun(dir, out, status, stderr, cli, stdout) { ? undefined : `${status === 3 && existsSync(join(out, `error.log`)) ? `error.log: ${namedScratch(readFileSync(join(out, `error.log`), `utf8`), scratch).slice(0, 2000)}\n` : ``}${named.slice(-1500)}`, reported: results !== undefined, + reportState, streamed, // Law 9 compares everything the run reports except how it was // produced: durations, the options it was given (the output directory @@ -1285,6 +1519,7 @@ async function incremental(dir, plan, cli, copy, observe, states) { const key = fingerprint(files, state, step === `otherInstall`) const how = { seed: state.seed, + outMode: plan.outMode, cli: step === `otherInstall` ? otherCli() : copy, ordinary: plan.ordinary, sources: sourceArgument(plan, `${dir}.link`), @@ -1310,6 +1545,8 @@ async function incremental(dir, plan, cli, copy, observe, states) { : await daemonChecks(step, files, how) if (!cold.has(key)) cold.set(key, runCli(dir, files, how)) const coldRun = cold.get(key) + if (plan.outMode === `default`) + states.add(`a daemon check with the default output directory`) for (const [i, run] of [daemon, ...others].entries()) observe(`incremental`, { step: i === 0 ? step : `${step} (a second check at once)`, @@ -1327,6 +1564,11 @@ async function incremental(dir, plan, cli, copy, observe, states) { // The stream through the daemon (law 15): both streams' lines. stream: run.stream, stdoutStream: run.stdoutStream, + // Its stream log, and the output directory it names. + log: run.log, + outDir: run.outDir, + reported: run.reported, + reportState: run.reportState, // The first line of a failure, for the message; readRun has named // the scratch package and CLI copy in it. stderr: run.stderr?.split(`\n`)[0], @@ -1597,10 +1839,18 @@ async function driver(plan, observe) { ordinary: plan.ordinary, sources: sourceArgument(plan, `${dir}.link`), budget: plan.budget, + outMode: plan.outMode, + logBlocked: plan.logBlocked, cli: plan.cliPath === `spaced` ? copy : here, ...configArgs(plan), } - const { replay, ...cli } = runCli(dir, files, how) + // Two runs on one output directory: the second's refusal is its own + // observation (overlappingRuns). + const overlapping = plan.overlap + ? await overlappingRuns(dir, files, how) + : undefined + const { replay, ...cli } = overlapping?.cli ?? runCli(dir, files, how) + if (overlapping) observe(`overlap`, overlapping.overlap) // How many units were computed or reused says how the result was // produced, not what it is, so it is not observed. observe(`cli`, { ...cli, evidence: undefined }) @@ -1609,6 +1859,12 @@ async function driver(plan, observe) { observe(`replay`, { command: replay, ran: replayed(dir, replay) }) // Reach comes from what the CLI reported, mapped onto the plan. const states = new Set() + if (plan.outMode === `default`) + states.add(`a run with the default output directory`) + if (plan.logBlocked && cli.exit === 2) + states.add(`a stream log that cannot be written`) + if (overlapping && overlapping.overlap.exit === 2) + states.add(`two runs on one output directory`) if (cli.exit === 0 && !plan.consoleOutput) states.add(`a completed run whose tests wrote nothing`) // Law 14: a partial run of one owned function's mutants, whose baseline @@ -1651,7 +1907,20 @@ async function driver(plan, observe) { // session directories the package has once the run ended, and how many // of them the previous stopped run had left. if (cli.exit === 0 && stopIds.length > 0) - for (const stop of plan.stopRuns) { + for (const stop of [ + ...plan.stopRuns, + ...(plan.spawnStop + ? [ + { + signal: plan.spawnSignal, + pool: `threads`, + when: `spawn`, + delay: plan.spawnDelay, + onStart: plan.spawnOnStart, + }, + ] + : []), + ]) { const run = await stoppedRun(dir, files, how, stopIds, stop) const stopped = { ...run, @@ -1662,6 +1931,8 @@ async function driver(plan, observe) { }, } observe(`stopped`, stopped) + if (plan.outMode === `default`) + states.add(`a stop with the default output directory`) if (previousSessions.length > 0) states.add(`a run after a stopped one that left its session`) previousSessions = run.sessions @@ -1670,6 +1941,10 @@ async function driver(plan, observe) { states.add(`a stop signal after the completed line`) continue } + if (stop.when === `spawn`) { + states.add(`a stop as the CLI starts`) + continue + } if (stop.when === `warm`) { states.add(`a stop signal while the config warms`) continue @@ -2078,7 +2353,7 @@ async function driver(plan, observe) { * Law 9: each check through the daemon reports what the cold run of the same * state reports. A refused state is compared by the cause it names. */ -function incrementalCheck(observations) { +function incrementalCheck(observations, expected) { for (const { value } of observations.filter( (o) => o.checkpoint === `incremental`, )) { @@ -2097,7 +2372,10 @@ function incrementalCheck(observations) { // A refusal through the daemon ends as a cold refusal does (law 12): // the CLI that asked the daemon writes it to its own output directory. if (cold.refusal !== undefined) { - const breach = refusalBreach(daemon.ending, { outKnown: true }) + const breach = refusalBreach(daemon.ending, { + outKnown: true, + logNote: owesNote(expected) ? logNoteOf(daemon) : undefined, + }) if (breach) return violation( `incremental`, @@ -2168,6 +2446,24 @@ function check(plan, observations, expected) { `config`, { actual: cli.configLines, expected: expected.configLines }, ) + // A second run on a live output directory refuses before it writes + // anything (F4, the coordinator's decision of 2026-11-13): the refusal + // names the directory, is one block with its fix, and its stdout is its + // done line alone. The first run's own laws are judged on `cli`. + const overlap = observations.find((o) => o.checkpoint === `overlap`)?.value + if (overlap !== undefined) { + if (overlap.exit !== 2) + return violation( + `overlap`, + `the second run on a live output directory exited ${overlap.exit}, not refusing it`, + `ownership`, + ) + const breach = refusalBreach(overlap.ending, { + cause: `another run (pid `, + outKnown: false, + }) + if (breach) return violation(`overlap`, breach, `report-shaped ending`) + } const stream = streamBreach(`cli`, cli, expected, plan) if (stream) return stream // Through the daemon, each check streams the same kinds of line, and a @@ -2177,7 +2473,7 @@ function check(plan, observations, expected) { (o) => o.checkpoint === `incremental`, )) { const { daemon, cold } = value - const ends = endsBreach(daemon) + const ends = endsBreach(daemon, { checked: owesNote(expected) }) if (ends) return violation( `incremental`, @@ -2234,9 +2530,22 @@ function check(plan, observations, expected) { const breach = refusalBreach(cli.ending, { cause: expected.refused.cause, outKnown: true, + logNote: expected.refused.arguments ? undefined : logNoteOf(cli), }) if (breach) return violation(`cli`, breach, `report-shaped ending`) - return incrementalCheck(observations) ?? daemonCheck(observations) ?? pass + // A refusal of the arguments, or before the check starts, writes no + // stream log (F8, the coordinator's review of #57). + if (expected.refused.arguments && cli.log.exists) + return violation( + `cli`, + `the refusal before the check started wrote a stream log`, + `greppable stream`, + ) + return ( + incrementalCheck(observations, expected) ?? + daemonCheck(observations) ?? + pass + ) } if (cli.exit !== 0 || !cli.reported) return violation( @@ -2498,7 +2807,11 @@ function check(plan, observations, expected) { { actual: after, expected: before }, ) } - return incrementalCheck(observations) ?? daemonCheck(observations) ?? pass + return ( + incrementalCheck(observations, expected) ?? + daemonCheck(observations) ?? + pass + ) } /** @@ -2514,10 +2827,22 @@ function check(plan, observations, expected) { * (stopped). `run` is a run as readRun records it, observed at * `checkpoint`. */ +// The stage at which a run refuses decides whether it owes its note (law 15, +// review of #57): a refusal of the arguments comes before any check, so its +// output is its done line alone. Both the cold run and a run through the daemon +// are judged by this one decision. +const owesNote = (expected) => !expected.refused?.arguments function streamBreach(checkpoint, run, expected, plan) { const breach = (difference, extra) => violation(checkpoint, difference, `greppable stream`, extra) - const ends = endsBreach(run) + // The spawn-delay law (review of #57): a stop before the CLI started (no + // started marker) is outcome (a): no output, no log, the exit by signal. + // Once the marker exists, the outcome is (b) in full. + if (run.started === false) + return run.stdoutLength === 0 && !run.log.exists && run.exit !== 0 + ? undefined + : breach(`a stop before the CLI started left output or a log`) + const ends = endsBreach(run, { checked: owesNote(expected) }) if (ends) return breach(ends.difference, ends.extra) if (run.stream.mutantBeforeVerdict) return breach( @@ -2560,17 +2885,43 @@ function streamBreach(checkpoint, run, expected, plan) { return undefined } +/** The line that names a run's stream log. */ +const logNoteOf = (run) => `oracle-gaps: note log ${run.outDir}/stream.log` + /** * What law 15 judges of every run, on both streams, whatever its outcome: * no stray line, and exactly one done line, which ends each stream, the * same on both, of the kind its exit status says (0 completed, 2 refused, - * 3 error, and a signal's code stopped). Undefined when it holds. + * 3 error, and a signal's code stopped). With `checked`, the run started a + * check, so its first line names its stream log. Undefined when it holds. */ -function endsBreach(run) { +function endsBreach(run, { checked = true } = {}) { const streams = [ [`stderr`, run.stream], [`stdout`, run.stdoutStream], ] + // The run's first line on each stream names its stream log, and the log + // holds the stderr's lines, in order (the maintainer's decision of + // 2026-11-12), for every run that starts a check. A refusal of the run's + // arguments comes before any check, and names no log. + const note = logNoteOf(run) + if (checked && (run.log.stderrFirst !== note || run.log.stdoutFirst !== note)) + return { + difference: `the run's first lines are ${JSON.stringify([run.log.stderrFirst, run.log.stdoutFirst])}, not ${JSON.stringify(note)} on each stream`, + } + if (checked && !run.log.exists) + return { difference: `the run wrote no stream.log` } + if (checked && !run.log.equal) + return { + difference: `the run's stream.log is not its stderr, line for line`, + } + // A completed run's results.json sits beside its stream log, in the + // directory the log names (law 15's stream log, the maintainer's decision + // of 2026-11-12). + if (checked && run.exit === 0 && run.reported === false) + return { + difference: `the run completed, but ${run.outDir}/results.json is ${run.reportState ?? `missing or unparseable`}, not beside its stream log`, + } for (const [where, stream] of streams) if (stream.strays.length > 0) return { @@ -3035,6 +3386,33 @@ calibrations.push( }, ), }, + { + // A completed run that starts a check, so its log is judged (law 15). + name: `the stream log misses a line`, + law: `greppable stream`, + history: { constructed: `two authored mutants` }, + corrupt: corruptCli((cli) => + cli.exit !== 0 + ? undefined + : { ...cli, log: { ...cli.log, equal: false } }, + ), + }, + { + name: `the line that names the stream log comes after another line`, + law: `greppable stream`, + history: { constructed: `two authored mutants` }, + corrupt: corruptCli((cli) => + cli.exit !== 0 + ? undefined + : { + ...cli, + log: { + ...cli.log, + stderrFirst: `oracle-gaps: config vite.config.ts`, + }, + }, + ), + }, { name: `a stopped run ends without its done line`, law: `greppable stream`, @@ -3099,6 +3477,12 @@ const staticStates = [ `owned through bind`, ] const required = [ + `a stop as the CLI starts`, + `a stream log that cannot be written`, + `two runs on one output directory`, + `a run with the default output directory`, + `a stop with the default output directory`, + `a daemon check with the default output directory`, `killed by a second owner`, `killed`, `random-only kill`, @@ -3265,7 +3649,19 @@ const faults = [ `setupThrows`, ...analyzedFaults, ] +// The states that each named constructed history enters, and only that +// history: they are not witnessed by "every outcome class" (a named history +// of its own must enter them, or registration fails). +const ownHistoryStates = [ + `a stop as the CLI starts`, + `a stream log that cannot be written`, + `two runs on one output directory`, + `a run with the default output directory`, + `a stop with the default output directory`, + `a daemon check with the default output directory`, +] const separate = [ + ...ownHistoryStates, `ordinary tests left out by default`, `two authored mutants judged apart`, `a package with a prebuilt addon deep in its tree ran in child processes`, @@ -3324,6 +3720,12 @@ const withDeclaration = (constructed) => build: (generated) => item.build({ ...generated, + // The end-to-end dimensions a generated base may draw: a construction + // that does not name them runs without them (review of #57, apiLess). + logBlocked: false, + overlap: false, + spawnStop: false, + stopWitness: false, declaration: index % 2 === 0 ? `wrapper` : `oracleProperty`, ownerless: false, owners: [`a`, ...generated.owners.slice(1)], @@ -3333,8 +3735,8 @@ const withDeclaration = (constructed) => oracleProperty({ id: `oracle-gaps.report`, law: { - statement: `The gap report enumerates the promised operator mutants, leaves out exactly the probe-only code, discovers every oracle that owns a function's call tree and refuses broken ownership, judges each mutant only by its owners' tests on their registered routes and counts only law verdicts on reproducible routes as kills, which outrank a run that never ends, anchors each mutant by its function and normalized statement, lists every gap and unowned mutant once with consistent counts, keeps keys stable under unrelated edits, reports through one daemon per package what a cold run reports at every state of a sequence of edits, and bounds the runs that confirm a mutant's cut-off copies by one budget per mutant and names each busy mutant in a heartbeat while it runs, leaves nothing in the package and writes nothing where Vitest's include could find it while it runs, leaves no process running once it is stopped, runs the package as Vitest does from any install path and any --source spelling: one instance of each workspace dependency, no refusal over an unbuildable devDependency, child processes for a native addon anywhere in the tree, a capture that a test's builtin spies cannot see, and a refusal of test.projects; leads its report with every test file's baseline, blocks only the mutants a failing oracle file owns and judges the rest, prints every line of its stream with the prefix oracle-gaps: and its kind, one decisive baseline line before any mutant, and a done line last, and every run that does not complete ends, cold or through the daemon, with a refusal that exits 2, prints a refused: block and one fix: directive with no stack trace, and writes the refusal to the run's output directory.`, - authority: `The maintainer's decisions in design runs v11, v14, and v19 (2026-11-04 and 2026-11-05), not in this package's records, including that a kill by reach agreement alone is a gap and that an incremental run through the daemon equals a cold one (v19.3); the project's oracle guide on quiet exclusions; the oracle rubric's outcome kinds (ORC-006); the CLI's documented operator list in schemata.mjs; the maintainer's dispositions G1, G2, G4, G5, G6, and G13 of the framework papercut ledger (2026-11-11) and the v20 freeze's namespacing, one-loop, and provisional-report decisions (2026-11-10); the maintainer's request for a greppable stream (2026-11-12, relayed by the coordinator with its decisions of that day: the same prefixes on stdout and stderr, a done line for a stopped run, and no --max-gaps).`, + statement: `The gap report enumerates the promised operator mutants, leaves out exactly the probe-only code, discovers every oracle that owns a function's call tree and refuses broken ownership, judges each mutant only by its owners' tests on their registered routes and counts only law verdicts on reproducible routes as kills, which outrank a run that never ends, anchors each mutant by its function and normalized statement, lists every gap and unowned mutant once with consistent counts, keeps keys stable under unrelated edits, reports through one daemon per package what a cold run reports at every state of a sequence of edits, and bounds the runs that confirm a mutant's cut-off copies by one budget per mutant and names each busy mutant in a heartbeat while it runs, leaves nothing in the package and writes nothing where Vitest's include could find it while it runs, leaves no process running once it is stopped, runs the package as Vitest does from any install path and any --source spelling: one instance of each workspace dependency, no refusal over an unbuildable devDependency, child processes for a native addon anywhere in the tree, a capture that a test's builtin spies cannot see, and a refusal of test.projects; leads its report with every test file's baseline, blocks only the mutants a failing oracle file owns and judges the rest, prints every line of its stream with the prefix oracle-gaps: and its kind, a first line on both streams that names the stream log in the output directory, which holds the stderr's lines in order, one decisive baseline line before any mutant, and a done line last, and every run that does not complete ends, cold or through the daemon, with a refusal that exits 2, prints a refused: block and one fix: directive with no stack trace, and writes the refusal to the run's output directory.`, + authority: `The maintainer's decisions in design runs v11, v14, and v19 (2026-11-04 and 2026-11-05), not in this package's records, including that a kill by reach agreement alone is a gap and that an incremental run through the daemon equals a cold one (v19.3); the project's oracle guide on quiet exclusions; the oracle rubric's outcome kinds (ORC-006); the CLI's documented operator list in schemata.mjs; the maintainer's dispositions G1, G2, G4, G5, G6, and G13 of the framework papercut ledger (2026-11-11) and the v20 freeze's namespacing, one-loop, and provisional-report decisions (2026-11-10); the maintainer's request for a greppable stream (2026-11-12, relayed by the coordinator with its decisions of that day: the same prefixes on stdout and stderr, a done line for a stopped run, and no --max-gaps); the maintainer's decision of 2026-11-12 that the CLI writes its stream to the output directory's stream.log and names it in its first line.`, limits: `Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come only between checks, so a file saved during a check is not generated (the runner invalidates a known input that changed while a run ran and does not store that run, which only inspection fixes; a witness needs a step that edits a file between a unit's start and its end), and a module that a run first loads and that changes during that same run is stored under its later hash; law 9's edits leave out files read through fs, other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated.`, }, histories: { @@ -3399,6 +3801,165 @@ oracleProperty({ consoleOutput: true, }), }, + { + // Without `--out`, the CLI picks the run's output directory + // (args.mjs). A run stopped while its config warms must still leave + // its stream log beside its results, with the done line last on both + // streams (law 15). The stop witness runs alone (witness `stop`), and a history + // with the witness module takes no steps. + name: `a stop with the default output directory`, + enters: [ + `a run with the default output directory`, + `a stop with the default output directory`, + ], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `push`, quote: `'` }]], + containers: [], + overloads: [], + owners: [`a`], + library: [], + libraryOwners: [], + runes: [], + wholeFile: `none`, + witness: false, + stopWitness: true, + tests: [{ ...judging(`a`, `fixed seed`, 1), fn: 0 }], + authored: null, + fault: `none`, + stability: `none`, + steps: null, + ordinary: false, + outMode: `default`, + budget: `two`, + stopRuns: [{ signal: `SIGINT`, pool: `threads`, when: `warm` }], + }), + }, + { + // Through the daemon, a check that the CLI parses without `--out` + // would write its results to a directory of its own, apart from the + // stream log the CLI names (law 15). + name: `a daemon check with the default output directory`, + enters: [ + `a run with the default output directory`, + `a daemon check with the default output directory`, + ], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `push`, quote: `'` }]], + containers: [], + overloads: [], + owners: [`a`], + library: [], + libraryOwners: [], + runes: [], + wholeFile: `none`, + witness: false, + tests: [{ ...judging(`a`, `fixed seed`, 1), fn: 0 }], + authored: null, + fault: `none`, + stability: `none`, + steps: [`test`], + ordinary: false, + outMode: `default`, + budget: `two`, + }), + }, + { + // A stop sent as the CLI spawns, before its note's run has started: + // the note is written first, and the stopped line follows it on both + // streams, with one done line (P1 of the review of #57). + name: `a stop as the CLI starts`, + enters: [`a stop as the CLI starts`], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `push`, quote: `'` }]], + containers: [], + overloads: [], + owners: [`a`], + library: [], + libraryOwners: [], + runes: [], + wholeFile: `none`, + witness: false, + stopWitness: true, + tests: [{ ...judging(`a`, `fixed seed`, 1), fn: 0 }], + authored: null, + fault: `none`, + stability: `none`, + steps: null, + ordinary: false, + outMode: `explicit`, + logBlocked: false, + overlap: false, + spawnStop: true, + // The signal waits for the CLI's started marker, so it lands during + // the loads of a CLI that loads before it installs its handlers. + spawnOnStart: true, + budget: `two`, + }), + }, + { + // A stream log that is a directory cannot be written: the CLI refuses + // before its first line (index.mjs opens the log first), with its + // block, its fix, and its report in the output directory, and its + // stdout is its done line alone (law 12, law 15, and F3 of the + // coordinator's review of #57). No stack trace, and exit 2 or 3. + name: `a stream log that cannot be written`, + enters: [`a stream log that cannot be written`], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `push`, quote: `'` }]], + containers: [], + overloads: [], + owners: [`a`], + library: [], + libraryOwners: [], + runes: [], + wholeFile: `none`, + witness: false, + tests: [{ ...judging(`a`, `fixed seed`, 1), fn: 0 }], + authored: null, + fault: `none`, + stability: `none`, + steps: null, + ordinary: false, + budget: `two`, + outMode: `explicit`, + logBlocked: true, + overlap: false, + }), + }, + { + // A second run on the first's output directory while the first is + // live refuses (F4, the coordinator's decision of 2026-11-13): the + // first run's stream log still equals its own stderr, line for line, + // and the refusal reports no directory it does not own. + name: `two runs on one output directory`, + enters: [`two runs on one output directory`], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `push`, quote: `'` }]], + containers: [], + overloads: [], + owners: [`a`], + library: [], + libraryOwners: [], + runes: [], + wholeFile: `none`, + witness: false, + tests: [{ ...judging(`a`, `fixed seed`, 1), fn: 0 }], + authored: null, + fault: `none`, + stability: `none`, + steps: null, + ordinary: false, + budget: `two`, + outMode: `explicit`, + logBlocked: false, + overlap: true, + }), + }, { // The budget witness checked through the daemon twice with nothing // changed between: the second check reuses the confirmations the diff --git a/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs index 97e4a23..082b176 100644 --- a/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs @@ -89,6 +89,7 @@ * target. An authored mutant's `source` field is used as written. */ import { + existsSync, mkdirSync, mkdtempSync, readFileSync, @@ -375,6 +376,14 @@ const matcherOwns = fc.oneof( ) // Each history of the shared grammar, with its argument fault and its // matcher's `owns`. +// The end-to-end dimensions the static oracle does not draw: `logBlocked` +// (a stream log the run cannot write) and `overlap` (a second run on a live +// output directory). The static run creates no stream log and starts no +// second CLI process, so these are off for it by deliberate scope; the +// end-to-end oracle (gap-report-oracle) owns both (F3 and F4 of the review of +// #57). +const staticResolve = (history) => + resolveHistory({ ...history, logBlocked: false, overlap: false }) const argumentHistories = fc.tuple(histories, badArguments, matcherOwns).map( ([history, bad, matcher]) => ({ ...history, @@ -500,13 +509,19 @@ function argumentsRunOf(dir, plan) { encoding: `utf8`, timeout: 120_000, }) - return endingOf( - run.status, - run.stderr, - out, - (text) => text.replaceAll(dir, ``), - run.stdout, - ) + return { + ...endingOf( + run.status, + run.stderr, + out, + (text) => text.replaceAll(dir, ``), + run.stdout, + ), + // A refusal of the arguments, or before a check starts, writes no stream + // log (F8, the coordinator's review of #57). Judged where the run names + // its output directory; a default one lives in the OS temp directory. + streamLog: out !== undefined && existsSync(join(out, `stream.log`)), + } } // ## The derived class, run @@ -1099,6 +1114,12 @@ function check(plan, observations, expected) { `with ${bad.kind} arguments${bad.daemon ? ` and --daemon` : ``}, ${breach}`, `report-shaped ending`, ) + if (argumentsRun.streamLog) + return violation( + `arguments`, + `with ${bad.kind} arguments${bad.daemon ? ` and --daemon` : ``}, the refusal wrote a stream log`, + `greppable stream`, + ) // An owner that fails, and a config the runner cannot load or honor, are // found only when the runner starts, so the analysis must not refuse for // them: it refuses as it would for the same plan without the fault. @@ -1802,7 +1823,7 @@ oracleProperty({ }, ]), }, - resolve: resolveHistory, + resolve: staticResolve, model, driver, check, diff --git a/packages/oracles/oracle-gaps/index.mjs b/packages/oracles/oracle-gaps/index.mjs index 4321082..f1ee301 100755 --- a/packages/oracles/oracle-gaps/index.mjs +++ b/packages/oracles/oracle-gaps/index.mjs @@ -121,60 +121,264 @@ // SIGINT or SIGTERM is sent on to the whole group; a process that has not // ended 2 s later is killed, and so is any process left in the group when // the run ends. The run kills its group itself if the CLI dies first. +// The stop handlers and the claim on the output directory come before the +// CLI's modules load (review of #57, P1): a signal during the loads is kept +// and acted on once the stream log's note is written, so a stop as the CLI starts still +// leaves the note and one done line. Only Node's own start is before them. import { spawn } from 'node:child_process' -import { existsSync, realpathSync } from 'node:fs' +import { + appendFileSync, + existsSync, + linkSync, + mkdirSync, + readFileSync, + realpathSync, + rmSync, + writeFileSync, +} from 'node:fs' import { constants } from 'node:os' +import { join } from 'node:path' import { pathToFileURL } from 'node:url' -import { parseArgs, usage } from './args.mjs' -import { codeIdentity, openSession } from './check.mjs' -import { received, requestDaemon, stopDaemon } from './daemon.mjs' -import { endRun } from './ending.mjs' -import { memoryStore } from './evidence.mjs' + +let early +const keepEarly = (signal) => { + early ??= signal +} +process.on(`SIGINT`, keepEarly) +process.on(`SIGTERM`, keepEarly) +// A test-only marker (ORACLE_GAPS_TEST_STARTED): the first statement that runs +// project code writes this file, before any project module loads, so a test can +// tell a stop that came before the CLI started from one that came during its loads. +if (process.env.ORACLE_GAPS_TEST_STARTED) + writeFileSync(process.env.ORACLE_GAPS_TEST_STARTED, ``) +// A test-only pause after the marker (ORACLE_GAPS_TEST_PAUSE_AFTER_STARTED, ms): +// it blocks this thread synchronously, so a stop sent during it lands before the +// handlers below are installed (the widened witness of review F2). Inert unless set. +const pauseMs = Number(process.env.ORACLE_GAPS_TEST_PAUSE_AFTER_STARTED ?? 0) +if (pauseMs > 0) + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, pauseMs) + +const { parseArgs, usage } = await import(`./args.mjs`) +const { endRun, Refusal } = await import(`./ending.mjs`) // How long the group has to end after a stop signal before it is killed. const STOP_GRACE_MS = 2000 -/** Signals every process of group `group`; false when none is left. */ -function signalGroup(group, signal) { +// The stream log (the maintainer's decision of 2026-11-12): a run that +// starts a check writes its stderr to `/stream.log` too, line for line, +// and names that file in its first line on both streams. The supervisor opens +// it before the child starts (startLog), and is then the only writer of the +// child's stderr to it (superviseRun). Undefined until the log is open. +let streamLog +let writingEnding = false +const logPathOf = (out) => join(out, `stream.log`) +/** Writes `text` to stderr and, once the log is open, to the stream log. */ +const writeStream = (text) => { + process.stderr.write(text) + if (streamLog) appendFileSync(streamLog, text) +} +const cannotWrite = (path, error) => + new Refusal( + `the stream log ${path} cannot be written (${error.code ?? error.message})`, + `Remove ${path}, or pass --out a directory whose stream.log can be written.`, + ) +/** Whether process `pid` runs. */ +const alive = (pid) => { try { - process.kill(-group, signal) + process.kill(pid, 0) return true - } catch { + } catch (error) { + return error.code === `EPERM` + } +} +// The pid this process claimed each output directory with, released when the +// process exits (on every exit but SIGKILL). +const claims = new Set() +process.on(`exit`, () => { + for (const claim of claims) { + try { + if (Number(readFileSync(claim, `utf8`)) === process.pid) + rmSync(claim, { force: true }) + } catch { + // Already gone. + } + } +}) +/** + * Claims output directory `out` for this process (review of #57, P2 and P3): + * a run keeps it while it lives, and a second run on it refuses without + * writing into it (the maintainer's decision of 2026-11-13). The claim is a + * temp file that already holds the pid, linked to `run.pid`, which fails when + * `run.pid` exists, so the claim is atomic. A claim whose pid no longer runs + * is taken over. + */ +function claimOut(out) { + const claim = join(out, `run.pid`) + const takeover = `${claim}.takeover` + const temp = `${claim}.${process.pid}.tmp` + try { + mkdirSync(out, { recursive: true }) + writeFileSync(temp, `${process.pid}\n`) + } catch (error) { + throw cannotWrite(claim, error) + } + try { + for (;;) { + try { + linkSync(temp, claim) + claims.add(claim) + return + } catch (error) { + if (error.code !== `EEXIST`) throw cannotWrite(claim, error) + } + const holder = readHolder(claim) + if (holder === undefined) continue + // An unreadable claim counts as live: wait for it, never take it over. + if (!(holder > 0)) { + sleepMs(10) + continue + } + if (holder !== process.pid && holder > 0 && alive(holder)) { + const refusal = new Refusal( + `another run (pid ${holder}) is using ${out}`, + `If no run is using it, delete ${claim}; otherwise pass --out a directory no run is using.`, + ) + refusal.reportless = true + throw refusal + } + // A stale claim: only the racer that creates the takeover mutex removes it, + // and only if it still names the stale pid it read (review F4). + testPause(`takeover`) + if (!takeMutex(takeover)) { + sleepMs(10) + continue + } + try { + if (readHolder(claim) === holder) rmSync(claim, { force: true }) + } finally { + rmSync(takeover, { recursive: true, force: true }) + } + } + } finally { + rmSync(temp, { force: true }) + } +} +/** The pid a claim names, or undefined when the claim is gone. */ +function readHolder(claim) { + try { + return Number(readFileSync(claim, `utf8`)) + } catch (error) { + if (error.code === `ENOENT`) return undefined + throw cannotWrite(claim, error) + } +} +/** + * Takes the takeover mutex. The pid is written to a temp file first and linked + * into place, so no reader ever sees an empty mutex (review F4; register: #57 + * claim rules). A mutex whose pid can't be read counts as live and is never + * taken over; only a mutex whose recorded pid is dead is removed, and then made + * again. + */ +function takeMutex(mutex) { + const temp = `${mutex}.${process.pid}.tmp` + writeFileSync(temp, `${process.pid}\n`) + try { + testPause(`mutex-create`) + linkSync(temp, mutex) + return true + } catch (error) { + if (error.code !== `EEXIST`) throw error + const owner = readHolder(mutex) + if (owner === undefined) return false + if (owner > 0 && !alive(owner)) { + rmSync(mutex, { force: true }) + return takeMutex(mutex) + } return false + } finally { + rmSync(temp, { force: true }) + } +} +/** A test-only pause (ORACLE_GAPS_TEST_PAUSE_AT, inert unless set), see README. */ +function testPause(point) { + if (process.env.ORACLE_GAPS_TEST_PAUSE_AT === point) + Atomics.wait( + new Int32Array(new SharedArrayBuffer(4)), + 0, + 0, + Number(process.env.ORACLE_GAPS_TEST_PAUSE_MS ?? 0), + ) +} +function sleepMs(ms) { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms) +} +/** + * Claims the output directory and writes the run's first line, on both + * streams, which names the log. Throws a Refusal when the directory is in use + * or the log cannot be written; nothing is written then. + */ +function startLog(out) { + const path = logPathOf(out) + claimOut(out) + try { + writeFileSync(path, ``) + } catch (error) { + throw cannotWrite(path, error) } + streamLog = path + const note = `oracle-gaps: note log ${path}\n` + process.stdout.write(note) + writeStream(note) +} + +/** Flushes both streams, then exits with `code` (after a test's hold). */ +/** + * A test-only hold (ORACLE_GAPS_TEST_HOLD_UNTIL, a path): after its done line the + * process keeps its exit, and its output claim, until the file exists. A test + * releases it once the overlapping run or the stop it sent has been observed, so + * the outcome does not depend on wall-clock timing. Inert unless set. + */ +function holdUntil() { + const file = process.env.ORACLE_GAPS_TEST_HOLD_UNTIL + if (file !== undefined) while (!existsSync(file)) sleepMs(20) +} +function exitWhenFlushed(code) { + process.stdout.write(``, () => + process.stderr.write(``, () => { + holdUntil() + process.exit(code) + }), + ) } /** - * Runs this CLI again as a child in a process group of its own, and ends - * with it: with its exit code, or, once a stop signal has ended its group, - * with 128 plus the signal's number. + * Runs this CLI again as a child in a process group of its own, and ends with + * it: with its exit code, or, once a stop signal has ended its group, with 128 + * plus the signal's number. The child gets `--out` appended, the directory this + * process resolved (review of #57, F2). */ -function superviseRun() { +function superviseRun(out) { let child let stopped // How the run ended: its exit code, or the signal that ended it. let ended // Whether the run said that it writes its ending (a message it sends - // before it writes): then its done line is the last on each stream, and - // it ends with its own code, whatever stop reaches it. + // before it writes): then its done line is the last on each stream, and it + // ends with its own code, whatever stop reaches it. let ending = false - // Whether the run's stderr, which the CLI passes on, said that V8 ran - // out of memory. + // Whether the run's stderr, which the CLI passes on, said that V8 ran out + // of memory. let outOfMemory = false let stderrTail = `` // Whether the run's streams and channel have closed. let closed = false - // Ends the CLI once the run has ended (or the grace has passed), with - // what is left of the group killed, and once the run's stderr has been - // passed on (at most 2 s more). A run that writes its ending wrote its - // done line, so the CLI ends with its code and writes none. Every other - // end gets one done line from the CLI, on both streams: a run that a stop - // ended stopped (review of #46, F3; re-review N1); a run whose process - // ended on its own, by a signal or a code, with no ending (a V8 - // out-of-memory abort is a SIGABRT) is an error, exit 3, with the remedy - // when it ran out of memory (the coordinator's decision of 2026-11-12 on - // P10). A write to a pipe is asynchronous on macOS, so the process exits - // once it is written. + // Ends the CLI once the run has ended (or the grace has passed), with what + // is left of the group killed. A run that writes its ending wrote its done + // line, so the CLI ends with its code and writes none. Every other end gets + // one done line from the CLI, on both streams: a stop ended the run + // (stopped); a run whose process ended on its own (a signal, a code, a V8 + // out-of-memory abort) is an error, exit 3, with the remedy when it ran out + // of memory. let finishing = false const finish = () => { if (finishing) return @@ -192,11 +396,13 @@ function superviseRun() { const line = stopped ? `oracle-gaps: done stopped (${stopped})\n` : `oracle-gaps: done error, the run's process exited with ${how}${outOfMemory ? `: it ran out of memory; give Node more (NODE_OPTIONS=--max-old-space-size=) or run fewer --contexts` : ``}\n` - process.stdout.write(line, () => + process.stdout.write(line, () => { + // The log was opened before the run started (startLog). + appendFileSync(streamLog, line) process.stderr.write(line, () => process.exit(stopped ? 128 + constants.signals[stopped] : 3), - ), - ) + ) + }) } if (child === undefined || closed) return decide() child.once(`close`, decide) @@ -218,25 +424,28 @@ function superviseRun() { finish() }, 50) } - // The stop handlers come first, so a signal that comes while the run - // starts is a stop too (re-review N3). + // The stop handlers come first, so a signal that comes while the run starts + // is a stop too. A stop that came during the CLI's loads ends the run before + // it starts: its stopped line follows the note the caller already wrote. process.on(`SIGINT`, stop) process.on(`SIGTERM`, stop) - // The test-only stop hook (ORACLE_GAPS_TEST_STOP, review F(f)): in a - // supervised run it goes through this supervisor's stop, which forwards to - // the child and writes the stopped line, as a SIGINT does. + // A test-only stop (ORACLE_GAPS_TEST_STOP, #58): once the named file exists, the + // stop is taken as a SIGINT would be. Inert unless set. if (process.env.ORACLE_GAPS_TEST_STOP !== undefined) { const file = process.env.ORACLE_GAPS_TEST_STOP setInterval(() => { if (existsSync(file)) stop(`SIGINT`) }, 50).unref() } + if (early !== undefined) return stop(early) // An inspector flag would bind the CLI's own port a second time. child = spawn( process.execPath, [ ...process.execArgv.filter((flag) => !/^--inspect/.test(flag)), ...process.argv.slice(1), + `--out`, + out, ], { detached: true, @@ -247,10 +456,12 @@ function superviseRun() { child.on(`message`, (message) => { if (message === `ending`) ending = true }) - // The run's stderr, passed on as it comes, and read for V8's - // out-of-memory message. + // The run's stderr, passed on as it comes, written to the stream log as it is + // (V8's own lines included, which the run does not write itself), and read + // for V8's out-of-memory message. child.stderr.on(`data`, (chunk) => { process.stderr.write(chunk) + appendFileSync(streamLog, chunk) stderrTail = (stderrTail + chunk).slice(-200) if (/heap out of memory/i.test(stderrTail)) outOfMemory = true }) @@ -263,9 +474,19 @@ function superviseRun() { }) } +/** Signals every process of group `group`; false when none is left. */ +function signalGroup(group, signal) { + try { + process.kill(-group, signal) + return true + } catch { + return false + } +} + /** - * The run's side of the group: once the CLI that started it is gone, the - * run kills its group, itself included. + * The run's side of the group: once the CLI that started it is gone, the run + * kills its group, itself included. */ function watchSupervisor(supervisor) { setInterval(() => { @@ -274,23 +495,28 @@ function watchSupervisor(supervisor) { } /** - * Whether arguments `argv` ask for a check, the only run that starts - * processes; a refusal of them is written by the run itself. + * The options of arguments `argv` when they ask for a check, the only run that + * starts processes; undefined for help, a stop, or arguments that refuse (a + * refusal is written by the run itself). */ function asksForCheck(argv) { try { const options = parseArgs(argv) - return !options.help && !options.stop + return options.help || options.stop ? undefined : options } catch { - return false + return undefined } } /** The summary of a check run in this process. */ async function checkCold(options) { + const { openSession } = await import(`./check.mjs`) + const { memoryStore } = await import(`./evidence.mjs`) const session = await openSession(options.pkg) try { - return await session.check(options, memoryStore()) + return await session.check(options, memoryStore(), (line) => + writeStream(`${line}\n`), + ) } finally { await session.close() } @@ -298,16 +524,35 @@ async function checkCold(options) { /** The summary of a check run by the package's daemon. */ async function checkThroughDaemon(options) { + const { codeIdentity } = await import(`./check.mjs`) + const { received, requestDaemon } = await import(`./daemon.mjs`) const reply = await requestDaemon( options.pkg, process.argv.slice(2), codeIdentity(), ) - if (reply.stderr) process.stderr.write(reply.stderr) + if (reply.stderr) writeStream(reply.stderr) if (reply.failed) throw received(reply.failed) return reply.stdout } +// The run's ending, once: its refusal or error is written to its output +// directory and stdout and stderr, and the process exits with its code. +let ended = false +const holdStops = () => { + writingEnding = true + process.send?.(`ending`) +} +function endWith(error, out) { + if (ended) return + ended = true + const ending = endRun(error, out) + holdStops() + writeStream(ending.stderr) + process.stdout.write(`${ending.done}\n`) + exitWhenFlushed(ending.exitCode) +} + // A file URL escapes a space, `#`, and the like, so the script's own URL is // compared with the URL of the path it was started as. const main = @@ -315,93 +560,78 @@ const main = // Windows has no process groups: there the run is the CLI's own process. const supervised = process.platform !== `win32` const supervisor = process.env.ORACLE_GAPS_GROUP -if ( - main && - supervised && - supervisor === undefined && - asksForCheck(process.argv.slice(2)) -) - superviseRun() -else if (main) { + +// Set once the note is written (the unsupervised run has started). +let runStarted = false +/** Ends an unsupervised run with its stopped line on both streams (#58). */ +function stopNow(signal) { + const line = `oracle-gaps: done stopped (${signal})\n` + process.stdout.write(line) + writeStream(line) + exitWhenFlushed(128 + constants.signals[signal]) +} + +async function run() { + // The supervisor: it claims the output directory, writes the note, and runs + // the child. A refusal of the claim or the log ends it through endWith (the + // single path for every refusal of a check). + if (supervised && supervisor === undefined) { + const checking = asksForCheck(process.argv.slice(2)) + if (checking !== undefined) { + try { + startLog(checking.out) + } catch (error) { + return endWith(error, checking.out) + } + return superviseRun(checking.out) + } + } if (supervisor !== undefined) watchSupervisor(Number(supervisor)) // A test that runs this CLI starts a run of its own, in a group of its own. delete process.env.ORACLE_GAPS_GROUP - // The run's options, once its arguments parse; a refusal or an error is - // written to their output directory. - let options - let ended = false - // Ends the run once. A write to a pipe is asynchronous on macOS, and - // exiting at once cuts it off at the pipe's buffer, so the process exits - // once both streams have written everything before it. - // A test-only hook for the CLI's own oracle, not for users: - // ORACLE_GAPS_TEST_HOLD_EXIT_MS holds the exit after the done line, so the - // CLI's oracle can send a signal in that window. - const end = (exitCode) => { - if (ended) return - ended = true - const hold = Number(process.env.ORACLE_GAPS_TEST_HOLD_EXIT_MS ?? 0) - process.stdout.write(``, () => - process.stderr.write(``, () => - setTimeout(() => process.exit(exitCode), hold), - ), - ) - } - // A stop ends the run at once, with no line: an error that the stop - // causes (a config load that it cuts off) is no refusal, and the - // supervisor writes the stopped line (re-review N1). Once the run writes - // its ending, it says so to the supervisor, and a stop no longer cuts it - // off: its done line is then the last line of each stream. - let writingEnding = false - // A run that has started owes its stop one stopped line on each stream - // (#57, all-or-nothing). A supervised run's supervisor writes it; an - // unsupervised run (win32, no process groups) writes it here. A stop before - // the run starts prints nothing, as the law allows. - let runStarted = false + // A stop ends the run at once, with no line: an error that the stop causes + // is no refusal, and the supervisor writes the stopped line. Once the run + // writes its ending, a stop no longer cuts it off. + // The all-or-nothing stop law (#57 and #58): a stop after the note writes one + // done stopped line on each stream; a supervised child's stop is the + // supervisor's line; a stop before the note waits for it (the early stop). const onStop = (signal) => { if (writingEnding) return - const code = 128 + constants.signals[signal] - if (!runStarted || supervised) return process.exit(code) - const line = `oracle-gaps: done stopped (${signal})\n` - process.stdout.write(line, () => - process.stderr.write(line, () => process.exit(code)), - ) + if (!supervised) return runStarted && stopNow(signal) + process.exit(128 + constants.signals[signal]) } process.on(`SIGINT`, onStop) process.on(`SIGTERM`, onStop) - // A test-only hook for the CLI's own oracle, not for users (as - // ORACLE_GAPS_TEST_HOLD_EXIT_MS): when the file ORACLE_GAPS_TEST_STOP names - // appears, the CLI takes the same stop as a SIGINT. CI's Windows witness uses - // it, since a non-console parent cannot deliver Ctrl+C there. - // Unsupervised runs only (win32, no supervisor); a supervised child's stop - // is the supervisor's. + // A test-only stop for unsupervised runs (ORACLE_GAPS_TEST_STOP, #58). if (process.env.ORACLE_GAPS_TEST_STOP !== undefined && !supervised) { const file = process.env.ORACLE_GAPS_TEST_STOP setInterval(() => { if (existsSync(file)) onStop(`SIGINT`) }, 50).unref() } - const holdStops = () => { - writingEnding = true - process.send?.(`ending`) - } - const fail = (error) => { - if (ended) return - const ending = endRun(error, options?.out) - holdStops() - process.stderr.write(ending.stderr) - process.stdout.write(`${ending.done}\n`) - end(ending.exitCode) - } - // An error thrown outside the awaited check, by a timer or a promise no - // one awaits, ends the run the same way. - process.on(`uncaughtException`, fail) - process.on(`unhandledRejection`, fail) + // A child of a supervisor that was stopped before its own handlers ran exits + // now (the supervisor writes the line). An unsupervised run keeps the stop + // and takes it once its note is written, below. + if (supervisor !== undefined && early !== undefined) + process.exit(128 + constants.signals[early]) + // The run's options, once its arguments parse. + let options + // An error thrown outside the awaited check, by a timer or a promise no one + // awaits, ends the run the same way. + process.on(`uncaughtException`, (error) => endWith(error, options?.out)) + process.on(`unhandledRejection`, (error) => endWith(error, options?.out)) try { options = parseArgs(process.argv.slice(2)) if (options.help) console.log(usage) - else if (options.stop) await stopDaemon(options.pkg) - else { + else if (options.stop) { + const { stopDaemon } = await import(`./daemon.mjs`) + await stopDaemon(options.pkg) + } else { + // Windows has no supervisor: this process opens the log and runs the + // check. A child of a supervisor writes its stderr alone. + if (supervisor === undefined) startLog(options.out) runStarted = true + if (early !== undefined) return stopNow(early) const summary = await (options.daemon ? checkThroughDaemon : checkCold)( options, ) @@ -409,10 +639,12 @@ else if (main) { // The summary ends with the done line (report.mjs), which ends stderr // too. console.log(summary) - process.stderr.write(`${summary.split(`\n`).at(-1)}\n`) + writeStream(`${summary.split(`\n`).at(-1)}\n`) } - end(0) + exitWhenFlushed(0) } catch (error) { - fail(error) + endWith(error, options?.out) } } + +if (main) await run() diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index 986c404..27191c5 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -41,6 +41,10 @@ const suite = `oracle-gaps.report` export const shards = { // 522 s in the measured run, about 221 s idle. 'e2e-1': [ + `fixed seed 20261105: a stop with the default output directory`, + `fixed seed 20261105: a daemon check with the default output directory`, + `fixed seed 20261105: a stream log that cannot be written`, + `fixed seed 20261105: two runs on one output directory`, `fixed seed 20261105: a .cjs config that calls require`, `fixed seed 20261105: a declaration fault: missingSource`, `fixed seed 20261105: a declaration fault: noTypescript`, @@ -74,6 +78,7 @@ export const shards = { ], // 522 s in the measured run, about 221 s idle. 'e2e-2': [ + `fixed seed 20261105: a stop as the CLI starts`, `fixed seed 20261105: a declaration fault: setupThrows`, `fixed seed 20261105: a package with no config`, `fixed seed 20261105: an ambiguous authored mutant`, @@ -106,6 +111,11 @@ export const shards = { ], // 522 s in the measured run, about 221 s idle. 'e2e-3': [ + `determinism: constructed history: a stop as the CLI starts`, + `random seed: a stop with the default output directory`, + `random seed: a daemon check with the default output directory`, + `random seed: a stream log that cannot be written`, + `random seed: two runs on one output directory`, `fixed seed 20261105: a config fault in a run that owns nothing`, `fixed seed 20261105: a declaration fault: brokenOwner`, `fixed seed 20261105: a declaration fault: configThrows`, @@ -136,6 +146,7 @@ export const shards = { ], // 522 s in the measured run, about 221 s idle. 'e2e-4': [ + `random seed: a stop as the CLI starts`, `fixed seed 20261105: a .ts config in a package with no module type that calls require`, `fixed seed 20261105: a config that --config names`, `fixed seed 20261105: a declaration fault: apiLessTypescript`, @@ -167,6 +178,9 @@ export const shards = { // 79 s and 70 s; the daemon-notes one, bound to its history, 2 s (it // took 433 s searching for one). `calibration: a streamed line has no prefix`, + // The stream log's calibrations, not yet measured. + `calibration: the stream log misses a line`, + `calibration: the line that names the stream log comes after another line`, `calibration: a refusal through the daemon drops the cold run's notes`, // Bound to its history (re-review N2), not yet measured. `calibration: a run names a test-output log it never captured`, @@ -178,6 +192,10 @@ export const shards = { ], // 522 s in the measured run, about 221 s idle. 'e2e-5': [ + `determinism: constructed history: a stop with the default output directory`, + `determinism: constructed history: a daemon check with the default output directory`, + `determinism: constructed history: a stream log that cannot be written`, + `determinism: constructed history: two runs on one output directory`, `fixed seed 20261105: an edit sequence through the daemon`, `fixed seed 20261105: ordinary tests left out by default`, `fixed seed 20261105: two authored mutants`, @@ -264,6 +282,8 @@ function listed() { const run = spawnSync( `pnpm`, [ + // --silent keeps pnpm's workspace banner off stdout, which is JSON. + `--silent`, `exec`, `vitest`, `list`, diff --git a/packages/oracles/scripts/pr-shards.mjs b/packages/oracles/scripts/pr-shards.mjs index 9e16288..bb47680 100644 --- a/packages/oracles/scripts/pr-shards.mjs +++ b/packages/oracles/scripts/pr-shards.mjs @@ -524,6 +524,8 @@ export const partitions = { "test.projects in a parent's config", ]), 2: anyOf([ + 'a stop with the default output directory', + 'a daemon check with the default output directory', 'a .ts config in a package with no module type that calls require', 'a declaration fault: brokenVitest', 'a declaration fault: noTypescript', @@ -533,6 +535,9 @@ export const partitions = { 'two authored mutants', ]), 3: anyOf([ + 'a stop as the CLI starts', + 'a stream log that cannot be written', + 'two runs on one output directory', 'a config fault in a run that owns nothing', 'a declaration fault: configNoDefault', 'a declaration fault: nonLiteral',