diff --git a/packages/oracles/oracle-gaps/check.mjs b/packages/oracles/oracle-gaps/check.mjs index 9ca8b5b..7acd11e 100644 --- a/packages/oracles/oracle-gaps/check.mjs +++ b/packages/oracles/oracle-gaps/check.mjs @@ -553,6 +553,8 @@ export default async (env) => { return lastServed.get(index) } const indexOf = new Map() + // The units the check's store holds, set by check() (its store's size). + let heldUnits = () => 0 let contexts = [] let contextsKey let contextsConfig @@ -561,6 +563,10 @@ export default async (env) => { let seedVariable = `ORACLE_SEED` const contextSpecs = new Map() let runCounter = 0 + // The current check's evidence, which runBatch counts its batches in (set by check). + let evidenceOf + // The output directory of the check being run (the test hook names each line with it). + let hookRun // A context loads the package's own config through its wrapper, so an // error there is the author's: the run refuses, naming the config file @@ -1018,7 +1024,11 @@ export default async (env) => { */ let copyCounter = 0 async function runBatch(context, out, items, seed) { + evidenceOf.batches += 1 const capture = join(out, `run-${process.pid}-${++runCounter}.jsonl`) + const before = process.env.ORACLE_GAPS_TEST_RETENTION + ? retentionTotals(context) + : undefined // Each copy is named beside its file, so that its imports resolve as the // file's do, and the wrapper's plugin serves its text: nothing is // written there. @@ -1275,11 +1285,189 @@ export default async (env) => { } } finally { for (const copy of copies) globalThis.__oracleGapsCopies.delete(copy) + // A run cut off by a hang is still running: its worker may still send + // updates, so its state is not touched here. Its context is replaced + // by the caller (replace), which closes it and drops its state. + if (result !== undefined) { + retireCopies(context, copies) + retiredEntries(context, before, copies) + } else retiredEntries(context, before, copies, true) rmSync(capture, { force: true }) } } + /** + * Removes a batch's copies from Vitest's state and from Vite's module + * graphs once their units are plain data (the register's C entry: the gap + * CLI keeps no Vitest or Vite state for a mutant copy after the copy's + * results are plain data). Each copy is a new file. Vitest keeps its path + * in `filesMap` and `pathsSet`, and the file and test tasks it ran in + * `idMap`, and Vite keeps a module node, with its transform, per copy. + * Vitest's `clearFiles` replaces the path's file entry only, and leaves + * `pathsSet` and the test tasks, so it is not used: each entry is removed. + * Every map touched is guarded, and a Vitest or Vite whose shape changes + * throws here rather than leaking quietly. During a run, Vitest holds the + * run's copies until it ends, so the in-run peak scales with the batch's + * share of copies, not with the suite. + */ + function retireCopies(context, copies) { + const { filesMap, idMap, pathsSet } = context.state ?? {} + if ( + !(filesMap instanceof Map) || + !(idMap instanceof Map) || + !(pathsSet instanceof Set) + ) + throw new Error( + `oracle-gaps: this Vitest keeps no filesMap, idMap, or pathsSet in its state, so a check cannot retire its copies`, + ) + const gone = new Set(copies) + for (const [id, task] of idMap) + if (gone.has(task.file?.filepath ?? task.filepath)) idMap.delete(id) + for (const copy of copies) { + filesMap.delete(copy) + pathsSet.delete(copy) + } + retireModules(context.getRootProject().vite, gone) + } + + /** + * Removes the copies' module nodes from every environment's Vite graph. + * Each node is invalidated through Vite's API (which drops its transform), + * then removed from the graph's id, url, etag, and file maps, and from its + * neighbours' import sets. The graph's map shapes are guarded. + */ + function retireModules(server, gone) { + const environments = viteEnvironments(server) + let removed = 0 + for (const environment of environments) { + const graph = environment.moduleGraph + const { + urlToModuleMap, + idToModuleMap, + etagToModuleMap, + fileToModulesMap, + } = graph ?? {} + if ( + !(urlToModuleMap instanceof Map) || + !(idToModuleMap instanceof Map) || + !(etagToModuleMap instanceof Map) || + !(fileToModulesMap instanceof Map) || + typeof graph.invalidateModule !== `function` + ) + throw new Error( + `oracle-gaps: this Vite's module graph has no id, url, etag, or file maps, so a check cannot retire its copies`, + ) + for (const copy of gone) { + for (const mod of fileToModulesMap.get(copy) ?? []) { + removed += 1 + graph.invalidateModule(mod) + for (const imported of mod.importedModules) + imported.importers.delete(mod) + for (const importer of mod.importers) + importer.importedModules.delete(mod) + idToModuleMap.delete(mod.id) + urlToModuleMap.delete(mod.url) + // Vite keys the etag map by the etag, whose value is the node. + for (const [etag, node] of etagToModuleMap) + if (node === mod) etagToModuleMap.delete(etag) + } + fileToModulesMap.delete(copy) + } + } + return removed + } + + // Vite's environments, each with its module graph. A Vite that exposes none + // would make the module step a silent no-op, so it throws here. + function viteEnvironments(server) { + const environments = Object.values(server?.environments ?? {}) + if (environments.length === 0 || environments.some((e) => !e?.moduleGraph)) + throw new Error( + `oracle-gaps: this Vitest exposes no Vite environments with module graphs, so a check cannot retire its copies' module nodes`, + ) + return environments + } + + // Every structure a copy can leave state in, unfiltered: Vitest's state + // (filesMap, pathsSet, idMap, errorsSet, leakSet) and each Vite environment's + // module graph (its four maps, and the nodes that hold a transform). + function retentionTotals(context) { + const { filesMap, pathsSet, idMap, errorsSet, leakSet } = context.state + const totals = { + // The cold check's store: units it keeps for reuse (none, see evidence.mjs). + store: heldUnits(), + // Maps keyed by a context: each holds one entry per live context. A replaced + // context left in one is stale: the excess over the live contexts. + staleIndexed: indexOf.size - contexts.length, + staleSpecs: contextSpecs.size - contexts.length, + filesMap: filesMap.size, + pathsSet: pathsSet.size, + idMap: idMap.size, + errorsSet: errorsSet.size, + leakSet: leakSet.size, + } + for (const [name, environment] of Object.entries( + context.getRootProject().vite?.environments ?? {}, + )) { + const graph = environment.moduleGraph + // Vite's keys are named `vite..`, so the law can tell them from Vitest's state. + totals[`vite.${name}.url`] = graph.urlToModuleMap.size + totals[`vite.${name}.id`] = graph.idToModuleMap.size + totals[`vite.${name}.etag`] = graph.etagToModuleMap.size + totals[`vite.${name}.file`] = graph.fileToModulesMap.size + totals[`vite.${name}.transformed`] = [ + ...graph.idToModuleMap.values(), + ].filter((node) => node.transformResult != null).length + } + return totals + } + + // The test hook (ORACLE_GAPS_TEST_RETENTION, a file path): after a run + // that settled and its copies are retired, one line holds each structure's + // total before the batch and after its retirement. The bounded-retention + // law reads it: every total returns to its size before the batch. + // Files, without their query, with more than one node in an environment's graph, outside the copies: + // a non-copy file is cached once, so each count must be 0 (the law's bound on originals). + function nonCopyDuplicates(context, owned) { + let dup = 0 + for (const environment of Object.values( + context.getRootProject().vite?.environments ?? {}, + )) { + const perFile = new Map() + for (const node of environment.moduleGraph.idToModuleMap.values()) { + const file = node.id.split(`?`)[0] + if (owned.has(file)) continue + perFile.set(file, (perFile.get(file) ?? 0) + 1) + } + for (const n of perFile.values()) if (n > 1) dup += 1 + } + return dup + } + + function retiredEntries(context, before, copies, cut = false) { + const file = process.env.ORACLE_GAPS_TEST_RETENTION + if (!file) return + // A cut run is not retired (its context is replaced), so its line has no after: its before still + // names the stale totals the next batch starts from. + // Vite's graph also holds modules that are not copies (a source or harness file is cached on its + // first import, legitimately), so for Vite the law counts the copies' own nodes, which must be 0. + const owned = new Set(copies) + let copyNodes = 0 + for (const environment of Object.values( + context.getRootProject().vite?.environments ?? {}, + )) + for (const copy of owned) + copyNodes += + environment.moduleGraph.fileToModulesMap.get(copy)?.size ?? 0 + appendFileSync( + file, + `${JSON.stringify(cut ? { run: hookRun, batch: evidenceOf.batches, before, cut: true } : { run: hookRun, batch: evidenceOf.batches, before, after: retentionTotals(context), copyNodes, nonCopyDup: nonCopyDuplicates(context, owned) })}\n`, + ) + } + async function check(options, store, log = console.error) { + hookRun = options.out + heldUnits = () => store.size?.() ?? 0 const authored = options.mutants ? loadAuthored(options.mutants) : [] seedVariable = `${options.envPrefix}_SEED` if (options.hangLimit > options.mutantLimit) @@ -1412,7 +1600,10 @@ export default async (env) => { unchanged: 0, // Confirmations that a mutant's budget cut short or left no time. budget: 0, + // Every batch run (runBatch), cut or not: each writes one retention line. + batches: 0, } + evidenceOf = evidence // Each file's imports in its unmutated run. A run that timed out reports // none, so it is stored under these: a mutant changes no import. const fileInputs = new Map() @@ -1726,6 +1917,10 @@ export default async (env) => { closeContext(slot.context).catch(() => undefined), sleep(10_000), ]) + // The replaced context leaves every map keyed by it, so it is no longer + // reachable (its Vitest and Vite state with it). + indexOf.delete(slot.context) + contextSpecs.delete(slot.context) slot.context = await warmContext(settings, slot.index) await globSpecs(slot.context) contexts[slot.index] = slot.context diff --git a/packages/oracles/oracle-gaps/evidence.mjs b/packages/oracles/oracle-gaps/evidence.mjs index dd018c1..5a2e958 100644 --- a/packages/oracles/oracle-gaps/evidence.mjs +++ b/packages/oracles/oracle-gaps/evidence.mjs @@ -57,14 +57,14 @@ const withVersion = (row, variant) => ), ].slice(0, VERSIONS) -/** A store that lives for one check. */ +/** + * The store of a cold check. It keeps no unit: a cold run serves no lookup + * (its evidence reports `reused` 0 on every klona run measured), so a unit + * kept for reuse would only be retained, per mutant copy, for the whole run. + * `size` counts what it holds, for the retention law (check.mjs). + */ export function memoryStore() { - const rows = new Map() - return { - get: (id) => rows.get(id), - put: (id, variant) => - rows.set(id, { id, variants: withVersion(rows.get(id), variant) }), - } + return { get: () => undefined, put: () => {}, size: () => 0 } } /** diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 9f0fe4c..1280029 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -639,6 +639,48 @@ const configArgs = (plan) => ({ contexts: plan.contexts, }) +// Exact: a run writes one retention line per batch it ran (evidence.batches, cut batches +// included), so its line count must equal that counter. Zero batches write none. +// The batch-line count is owed only with a report: a run with no report is judged by the completion +// law (cli: the model predicts a report unless it predicts a refusal), which fails it there with its +// stderr. This check does not fail a run for lacking a report, so the two laws never both judge it. +const unlogged = (run) => + run.evidence === undefined || run.retention.length === run.evidence.batches + ? 0 + : 1 + +// A batch line (ORACLE_GAPS_TEST_RETENTION) left state behind when a total +// after its retirement differs from its size before the batch. +const leftBehind = (line) => + (line.copyNodes ?? 0) !== 0 || + (line.nonCopyDup ?? 0) !== 0 || + Object.keys(line.before).some( + (key) => + // Vite's totals are `vite..`: its non-copy modules are cached legitimately + // (checked by the copies' own nodes, copyNodes, and by the bound still to add). + !key.startsWith(`vite.`) && + ((key.startsWith(`stale`) && line.before[key] !== 0) || + (line.after !== undefined && line.before[key] !== line.after[key])), + ) + +const hookFiles = new Map() +let runSeq = 0 +const hookOf = (dir) => { + if (!hookFiles.has(dir)) + hookFiles.set( + dir, + join(mkdtempSync(join(tmpdir(), `gap-retention-`)), `retention.jsonl`), + ) + return hookFiles.get(dir) +} +const retentionLinesOf = (file) => + existsSync(file) + ? readFileSync(file, `utf8`) + .split(`\n`) + .filter(Boolean) + .map((line) => JSON.parse(line)) + : [] + /** * Writes the fixture's files and runs the CLI on it. A file of an earlier * state that this state lacks is removed. @@ -646,14 +688,25 @@ const configArgs = (plan) => ({ function runCli(dir, files, how = {}) { writeFixture(dir, files) const before = new Set(filesUnder(dir)) - const { args, out } = cliArgs(dir, files, how) + const { args, out: outBase } = cliArgs(dir, files, how) + // Each call has its own output directory: its retention lines are then its own, even when a + // daemon serves several checks of one test. + const out = `${outBase}-${++runSeq}` + args[args.indexOf(`--out`) + 1] = out + // The retention hook (ORACLE_GAPS_TEST_RETENTION): one line per batch, + // written beside the run's own temporary files, not in the package. + // One hook file per package: a daemon keeps the environment of the call that started it, so every call + // for this package writes to the same file; each line names its run (below). + const hook = hookOf(dir) const child = spawnSync(process.execPath, args, { cwd: repo, encoding: `utf8`, timeout: 300_000, maxBuffer: 64 * 1024 * 1024, - env: cleanEnv(), + env: { ...cleanEnv(), ORACLE_GAPS_TEST_RETENTION: hook }, }) + // Each line names its check's output directory, so this run keeps only its own (a daemon serves many calls). + const retention = retentionLinesOf(hook).filter((line) => line.run === out) const run = readRun( dir, out, @@ -667,7 +720,12 @@ function runCli(dir, files, how = {}) { .split(`\n`) .find((line) => line.startsWith(` replay: `)) ?.slice(` replay: `.length) - return { ...run, strayWrites: strays(dir, before), replay } + return { + ...run, + strayWrites: strays(dir, before, relative(dir, out)), + replay, + retention, + } } /** @@ -677,12 +735,13 @@ function runCli(dir, files, how = {}) { * directory, `.oracle-gaps-/` (its capture setup and Vitest configs, * which resolve the package's modules from there), is its own too. */ -const strays = (dir, before, running = false) => +const strays = (dir, before, own, running = false) => filesUnder(dir).filter( (file) => !before.has(file) && !(running && /^\.oracle-gaps-\d+\//.test(file)) && - !OWN_WRITES.some((home) => file === home || file.startsWith(`${home}/`)), + // The one path a run may write under: the --out the driver gave it (cliArgs names it). + !(file === own || file.startsWith(`${own}/`)), ) /** @@ -780,7 +839,12 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { : warm ? existsSync(join(pids, `warming`)) && !exited : readdirSync(pids).length >= 2 - const strayWhileRunning = strays(dir, before, true) + const strayWhileRunning = strays( + dir, + before, + relative(dir, join(dir, `out-stop`)), + true, + ) if (!abort) child.kill(signal) const ended = await Promise.race([exit, sleep(30_000).then(() => false)]) await sleep(1000) @@ -835,7 +899,7 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { /^\.oracle-gaps-\d+$/.test(name), ), // What else it left in the package. - left: strays(dir, before).filter( + left: strays(dir, before, relative(dir, join(dir, `out-stop`))).filter( (file) => !/^\.oracle-gaps-\d+\//.test(file), ), } @@ -894,7 +958,6 @@ function filesUnder(root, within = ``) { // (maintainer decision, 2026-11-11: the gap check's working files stay out // of the repository), so a package, and a workspace dependency in it, keeps // nothing of a run. -const OWN_WRITES = [`out`, `out-daemon`, `out-stop`, `out-only`] /** Runs the CLI several times at once on one state of the fixture. */ async function runClisAtOnce(dir, files, hows) { @@ -1310,7 +1373,16 @@ async function incremental(dir, plan, cli, copy, observe, states) { : await daemonChecks(step, files, how) if (!cold.has(key)) cold.set(key, runCli(dir, files, how)) const coldRun = cold.get(key) - for (const [i, run] of [daemon, ...others].entries()) + for (const [i, run] of [daemon, ...others].entries()) { + observe( + `retention:daemon`, + (run.retention ?? []).filter(leftBehind).length, + ) + observe(`retentionLines:daemon`, unlogged(run)) + observe( + `retention:cold`, + (coldRun.retention ?? []).filter(leftBehind).length, + ) observe(`incremental`, { step: i === 0 ? step : `${step} (a second check at once)`, cold: { @@ -1318,6 +1390,11 @@ async function incremental(dir, plan, cli, copy, observe, states) { refusal: coldRun.refusal, report: coldRun.report, notes: coldRun.stream.notes, + // An internal error's stack (its error.log, which readRun reads + // for a run that exits 3), for the violation's message. Only + // then: a completed run's stderr holds timings, which differ + // between two runs of one history. + stderr: coldRun.exit === 3 ? coldRun.stderr : undefined, }, daemon: { exit: run.exit, @@ -1332,6 +1409,7 @@ async function incremental(dir, plan, cli, copy, observe, states) { stderr: run.stderr?.split(`\n`)[0], }, }) + } if (twice && step === `start`) states.add(`two checks started the daemon at once`) if ( @@ -1600,15 +1678,23 @@ async function driver(plan, observe) { cli: plan.cliPath === `spaced` ? copy : here, ...configArgs(plan), } - const { replay, ...cli } = runCli(dir, files, how) + const { replay, retention, ...cli } = runCli(dir, files, how) // How many units were computed or reused says how the result was // produced, not what it is, so it is not observed. observe(`cli`, { ...cli, evidence: undefined }) + // Deterministic: how many batches left a retired copy's state behind (0), + // not the per-batch lines, whose batch splits vary with hang cuts. + observe(`retention`, retention.filter(leftBehind).length) + observe( + `retentionLines:cold`, + unlogged({ retention, evidence: cli.evidence }), + ) // A failing owner's replay, run as the author would run it (law 3). if (plan.fault === `brokenOwner`) observe(`replay`, { command: replay, ran: replayed(dir, replay) }) // Reach comes from what the CLI reported, mapped onto the plan. const states = new Set() + if (retention.length > 1) states.add(`a check that ran more than one batch`) if (cli.exit === 0 && !plan.consoleOutput) states.add(`a completed run whose tests wrote nothing`) // Law 14: a partial run of one owned function's mutants, whose baseline @@ -1625,6 +1711,8 @@ async function driver(plan, observe) { only: ids, outName: `out-only`, }) + observe(`retention:partial`, partial.retention.filter(leftBehind).length) + observe(`retentionLines:partial`, unlogged(partial)) observe(`partial`, { function: chosen.function, source: chosen.source, @@ -2049,7 +2137,13 @@ async function driver(plan, observe) { const moved = { ...files, [SOURCE]: edited(plan) } // The evidence counts are not observed, as for `cli`: they depend on // timing (oracles-followups.md G1). Verdicts and keys are. - observe(`stability`, { ...runCli(dir, moved, how), evidence: undefined }) + const { retention: movedRetention, ...movedRun } = runCli(dir, moved, how) + observe(`stability`, { ...movedRun, evidence: undefined }) + observe(`retention:stability`, movedRetention.filter(leftBehind).length) + observe( + `retentionLines:stability`, + unlogged({ retention: movedRetention, evidence: movedRun.evidence }), + ) states.add(`stability rerun`) } if (plan.steps.length > 0) @@ -2090,7 +2184,7 @@ function incrementalCheck(observations) { ) return violation( `incremental`, - `after ${step}, the daemon's check exited ${daemon.exit} (${daemon.refusal ?? daemon.stderr ?? `no refusal`}) and the cold run exited ${cold.exit} (${cold.refusal ?? `no refusal`})`, + `after ${step}, the daemon's check exited ${daemon.exit} (${daemon.refusal ?? daemon.stderr ?? `no refusal`}) and the cold run exited ${cold.exit} (${cold.refusal ?? `no refusal`})${cold.exit === 3 && cold.stderr ? `; its error log: ${cold.stderr}` : ``}`, `incremental equals cold`, { actual: daemon, expected: cold }, ) @@ -2170,6 +2264,32 @@ function check(plan, observations, expected) { ) const stream = streamBreach(`cli`, cli, expected, plan) if (stream) return stream + // Bounded retention (the register's C entry): once a batch's copies are + // retired, Vitest's filesMap, pathsSet, and idMap and Vite's module graph + // name none of them, so each count the hook reports is 0. + const unlogged_ = observations + .filter((o) => o.checkpoint.startsWith(`retentionLines`)) + .reduce((sum, o) => sum + o.value, 0) + if (unlogged_ > 0) + return violation( + `retention`, + `${unlogged_} run(s) wrote a retention line count different from their batches (${observations + .filter((o) => o.checkpoint.startsWith(`retentionLines`) && o.value > 0) + .map((o) => o.checkpoint) + .join(`, `)})`, + `bounded retention`, + { actual: unlogged_, expected: 0 }, + ) + const held = observations + .filter((o) => o.checkpoint.startsWith(`retention`)) + .reduce((sum, o) => sum + o.value, 0) + if (held > 0) + return violation( + `retention`, + `${held} batch(es) left a retired copy's Vitest or Vite state behind`, + `bounded retention`, + { actual: held, expected: 0 }, + ) // Through the daemon, each check streams the same kinds of line, and a // completed one ends with its done line (law 15). The state of each step // differs, so its verdict and counts are judged by law 9's comparison. @@ -3350,6 +3470,8 @@ oracleProperty({ ...staticStates, `extra work in a generated function`, `killed by a work bound in a generated function`, + // A check whose runs have more than one batch: the retention law's multi-batch reach. + `a check that ran more than one batch`, ], runs: 2, fixedSeeds: [20261105],