Preserve listing coverage before comparing growth and forecasting history

src/api/openapi.ts

@@ -1487,7 +1487,7 @@ export const ENDPOINTS: EndpointDescriptor[] = [
     responses: {
       '200': jsonResponse('Diff', ref('CompareResult')),
       '400': errorResponse('Scans cover different roots'),
-      '409': errorResponse('BOUNDARY_CHANGED — the two scans stopped at different drives mounted inside the folder (one under "Stay on this drive" left a drive out that the other did not, or one went into a drive the other did not find), so a diff would show a drive as files added or removed'),
+      '409': errorResponse('BOUNDARY_CHANGED — the scans counted different mounted drives; COVERAGE_INCOMPARABLE — either scan reported incomplete or unknown listing coverage, so the difference cannot prove file growth or shrinkage'),
     },
   },
   {
@@ -1876,7 +1876,7 @@ export const ENDPOINTS: EndpointDescriptor[] = [
     tag: 'history',
     destructive: false,
     parameters: [queryParam('path', 'Root to list snapshots for'), queryParam('all', "'true' for every snapshot, slim", bool())],
-    responses: { '200': jsonResponse('History', opaque('{roots[]} | {rootPath, snapshots[]} | {snapshots[]}. A snapshot carries boundary — { rule: "stay", notEntered[] } or { rule: "cross", entered[] } — only when a drive was mounted inside its folder (Track U); history never compares across two different boundaries')) },
+    responses: { '200': jsonResponse('History', opaque('{roots[]} | {rootPath, snapshots[]} | {snapshots[]}. Snapshot coverage is complete, partial or unknown for reported listing errors; absent on legacy history means unknown. A snapshot carries boundary — { rule: "stay", notEntered[] } or { rule: "cross", entered[] } — only when a drive was mounted inside its folder (Track U); history never compares across different boundaries or incomplete/unknown coverage')) },
   },
   {
     method: 'get',
@@ -1890,7 +1890,7 @@ export const ENDPOINTS: EndpointDescriptor[] = [
     ],
     responses: {
       '200': jsonResponse(
-        'Historical layout. prevTakenAt and every prevSize are absent (null) when no earlier snapshot exists, or when the one before stopped at different drives mounted inside the folder (Track U, U-24)',
+        'Historical layout. prevTakenAt and every prevSize are absent (null) when no earlier snapshot exists, the scans counted different mounted drives, or either snapshot has incomplete or unknown listing coverage',
         opaque('snapshot, prevTakenAt, root, scanRootPath, maxDepth, minSize, nodes[], tree'),
       ),
       '404': errorResponse('No snapshot trees recorded'),
@@ -1906,7 +1906,7 @@ export const ENDPOINTS: EndpointDescriptor[] = [
     responses: {
       '200': jsonResponse('Diff', opaque('SnapshotDiff: a, b, rootPath, totalDelta, entries[]')),
       '404': errorResponse('Unknown snapshot id'),
-      '409': errorResponse('BOUNDARY_CHANGED — the two snapshots stopped at different drives mounted inside the folder, so a diff would show a drive as files added or removed'),
+      '409': errorResponse('BOUNDARY_CHANGED — the snapshots counted different mounted drives; COVERAGE_INCOMPARABLE — either snapshot has incomplete or unknown listing coverage (including history recorded before coverage was persisted)'),
     },
   },
   {

src/models/types.ts

@@ -663,6 +663,8 @@ export interface Snapshot {
   topEntries: SnapshotTopEntry[];
   /** True when a deeper tree was stored for the time-slider treemap. */
   hasTree?: boolean;
+  /** Reported listing coverage at scan time. Absent on old history means unknown. */
+  coverage?: 'complete' | 'partial' | 'unknown';
   /**
    * Present only when a drive was mounted inside the scanned folder (Track U,
    * U-24): under `stay` the drives the scan did not enter, under `cross` the

src/services/boundaryReport.ts

@@ -12,7 +12,7 @@ import {
 } from '../utils/mountBoundaries';
 import { storeOf } from './scanStore';
 import { storageModeOf } from './storageMode';
-import { boundaryKeyOf, scanBoundaryKey, snapshotBoundaryOf } from './snapshots';
+import { assertComparableCoverage, boundaryKeyOf, scanBoundaryKey, scanComparisonEvidence, snapshotBoundaryOf } from './snapshots';
 import { AppError } from '../middleware/errorHandler';
 import { rootLivesAt } from '../utils/pathSanitizer';
 
@@ -117,8 +117,8 @@ export function boundaryReport(scan: ScanResult): BoundaryReport {
  * alike, with the reason. `scanBoundaryKey` is the same key history uses.
  */
 export function assertSameBoundary(scanA: ScanResult, scanB: ScanResult): void {
-  if (scanBoundaryKey(scanA) === scanBoundaryKey(scanB)) return;
-  throw boundaryChanged('scans', scanA.rootPath, snapshotBoundaryOf(scanA), snapshotBoundaryOf(scanB));
+  if (scanBoundaryKey(scanA) !== scanBoundaryKey(scanB)) throw boundaryChanged('scans', scanA.rootPath, snapshotBoundaryOf(scanA), snapshotBoundaryOf(scanB));
+  assertComparableCoverage(scanComparisonEvidence(scanA), scanComparisonEvidence(scanB));
 }
 
 /**
@@ -127,8 +127,8 @@ export function assertSameBoundary(scanA: ScanResult, scanB: ScanResult): void {
  * "freed").
  */
 export function assertSameSnapshotBoundary(a: Snapshot, b: Snapshot): void {
-  if (boundaryKeyOf(a.boundary) === boundaryKeyOf(b.boundary)) return;
-  throw boundaryChanged('snapshots', a.rootPath, a.boundary, b.boundary);
+  if (boundaryKeyOf(a.boundary) !== boundaryKeyOf(b.boundary)) throw boundaryChanged('snapshots', a.rootPath, a.boundary, b.boundary);
+  assertComparableCoverage(a, b);
 }
 
 function boundaryChanged(what: string, rootPath: string, a: Snapshot['boundary'], b: Snapshot['boundary']): AppError {

src/services/budgetGauges.ts

@@ -34,6 +34,7 @@ export interface BudgetPoint {
   t: number;
   v: number;
   boundary?: SnapshotBoundary;
+  coverage?: Snapshot['coverage'];
 }
 
 export interface BudgetHistorySeries {
@@ -119,8 +120,8 @@ function boundaryWithin(boundary: SnapshotBoundary | undefined, folder: string):
   return entered.length > 0 ? { rule: 'cross', entered } : undefined;
 }
 
-function pointOf(t: number, v: number, boundary: SnapshotBoundary | undefined): BudgetPoint {
-  return boundary ? { t, v, boundary } : { t, v };
+function pointOf(t: number, v: number, boundary: SnapshotBoundary | undefined, coverage: Snapshot['coverage']): BudgetPoint {
+  return { t, v, coverage: coverage ?? 'unknown', ...(boundary ? { boundary } : {}) };
 }
 
 /**
@@ -132,7 +133,7 @@ function pointOf(t: number, v: number, boundary: SnapshotBoundary | undefined):
 export async function budgetHistorySeries(budgetPath: string): Promise<BudgetHistorySeries> {
   const own = await listSnapshots(budgetPath);
   if (own.length > 0) {
-    return { points: own.map((s) => pointOf(s.takenAt, s.totalSize, boundaryWithin(s.boundary, budgetPath))), source: 'own-scans' };
+    return { points: own.map((s) => pointOf(s.takenAt, s.totalSize, boundaryWithin(s.boundary, budgetPath), s.coverage)), source: 'own-scans' };
   }
 
   // No direct history — look for scanned roots the budget folder sits under
@@ -153,7 +154,7 @@ export async function budgetHistorySeries(budgetPath: string): Promise<BudgetHis
       const tree = trees[s.id];
       if (!tree) continue;
       const node = descend(tree, segments);
-      if (node) points.push(pointOf(s.takenAt, node.s, boundaryWithin(s.boundary, budgetPath)));
+      if (node) points.push(pointOf(s.takenAt, node.s, boundaryWithin(s.boundary, budgetPath), s.coverage));
     }
   }
   if (points.length === 0) return { points: [], source: 'none' };
@@ -199,6 +200,7 @@ export function computeBudgetProjection(
     fileCount: 0,
     dirCount: 0,
     topEntries: [],
+    coverage: p.coverage,
     ...(p.boundary ? { boundary: p.boundary } : {}),
   }));
   const f = computeForecast(snaps, budget.maxBytes - currentBytes, now);

src/services/forecast.ts

@@ -108,10 +108,13 @@ export function computeForecast(
   };
 
   if (count < MIN_SNAPSHOTS || spanDays < MIN_SPAN_DAYS) {
+    const exclusion = snaps.some((s) => s.coverage !== 'complete')
+      ? 'had incomplete or unknown listing coverage, or stopped at different drives mounted inside this folder'
+      : 'stopped at different drives mounted inside this folder';
     return {
       ...base,
       status: 'insufficient',
-      reason: `Not enough history yet — ${count} scan${count === 1 ? '' : 's'} over ${spanDays < 0.05 ? 'less than an hour' : `${round2(spanDays)} day${spanDays === 1 ? '' : 's'}`}. Forecasting needs at least ${MIN_SNAPSHOTS} scans spanning ${MIN_SPAN_DAYS}+ days.${earlier > 0 ? ` ${earlier} earlier scan${earlier === 1 ? '' : 's'} stopped at different drives mounted inside this folder, so ${earlier === 1 ? 'it is' : 'they are'} not compared with these.` : ''}`,
+      reason: `Not enough history yet — ${count} scan${count === 1 ? '' : 's'} over ${spanDays < 0.05 ? 'less than an hour' : `${round2(spanDays)} day${spanDays === 1 ? '' : 's'}`}. Forecasting needs at least ${MIN_SNAPSHOTS} fully listed scans spanning ${MIN_SPAN_DAYS}+ days.${earlier > 0 ? ` ${earlier} earlier scan${earlier === 1 ? '' : 's'} ${exclusion}, so ${earlier === 1 ? 'it is' : 'they are'} not compared with these.` : ''}`,
     };
   }
 

src/services/journal.ts

@@ -6,7 +6,7 @@ import { appDataDir } from './storage';
 import { isEphemeral } from './portableMode';
 import { getAppAttribution } from './appAttribution';
 import { readAudit } from './audit';
-import { boundaryKeyOf, buildSnapshotTree, getSnapshotTreeAt, scanBoundaryKey } from './snapshots';
+import { buildSnapshotTree, getSnapshotTreeAt, scanComparisonEvidence, snapshotsComparable } from './snapshots';
 import { storeOf } from './scanStore';
 import { isInside } from '../utils/pathSanitizer';
 import { separatorOf } from '../utils/pathSeparator';
@@ -302,7 +302,7 @@ export async function recordScanJournal(scan: ScanResult, prev: Snapshot): Promi
   // The two scans stopped at different drives mounted inside the folder
   // (Track U, U-24): the difference is a drive, not something a process did,
   // and the journal names no culprit for it. The next scan is the baseline.
-  if (scanBoundaryKey(scan) !== boundaryKeyOf(prev.boundary)) return 0;
+  if (!snapshotsComparable(prev, scanComparisonEvidence(scan))) return 0;
   const store = storeOf(scan);
   const curr = buildSnapshotTree(store);
 

src/services/scheduler.ts

@@ -3,7 +3,7 @@ import { GrowthNotification, ScanResult, ScheduleConfig, Snapshot } from '../mod
 import { startScan, getScan, scanRootPathOf } from './diskScanner';
 import { getSettings, patchSchedule } from './settings';
 import { runDuePolicies } from './autopilot';
-import { boundaryKeyOf, listSnapshots, scanBoundaryKey } from './snapshots';
+import { listSnapshots, scanComparisonEvidence, snapshotsComparable } from './snapshots';
 import { getForecast } from './forecast';
 import { recordScanJournal } from './journal';
 import { sanitizePath } from '../utils/pathSanitizer';
@@ -177,8 +177,7 @@ export function growthAlertFor(sched: ScheduleConfig, target: string, prev: Snap
   if (!done.store) return null;
   // Another folder's snapshot is no previous size of this one: a link the
   // scheduler resolved and the scan then followed elsewhere (QA D78).
-  if (prev.rootPath !== done.rootPath) return null;
-  if (scanBoundaryKey(done) !== boundaryKeyOf(prev.boundary)) return null;
+  if (!snapshotsComparable(prev, scanComparisonEvidence(done))) return null;
   const newSize = done.store.size(done.store.rootId);
   const delta = newSize - prev.totalSize;
   const pct = prev.totalSize > 0 ? (delta / prev.totalSize) * 100 : 0;

src/services/snapshots.ts

@@ -7,6 +7,7 @@ import { isEphemeral } from './portableMode';
 import { ScanStore, TreeSource, asStore, storeOf } from './scanStore';
 import { mountOutcome } from '../utils/mountBoundaries';
 import { separatorOf } from '../utils/pathSeparator';
+import { AppError } from '../middleware/errorHandler';
 
 /**
  * Snapshots — lightweight scan history persisted to snapshots.json in the
@@ -267,7 +268,7 @@ export async function getSnapshotTreeAt(rootPath: string, at: number): Promise<{
   if (!snapshot) return null;
   const before = withTrees.filter((s) => s.takenAt < snapshot.takenAt);
   const last = before.length ? before[before.length - 1] : null;
-  const prevSnap = last && boundaryKeyOf(last.boundary) === boundaryKeyOf(snapshot.boundary) ? last : null;
+  const prevSnap = last && snapshotsComparable(last, snapshot) ? last : null;
   return {
     snapshot,
     tree: trees[snapshot.id],
@@ -315,13 +316,41 @@ export function scanBoundaryKey(scan: ScanResult): string {
   return boundaryKeyOf(snapshotBoundaryOf(scan));
 }
 
+/** A settled scan can still report failed listings. This cannot detect errors an engine never reports. */
+export function scanCoverage(scan: ScanResult): NonNullable<Snapshot['coverage']> {
+  if ([scan.deniedDirs, scan.unreadableDirs, scan.deniedEntries, scan.unreadableEntries, scan.vanishedDirs].some((n) => (n ?? 0) > 0)) return 'partial';
+  if (scan.status !== 'complete') return 'unknown';
+  return 'complete';
+}
+
+type ComparisonEvidence = Pick<Snapshot, 'rootPath' | 'coverage' | 'boundary'>;
+
+/** Current scan evidence, in the same shape persisted history uses. */
+export function scanComparisonEvidence(scan: ScanResult): ComparisonEvidence {
+  return { rootPath: scan.rootPath, coverage: scanCoverage(scan), boundary: snapshotBoundaryOf(scan) };
+}
+
+/** Unknown or partial listings cannot prove files appeared, disappeared, grew or shrank. */
+export function snapshotsComparable(a: ComparisonEvidence, b: ComparisonEvidence): boolean {
+  return a.coverage === 'complete' && b.coverage === 'complete' && a.rootPath === b.rootPath &&
+    boundaryKeyOf(a.boundary) === boundaryKeyOf(b.boundary);
+}
+
+export function assertComparableCoverage(a: ComparisonEvidence, b: ComparisonEvidence): void {
+  if (a.coverage === 'complete' && b.coverage === 'complete') return;
+  throw new AppError(409, 'COVERAGE_INCOMPARABLE',
+    `These measurements have ${a.coverage ?? 'unknown'} and ${b.coverage ?? 'unknown'} listing coverage. ` +
+    'Incomplete or unknown coverage cannot prove file growth or shrinkage; compare two fully listed scans.');
+}
+
 /** The snapshots since the boundary last changed: the latest one's run of equal keys, oldest first. */
 export function currentBaseline(snaps: readonly Snapshot[]): Snapshot[] {
   const sorted = [...snaps].sort((a, b) => a.takenAt - b.takenAt);
   if (sorted.length === 0) return sorted;
-  const key = boundaryKeyOf(sorted[sorted.length - 1].boundary);
+  const latest = sorted[sorted.length - 1];
+  if (latest.coverage !== 'complete') return [];
   let start = sorted.length - 1;
-  while (start > 0 && boundaryKeyOf(sorted[start - 1].boundary) === key) start--;
+  while (start > 0 && snapshotsComparable(sorted[start - 1], latest)) start--;
   return sorted.slice(start);
 }
 
@@ -360,6 +389,7 @@ async function writeSnapshot(scan: ScanResult): Promise<Snapshot | null> {
     dirCount: scan.dirCount,
     topEntries,
     hasTree: true,
+    coverage: scanCoverage(scan),
   };
   const boundary = snapshotBoundaryOf(scan);
   if (boundary) snapshot.boundary = boundary;
@@ -446,6 +476,7 @@ export async function getSnapshot(id: string): Promise<Snapshot | undefined> {
 
 /** Size deltas between two snapshots (top-level entries matched by path). */
 export function diffSnapshots(a: Snapshot, b: Snapshot): SnapshotDiff {
+  assertComparableCoverage(a, b);
   const byPath = new Map<string, SnapshotDeltaEntry>();
   for (const e of a.topEntries) {
     byPath.set(e.path, { name: e.name, path: e.path, type: e.type, sizeA: e.size, sizeB: null, delta: -e.size });

tests/budgetGauges.test.ts

@@ -24,12 +24,12 @@ const GB = 1024 ** 3;
 const NOW = 1_800_000_000_000; // fixed "now" so every projection is deterministic
 
 /** A series point `daysAgo` days back from NOW at `v` bytes. */
-function pt(daysAgo: number, v: number): { t: number; v: number } {
-  return { t: NOW - daysAgo * DAY, v };
+function pt(daysAgo: number, v: number): { t: number; v: number; coverage: 'complete' } {
+  return { t: NOW - daysAgo * DAY, v, coverage: 'complete' };
 }
 
 function snap(rootPath: string, id: string, daysAgo: number, totalSize: number): Snapshot {
-  return { id, rootPath, takenAt: NOW - daysAgo * DAY, totalSize, fileCount: 1, dirCount: 1, topEntries: [] };
+  return { id, rootPath, takenAt: NOW - daysAgo * DAY, totalSize, fileCount: 1, dirCount: 1, topEntries: [], coverage: 'complete' };
 }
 
 /** Overwrite the isolated snapshot store with exactly these snapshots. */

tests/forecast.test.ts

@@ -21,6 +21,7 @@ function snap(daysAgo: number, totalSize: number, tops: { name: string; size: nu
     fileCount: 1,
     dirCount: 1,
     topEntries: tops.map((t) => ({ name: t.name, path: `/r/${t.name}`, size: t.size, type: 'dir' as const })),
+    coverage: 'complete',
   };
 }
 

tests/journal.test.ts

@@ -399,6 +399,7 @@ test('recordScanJournal turns a scan-vs-snapshot delta into an attributed entry'
   // journal must write is a removal it cannot attribute to any app.
   const prev: Snapshot = {
     id: 'prev-snapshot',
+    coverage: 'complete',
     rootPath: fixture,
     takenAt: Date.now() - 60_000,
     totalSize: currentTotal + 300 * MB,
@@ -432,6 +433,7 @@ test('a scan that changed nothing significant writes nothing', async () => {
   const size = store.size(store.rootId);
   const prev: Snapshot = {
     id: 'prev-still', rootPath: fixture, takenAt: Date.now() - 60_000,
+    coverage: 'complete',
     totalSize: size, fileCount: 1, dirCount: 1,
     topEntries: [{ name: 'small.txt', path: path.join(fixture, 'small.txt'), size, type: 'file' }],
   };

tests/knownSizeOfCost.test.ts

@@ -11,6 +11,7 @@ process.env.TREEMAP_NO_GDU = '1';
 import { startScan } from '../src/services/diskScanner';
 import { updateSettings } from '../src/services/settings';
 import { knownSizeOf } from '../src/services/policy';
+import { canonicalDirOf } from '../src/utils/pathSanitizer';
 import type { ScanResult } from '../src/models/types';
 
 /**
@@ -214,6 +215,9 @@ test('a name found by its fold counts only on the volume\'s own word that both n
   const file = path.join(APART, 'y1'); // made after the scan, beside Ｙ1, whose name folds the same
   const sibling = path.join(APART, 'Ｙ1');
   fs.writeFileSync(file, Buffer.alloc(13, 1));
+  // The policy asks at the native parent spelling (Windows temp roots can use an 8.3 alias).
+  const canonicalFile = canonicalDirOf(file);
+  const canonicalSibling = canonicalDirOf(sibling);
   /**
    * What the stand-in answers when the identity of `file`, then of `sibling`, is asked (bigint, links
    * not followed). 'gone' is what the volume says of a name nothing is at any more — a scanned sibling
@@ -234,7 +238,7 @@ test('a name found by its fold counts only on the volume\'s own word that both n
   try {
     for (const [what, [first, second]] of Object.entries(answers)) {
       host.lstatSync = (p, options) => {
-        const answer = p === file ? first : p === sibling ? second : null;
+        const answer = p === canonicalFile ? first : p === canonicalSibling ? second : null;
         if (answer === null || options?.bigint !== true) return real(p, options);
         if (answer === 'throws') throw Object.assign(new Error(`EIO: i/o error, lstat '${String(p)}'`), { code: 'EIO' });
         if (answer === 'gone') {