fix(health): stop reporting time series as supersede candidates, and hot topics as sediment gaps

.github/workflows/ci.yml

@@ -39,6 +39,9 @@ jobs:
       - name: Unit tests (meta-gate)
         run: node meta-gate.test.mjs
 
+      - name: Unit tests (series detection — keeps timelines out of the supersede band)
+        run: node series-detection.test.mjs
+
       - name: Integration test (meta-gate write-gate)
         env:
           TOKENMEM_DB_PATH: ${{ runner.temp }}/mneme-ci-meta-gate.db

memory-health.mjs

@@ -80,8 +80,21 @@ export const DEFAULTS = Object.freeze({
   // recall_log analytics window in days.
   recallLogDays: 7,
   // Minimum frequency at which a repeat query surfaces as a "you keep looking
-  // this up — maybe store the answer" candidate.
+  // this up — maybe store the answer" candidate. Frequency alone is not
+  // evidence of a gap: a query can repeat because recall answers it well and
+  // the topic is hot. Only repeats whose recall actually comes back thin are
+  // reported — see repeatQueryMaxAvgHits.
   repeatQueryMin: 3,
+  // A repeating query is a sediment gap only if it typically returns less than
+  // this many rows. Measured 2026-09-01: every freq>=3 query in a 7d window
+  // averaged 5-20 hits, i.e. the un-filtered signal was 100% false positive.
+  repeatQueryMaxAvgHits: 1,
+  // Two rows written inside this window are treated as one working session: a
+  // log, a request and its reply, a decision and its refinement — not two
+  // drafts of one fact. Calibrated on 10 hand-classified pairs (2026-09-01);
+  // the closest true rewrite in that set sat 5 days apart, the widest
+  // same-session false positive 6.8h.
+  seriesSameSessionHours: 8,
   // A recall_log source must have written at least this many rows historically
   // before its silence is treated as a stall. Below it, "no rows this week" is
   // just a quiet occasional caller — one-off CLI probes and debug labels live
@@ -252,6 +265,7 @@ export function detectIntegrity(db, opts = {}) {
 export function detectBlindspot(db, opts = {}) {
   const days = opts.recallLogDays ?? DEFAULTS.recallLogDays
   const minFreq = opts.repeatQueryMin ?? DEFAULTS.repeatQueryMin
+  const maxAvgHits = opts.repeatQueryMaxAvgHits ?? DEFAULTS.repeatQueryMaxAvgHits
   const since = Date.now() - days * 86400_000
   // Not every mneme deployment writes to recall_log — it's optional
   // instrumentation. Skip cleanly if the table is missing.
@@ -286,11 +300,19 @@ export function detectBlindspot(db, opts = {}) {
   const bySource = db.prepare(`SELECT source, COUNT(*) c FROM recall_log WHERE ts > ? GROUP BY source ORDER BY c DESC`).all(since)
   const strictZero = db.prepare(`SELECT COUNT(*) c FROM recall_log WHERE ts > ? AND hit_count = 0`).get(since).c
   const finalZero = db.prepare(`SELECT COUNT(*) c FROM recall_log WHERE ts > ? AND final_hit_count = 0`).get(since).c
-  const repeats = db.prepare(`
+  // A repeating query is only evidence of a missing memory if recall keeps
+  // coming back thin. Frequency on its own measures how hot a topic is, not
+  // whether it is answered — and hot-and-answered is the normal case: hooks
+  // fire recall on every prompt containing the word. Reporting those as
+  // "sediment-worthy" sends you off to write a card that already exists.
+  const repeatsRaw = db.prepare(`
     SELECT query, COUNT(*) freq, SUM(hit_count) hits
     FROM recall_log WHERE ts > ? AND query IS NOT NULL AND length(query) > 0
     GROUP BY query HAVING freq >= ? ORDER BY freq DESC LIMIT 30
   `).all(since, minFreq).filter(r => !isNoiseQuery(r.query))
+  const avgHits = (r) => (r.freq > 0 ? (r.hits ?? 0) / r.freq : 0)
+  const repeats = repeatsRaw.filter(r => avgHits(r) < maxAvgHits)
+  const repeatsAnswered = repeatsRaw.length - repeats.length
   const zeroQueries = db.prepare(`
     SELECT DISTINCT query FROM recall_log
     WHERE ts > ? AND (hit_count = 0 OR final_hit_count = 0) AND query IS NOT NULL AND length(query) > 0
@@ -299,7 +321,11 @@ export function detectBlindspot(db, opts = {}) {
   return {
     available: true, window_days: days, total_calls: total,
     by_source: bySource, strict_zero: strictZero, final_zero: finalZero,
-    repeat_queries: repeats.slice(0, 15).map(r => ({ q: clip(r.query, 60), freq: r.freq, hits: r.hits })),
+    repeat_queries: repeats.slice(0, 15).map(r => ({
+      q: clip(r.query, 60), freq: r.freq, hits: r.hits,
+      avg_hits: +(avgHits(r)).toFixed(1),
+    })),
+    repeat_queries_answered: repeatsAnswered,
     zero_hit_real_queries: zeroQueries,
     stalled_sources: detectStalledSources(db, opts),
   }
@@ -441,14 +467,10 @@ export function detectNearDup(db, opts = {}) {
           // dropped — an old plain row next to an anchor is still a valid
           // supersede target, and only the reviewer can tell which side is which.
           if (band === 'supersede') {
-            // Two concrete_trace rows that look alike are almost never an
-            // iteration of one fact — they are two runs of the same routine
-            // (nightly metric snapshots, repeated ops logs). By our own level
-            // semantics those are one-off traces that decay is supposed to bury,
-            // so superseding them would destroy a legitimate time series.
-            // Flagged rather than dropped: the reviewer still sees the pair.
-            const likelySeries = items[i].memory_level === 'concrete_trace'
-              && items[j].memory_level === 'concrete_trace'
+            // Time series masquerade as rewrites at this cosine band. See
+            // isLikelySeries for the three signals and what calibrated them.
+            // Flagged rather than dropped: the reviewer still sees the count.
+            const likelySeries = isLikelySeries(items[i], items[j])
             entry.detail = {
               a: sideDetail(items[i], t0), b: sideDetail(items[j], t0),
               newer_rowid: (items[i].created_at ?? 0) >= (items[j].created_at ?? 0) ? items[i].rowid : items[j].rowid,
@@ -621,6 +643,61 @@ function defaultDbPath() {
           : resolve(__dirname, 'tokenmem.db'))
 }
 
+// ============================================================
+// Series detection — keeps timelines out of the supersede band
+// ============================================================
+
+// Matches a temporal or ordinal marker anywhere in a summary: ISO dates,
+// clock times, slash dates, CJK dates. Rows whose summaries lead with one of
+// these are almost always entries in a series (nightly snapshots, timestamped
+// log lines) rather than drafts of a single fact.
+const TEMPORAL_MARKER = /(?:20\d{2}-\d{1,2}-\d{1,2}|\d{1,2}:\d{2}|\d{1,2}\/\d{1,2}|\d{1,2}月\d{1,2}日)/g
+
+function temporalMarkers(text) {
+  if (typeof text !== 'string' || !text) return null
+  const found = text.match(TEMPORAL_MARKER)
+  return found && found.length ? new Set(found) : null
+}
+
+/**
+ * True when a near-duplicate pair is a point in a time series rather than a
+ * stale rewrite of one fact.
+ *
+ * The supersede band exists to catch "an already-replaced version stays active
+ * and can be recalled as if current". A time series has no replaced version —
+ * every entry is still true about its own moment, and superseding one destroys
+ * the sequence. Reported separately instead of dropped, so the count stays
+ * visible.
+ *
+ * Three signals, any one of which is sufficient:
+ *   1. both rows are concrete_trace — two runs of one routine, which decay is
+ *      already supposed to bury
+ *   2. written inside one working session — a log or a request/reply pair
+ *   3. both summaries carry a temporal marker and the markers differ — a dated
+ *      series whose entries can sit arbitrarily far apart
+ *
+ * Signal 3 requires the markers to DIFFER: two rows citing the same date are
+ * one event described twice, which is exactly the rewrite we want to surface.
+ */
+export function isLikelySeries(a, b, opts = {}) {
+  if (a?.memory_level === 'concrete_trace' && b?.memory_level === 'concrete_trace') return true
+
+  // Missing timestamps must not read as a zero gap — that would swallow every
+  // pair with an unset created_at into "same session".
+  const ta = a?.created_at, tb = b?.created_at
+  if (Number.isFinite(ta) && Number.isFinite(tb)) {
+    const windowMs = (opts.sameSessionHours ?? DEFAULTS.seriesSameSessionHours) * 3600_000
+    if (Math.abs(ta - tb) < windowMs) return true
+  }
+
+  const ma = temporalMarkers(a?.summary), mb = temporalMarkers(b?.summary)
+  if (ma && mb) {
+    for (const m of ma) if (!mb.has(m)) return true
+    for (const m of mb) if (!ma.has(m)) return true
+  }
+  return false
+}
+
 // ============================================================
 // Text render — human-readable report from the JSON return
 // ============================================================
@@ -674,7 +751,8 @@ export function renderTextReport(report, opts = {}) {
   L.push(`  Not a duplicate scan — these are pairs whose wording drifted, which is where`)
   L.push(`  an already-replaced version stays active and can be recalled as if current.`)
   if (scSeries.length) {
-    L.push(`  (${scSeries.length} concrete_trace pairs hidden — repeated runs of one routine, not iterations; decay buries those)`)
+    L.push(`  (${scSeries.length} time-series pairs hidden — same-session logs, dated snapshots, repeated routine runs.`)
+    L.push(`   Every entry is still true about its own moment, so superseding one destroys the sequence.)`)
   }
   if (!sc.length) L.push(`  (none)`)
   for (const c of sc.slice(0, 40)) {
@@ -728,8 +806,11 @@ export function renderTextReport(report, opts = {}) {
       L.push(`  real zero-hit queries (noise filtered): ${bs.zero_hit_real_queries.map(q => `"${clip(q, 30)}"`).join(', ')}`)
     }
     if (bs.repeat_queries.length) {
-      L.push(`  repeat queries (freq>=${DEFAULTS.repeatQueryMin}, sediment-worthy):`)
-      for (const r of bs.repeat_queries) L.push(`    freq=${r.freq} "${r.q}"`)
+      L.push(`  repeat queries that recall answers thinly (freq>=${DEFAULTS.repeatQueryMin}, avg hits<${DEFAULTS.repeatQueryMaxAvgHits}) — these are the sediment gaps:`)
+      for (const r of bs.repeat_queries) L.push(`    freq=${r.freq} avg_hits=${r.avg_hits} "${r.q}"`)
+    }
+    if (bs.repeat_queries_answered) {
+      L.push(`  (${bs.repeat_queries_answered} more repeat queries hidden — recall answers them, so they are hot topics, not gaps)`)
     }
   }
 

series-detection.test.mjs

@@ -0,0 +1,106 @@
+// Unit test for isLikelySeries — the (a2) supersede-band filter.
+//
+// Fixtures are the real shapes measured out of a 9301-row library on 2026-09-01,
+// where (a2) reported 125 "supersede candidates" and a hand review found that
+// nearly all of them were time series, not stale rewrites. Each case below
+// records what that pair actually was, so a future change to the predicate has
+// to argue with the data rather than with the rule.
+//
+// The one TRUE positive (#49/#272) is the shape the band exists to find: one
+// fact restated days later, where the older wording stays recallable as if
+// current. If a change makes that case fall out, the filter has eaten its
+// own purpose.
+
+import { test } from 'node:test'
+import assert from 'node:assert/strict'
+import { isLikelySeries } from './memory-health.mjs'
+
+const H = 3600_000
+const D = 24 * H
+const t = (hoursAgo) => 1756_000_000_000 - hoursAgo * H
+
+const row = (level, hoursAgo, summary) => ({
+  memory_level: level, created_at: t(hoursAgo), summary,
+})
+
+// ── series: must be filtered out of the review list ────────────────────────
+
+test('same-session log entries minutes apart are a timeline, not a rewrite', () => {
+  // #9782/#9787 — one person narrating a decision then refining it 3 min later
+  const a = row('semi_abstract', 100, '小毛 20:22 决策路径: 直接开游戏录真实运行 (不装 Godot)')
+  const b = row('semi_abstract', 99.9, '小毛 20:25 细化: 优先 hyperbeam native 60fps MP4')
+  assert.equal(isLikelySeries(a, b), true)
+})
+
+test('a request and its reply written in the same minute are not versions of each other', () => {
+  // #2640/#2641 — desktop asks, group session answers
+  const a = row('semi_abstract', 200, '桌面→群聊 千夏 V4-B promote 进展 query · 5/17 03:30')
+  const b = row('semi_abstract', 200, '群聊→桌面 V4-B promote query 回包: 5 答 evidence-first')
+  assert.equal(isLikelySeries(a, b), true)
+})
+
+test('dated snapshots a week apart are a series even though the gap is wide', () => {
+  // #6317/#5960 — nightly autosleep metrics; the leading date is the whole point
+  const a = row('semi_abstract', 0, '2026-07-08 autosleep 指标快照 meta70.9% imp≥7=91%')
+  const b = row('concrete_trace', 7 * 24, '2026-07-01 autosleep 指标快照 meta74.6% imp≥7=91.1%')
+  assert.equal(isLikelySeries(a, b), true)
+})
+
+test('consecutive daily snapshots are a series at 23.8h apart', () => {
+  // #10756/#10676 — just outside the same-session window, caught by the date literal
+  const a = row('semi_abstract', 0, '2026-08-29 autosleep 快照：active9215 meta59.5%(持降)')
+  const b = row('semi_abstract', 23.8, '2026-08-28 autosleep 快照：active9141 meta59.9%(持降)')
+  assert.equal(isLikelySeries(a, b), true)
+})
+
+test('ordinal progression within one session is a series', () => {
+  // #5586/#5588 — 切片2b-i then 2c-i; no date literal, caught by the window
+  const a = row('semi_abstract', 50, 'TANDEM火控切片2b-i完成: σ(w)+ω_max(w)联动→trade-off曲线立住')
+  const b = row('semi_abstract', 49.6, 'TANDEM火控切片2c-i完成: 动态w策略 > 静态最优')
+  assert.equal(isLikelySeries(a, b), true)
+})
+
+test('two unrelated todos filed together are not a rewrite', () => {
+  // #7940/#7941 — different subjects entirely; pure embedding false positive
+  const a = row('semi_abstract', 300, '【待办·Discord session】把 Discord 各处的 Steam 链接改成带 utm_source')
+  const b = row('semi_abstract', 300, '【待办·隔壁session】press kit 挂 UTM：CF Pages 部署但本地非 git 仓库')
+  assert.equal(isLikelySeries(a, b), true)
+})
+
+test('the existing concrete_trace rule still holds on its own', () => {
+  const a = row('concrete_trace', 0, '跑了一次同步')
+  const b = row('concrete_trace', 40 * 24, '跑了一次同步')
+  assert.equal(isLikelySeries(a, b), true)
+})
+
+// ── the real target: must survive the filter ───────────────────────────────
+
+test('one fact restated days later is a genuine supersede candidate', () => {
+  // #49/#272 — this is what (a2) exists to surface. 5 days apart, no temporal
+  // marker in either summary, neither is a routine trace.
+  const a = row('semi_abstract', 5 * 24, '千夏不爱吃香菜')
+  const b = row('meta_knowledge', 0, '千夏讨厌香菜的味道')
+  assert.equal(isLikelySeries(a, b), false)
+})
+
+test('a shared date literal does not make two rows a series', () => {
+  // Both mention the same date — that is one event described twice, which IS
+  // a rewrite. Only *differing* markers indicate a sequence.
+  const a = row('semi_abstract', 5 * 24, '2026-07-01 部署失败根因是端口占用')
+  const b = row('semi_abstract', 0, '2026-07-01 那次部署失败其实是端口被占')
+  assert.equal(isLikelySeries(a, b), false)
+})
+
+test('missing summaries do not crash or silently classify as series', () => {
+  const a = { memory_level: 'semi_abstract', created_at: t(5 * 24), summary: null }
+  const b = { memory_level: 'semi_abstract', created_at: t(0), summary: undefined }
+  assert.equal(isLikelySeries(a, b), false)
+})
+
+test('missing timestamps fall back to the marker rule instead of matching at zero gap', () => {
+  // created_at null on both sides would read as a 0ms gap and swallow every
+  // pair into "same session". Guard against that.
+  const a = { memory_level: 'semi_abstract', created_at: null, summary: '千夏不爱吃香菜' }
+  const b = { memory_level: 'semi_abstract', created_at: null, summary: '千夏讨厌香菜的味道' }
+  assert.equal(isLikelySeries(a, b), false)
+})