fix(recall): cold pool selects by demonstrated reuse, not by self-rated importance

.github/workflows/ci.yml

@@ -113,5 +113,10 @@ jobs:
           TOKENMEM_DB_PATH: ${{ runner.temp }}/mneme-ci-query-rewrite.db
         run: node query-rewrite.test.mjs
 
+      - name: Integration test (cold pool selects by reuse, not self-rated importance)
+        env:
+          TOKENMEM_DB_PATH: ${{ runner.temp }}/mneme-ci-cold-pool.db
+        run: node cold-pool-gate.test.mjs
+
       - name: Adapter test (codex-recovery-proxy fault injection)
         run: node adapters/codex-recovery-proxy/proxy.test.mjs

cold-pool-gate.test.mjs

@@ -0,0 +1,99 @@
+// End-to-end: cold-pool candidate selection is by demonstrated reuse, not self-rating
+// Run: node cold-pool-gate.test.mjs
+//
+// The cold pool produces the "oh, I just remembered something" surface: when a
+// recall comes back short, 25% of the time it adds 1-3 rows that are old,
+// untouched, and still worth seeing.
+//
+// "Still worth seeing" was implemented as importance >= 8 — a number the author
+// types at write time. Measured on a 9301-row library (2026-09-01), that gate
+// excluded 441 rows from a 1124-row pool, and the excluded set was led by the
+// single most-recalled memory in the whole database (access_count = 1879,
+// importance = 5). The decay floor the pool already applies is the real filter:
+// a 30-day-cold row only holds decay_score >= 0.3 if it was reused a lot, so
+// every row reaching the pool has access_count >= 8 by construction.
+
+import { initMemory, selectColdPoolCandidates, closeMemory } from './index.mjs'
+import Database from 'better-sqlite3'
+import { resolve, dirname } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { existsSync } from 'node:fs'
+
+initMemory()
+
+const __dirname_at = dirname(fileURLToPath(import.meta.url))
+const DB_PATH = process.env.TOKENMEM_DB_PATH
+  || (existsSync(resolve(__dirname_at, 'engram.db'))
+        ? resolve(__dirname_at, 'engram.db')
+        : resolve(__dirname_at, 'tokenmem.db'))
+
+const SIMPLE_EXT_PATH = resolve(__dirname_at, 'lib', 'libsimple-windows-x64', 'simple')
+function tryLoadSimple(db) {
+  try {
+    if (existsSync(SIMPLE_EXT_PATH + '.dll') || existsSync(SIMPLE_EXT_PATH)) {
+      db.loadExtension(SIMPLE_EXT_PATH)
+    }
+  } catch {}
+}
+
+const db = new Database(DB_PATH)
+tryLoadSimple(db)
+
+let pass = 0, fail = 0
+const ok = (name, cond) => {
+  if (cond) { pass++; console.log(`✓ ${name}`) }
+  else { fail++; console.log(`✗ ${name}`) }
+}
+
+const DAY = 86400_000
+const now = Date.now()
+const cold = now - 45 * DAY   // comfortably past the 30d staleness cutoff
+const warm = now - 2 * DAY
+const TAG = `coldpool-${process.pid}`
+
+const insert = db.prepare(`
+  INSERT INTO memories (id, content, summary, importance, access_count,
+                        last_accessed, created_at, decay_score, memory_level, memory_type)
+  VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'semi_abstract', 'long_term')
+`)
+const add = (id, imp, acc, last, decay) =>
+  insert.run(`${TAG}-${id}`, `cold pool fixture ${id}`, `fixture ${id}`, imp, acc, last, cold, decay)
+
+// The row the old gate wrongly excluded: heavily reused, self-rated low.
+add('proven-but-low-rated', 5, 1879, cold, 0.9)
+// The row the old gate admitted on its rating alone.
+add('rated-high', 9, 12, cold, 0.5)
+// Must stay out: not cold yet.
+add('too-warm', 10, 400, warm, 0.9)
+// Must stay out: decayed below the floor.
+add('below-decay-floor', 10, 400, cold, 0.1)
+
+const mine = (rows) => new Set(
+  rows.filter(r => String(r.id).startsWith(TAG)).map(r => String(r.id).slice(TAG.length + 1)))
+
+// take is deliberately large: selection is ORDER BY RANDOM(), so asserting on a
+// small sample would be flaky. We assert on eligibility, not on which row won.
+const picked = mine(selectColdPoolCandidates(db, { take: 5000, nowMs: now }))
+
+ok('a heavily reused memory is eligible even when its importance is low',
+   picked.has('proven-but-low-rated'))
+ok('a high self-rating still gets in — this widens the pool, it does not invert it',
+   picked.has('rated-high'))
+ok('the staleness cutoff still holds', !picked.has('too-warm'))
+ok('the decay floor still holds — it is the real relevance filter',
+   !picked.has('below-decay-floor'))
+
+const all = selectColdPoolCandidates(db, { take: 5000, nowMs: now })
+const drop = all[0]?.rowid
+const rest = selectColdPoolCandidates(db, { take: 5000, nowMs: now, excludeRowids: [drop] })
+ok('excluded rowids are honoured so the surface cannot repeat a hit',
+   drop != null && !rest.some(r => r.rowid === drop))
+
+ok('take caps the result size', selectColdPoolCandidates(db, { take: 1, nowMs: now }).length === 1)
+
+db.prepare(`DELETE FROM memories WHERE id LIKE ?`).run(`${TAG}-%`)
+db.close()
+closeMemory()
+
+console.log(`\n${fail ? 'FAIL' : 'PASS'}: ${pass} passed / ${fail} failed`)
+process.exit(fail ? 1 : 0)

index.mjs

@@ -348,7 +348,22 @@ export function initMemory() {
     log('Migration: added prior_versions column')
   } catch {}
   try {
-    db.exec(`CREATE INDEX IF NOT EXISTS idx_mem_surface_pool ON memories(importance, last_accessed, decay_score) WHERE deleted_at IS NULL AND superseded_by IS NULL AND importance >= 8`)
+    // Partial index for the cold pool. Rebuilt 2026-09-01: the old definition led
+    // on importance and was partial on `importance >= 8`. With that term gone from
+    // the pool query, SQLite can no longer use it and falls back to a scan. The
+    // new one leads on last_accessed, which is what the query range-filters.
+    //
+    // This whole migration block re-runs on every initMemory(), so the DROP is
+    // guarded on the old definition actually being present — otherwise every
+    // process start would rebuild the index, and that cost grows with the library.
+    const surfaceIdx = db.prepare(
+      `SELECT sql FROM sqlite_master WHERE type = 'index' AND name = 'idx_mem_surface_pool'`
+    ).get()
+    if (surfaceIdx?.sql && /importance/i.test(surfaceIdx.sql)) {
+      db.exec(`DROP INDEX idx_mem_surface_pool`)
+      log('Migration: rebuilt idx_mem_surface_pool (was keyed on importance)')
+    }
+    db.exec(`CREATE INDEX IF NOT EXISTS idx_mem_surface_pool ON memories(last_accessed, decay_score) WHERE deleted_at IS NULL AND superseded_by IS NULL`)
   } catch {}
   // migration 008 (v2.6): is_anchor / is_pinned scarcity-as-structure layer.
   // Ombre-Brain inspired: importance 1-10 is a weak prior that gets inflated
@@ -1941,7 +1956,7 @@ export function recallMemories(opts = {}) {
   let result = filtered.slice(0, limit)
 
   // migration 003: surfaced_random — when result < limit, 25% chance of pulling
-  // 1-3 records from the cold pool (importance >= 8 AND 30d untouched AND decay >= 0.3)
+  // 1-3 records from the cold pool (30d untouched AND decay >= 0.3 — see selectColdPoolCandidates)
   // Skipped when called internally by recallMemoriesHybrid (avoid double-surfacing)
   if (!opts._internal && queryText && result.length < limit) {
     const surfaced = surfaceRandomMemories(db, result.map(r => r.rowid), limit - result.length, now)
@@ -2635,21 +2650,34 @@ export async function buildMemoryContext(opts = {}) {
 
 // ── migration 003: surfaced_random pool ─────────────────────
 // When recall result count < limit, with 25% probability, surface 1-3 records
-// from the cold pool: importance >= 8 AND last_accessed < (now - 30d)
-//   AND decay_score >= 0.3 AND deleted_at IS NULL AND superseded_by IS NULL.
+// from the cold pool: last_accessed < (now - 30d) AND decay_score >= 0.3
+//   AND deleted_at IS NULL AND superseded_by IS NULL.
 // Models the "I just remembered something" feeling — covers cold-recall blind
-// spots and lets long-decayed but still-important memories resurface.
+// spots and lets long-decayed but still-useful memories resurface.
+//
+// 2026-09-01: dropped the importance >= 8 term. It was meant to read as "still
+// important", but importance is self-rated at write time and 89% of the library
+// sits at >= 7, so it was not selecting for importance — it was excluding 441 of
+// 1124 eligible rows on the basis of a number nobody calibrated. The excluded set
+// was led by the single most-recalled memory in the database (access_count=1879,
+// importance=5).
+//
+// The decay floor already does this job, and does it on earned signal: decay is
+// w(age) * reuseBoost(access_count), so a row that has been cold for 30 days only
+// still holds decay_score >= 0.3 if it was reused heavily. Measured on the same
+// library, every row reaching the pool had access_count >= 8 (avg 135). Selecting
+// by reuse is what "still worth remembering" was always trying to approximate.
 const SURFACE_RANDOM_PROB = 0.25
 const SURFACE_RANDOM_MAX = 3
 const SURFACE_AGE_MS = 30 * 86400_000
 const SURFACE_DECAY_FLOOR = 0.3
-const SURFACE_IMPORTANCE_MIN = 8
 
-function surfaceRandomMemories(db, excludeRowids, slotsAvailable, nowMs) {
-  if (slotsAvailable <= 0) return []
-  if (Math.random() > SURFACE_RANDOM_PROB) return []
-  const cutoff = nowMs - SURFACE_AGE_MS
-  const take = Math.min(SURFACE_RANDOM_MAX, slotsAvailable)
+/**
+ * The cold-pool query, split out from the probability roll so it can be tested
+ * without fighting Math.random() or ORDER BY RANDOM().
+ */
+export function selectColdPoolCandidates(db, { excludeRowids, take, nowMs } = {}) {
+  const cutoff = (nowMs ?? Date.now()) - SURFACE_AGE_MS
   const excludeClause = excludeRowids?.length
     ? `AND rowid NOT IN (${excludeRowids.map(() => '?').join(',')})`
     : ''
@@ -2658,13 +2686,12 @@ function surfaceRandomMemories(db, excludeRowids, slotsAvailable, nowMs) {
       SELECT rowid, * FROM memories
       WHERE deleted_at IS NULL
         AND superseded_by IS NULL
-        AND importance >= ?
         AND last_accessed < ?
         AND decay_score >= ?
         ${excludeClause}
       ORDER BY RANDOM()
       LIMIT ?
-    `).all(SURFACE_IMPORTANCE_MIN, cutoff, SURFACE_DECAY_FLOOR, ...(excludeRowids || []), take)
+    `).all(cutoff, SURFACE_DECAY_FLOOR, ...(excludeRowids || []), take ?? SURFACE_RANDOM_MAX)
     return rows.map(r => ({
       ...r,
       score: 0,
@@ -2673,11 +2700,21 @@ function surfaceRandomMemories(db, excludeRowids, slotsAvailable, nowMs) {
       recall_source: 'surfaced_random',  // callers can distinguish from query matches
     }))
   } catch (e) {
-    log(`surfaceRandomMemories failed: ${e.message}`)
+    log(`selectColdPoolCandidates failed: ${e.message}`)
     return []
   }
 }
 
+function surfaceRandomMemories(db, excludeRowids, slotsAvailable, nowMs) {
+  if (slotsAvailable <= 0) return []
+  if (Math.random() > SURFACE_RANDOM_PROB) return []
+  return selectColdPoolCandidates(db, {
+    excludeRowids,
+    take: Math.min(SURFACE_RANDOM_MAX, slotsAvailable),
+    nowMs,
+  })
+}
+
 // ── Memory Management ───────────────────────────────────────
 
 /** Clean up expired memories (both TTL path and supersede chain) */

migrations/003-add-decay-and-priors.sql

@@ -20,10 +20,15 @@ ALTER TABLE memories ADD COLUMN decay_score REAL NOT NULL DEFAULT 1.0;
 -- 2. prior_versions -- paper trail; on supersede, push old content/summary/ts into this array
 ALTER TABLE memories ADD COLUMN prior_versions TEXT NOT NULL DEFAULT '[]';
 
--- 3. Index for surfaced_random cold pool (small subset; index accelerates RANDOM() sampling)
+-- 3. Index for surfaced_random cold pool (index accelerates RANDOM() sampling).
+-- 2026-09-01: was ON (importance, last_accessed, decay_score) partial on
+-- `importance >= 8`. The pool no longer filters on importance -- it selects by
+-- staleness + decay, which is reuse-derived rather than self-rated -- so the old
+-- index could not serve the query. Leads on last_accessed, which the pool
+-- range-filters. index.mjs carries a matching guarded rebuild for existing DBs.
 CREATE INDEX IF NOT EXISTS idx_mem_surface_pool
-  ON memories(importance, last_accessed, decay_score)
-  WHERE deleted_at IS NULL AND superseded_by IS NULL AND importance >= 8;
+  ON memories(last_accessed, decay_score)
+  WHERE deleted_at IS NULL AND superseded_by IS NULL;
 
 -- Verify:
 --   PRAGMA table_info(memories);

schema.sql

@@ -93,10 +93,13 @@ CREATE INDEX IF NOT EXISTS idx_mem_superseded_by
   ON memories(superseded_by)
   WHERE superseded_by IS NOT NULL AND deleted_at IS NULL;
 
--- v2.1: Surfaced-random cold pool index (importance >= 8 AND 30d untouched AND decay >= 0.3)
+-- v2.1: Surfaced-random cold pool index (30d untouched AND decay >= 0.3).
+-- 2026-09-01: dropped the importance leg. The pool selects by staleness + decay
+-- (reuse-derived) rather than by self-rated importance, so an importance-keyed
+-- partial index could not serve the query.
 CREATE INDEX IF NOT EXISTS idx_mem_surface_pool
-  ON memories(importance, last_accessed, decay_score)
-  WHERE deleted_at IS NULL AND superseded_by IS NULL AND importance >= 8;
+  ON memories(last_accessed, decay_score)
+  WHERE deleted_at IS NULL AND superseded_by IS NULL;
 
 -- FTS5 virtual table (full-text search)
 -- Default tokenize='unicode61' (built-in, zero-dependency — boots on any SQLite,