feat(v2.10): Float32 BLOB vectors — 5x smaller, lossless, migrated in bounded slices

.github/workflows/ci.yml

@@ -123,5 +123,10 @@ jobs:
           TOKENMEM_DB_PATH: ${{ runner.temp }}/mneme-ci-embedding-timeout.db
         run: node embedding-timeout.integration.test.mjs
 
+      - name: Integration test (vector storage — Float32 BLOB codec + Migration 013)
+        env:
+          TOKENMEM_DB_PATH: ${{ runner.temp }}/mneme-ci-vector-blob.db
+        run: node vector-blob.integration.test.mjs
+
       - name: Adapter test (codex-recovery-proxy fault injection)
         run: node adapters/codex-recovery-proxy/proxy.test.mjs

README.md

@@ -445,6 +445,9 @@ TOKENMEM_COMPACT_SUMMARY="..." node index.mjs --store-compact-summary
 # Backfill embeddings for existing memories
 node backfill-embeddings.mjs --concurrency 3
 node backfill-embeddings.mjs --dry-run  # count only
+
+# Convert pre-2.10 JSON vectors to Float32 BLOBs in one go (see "Vector storage format")
+node index.mjs --migrate-vectors
 ```
 
 ---
@@ -455,6 +458,17 @@ node backfill-embeddings.mjs --dry-run  # count only
 
 Batch-generates embedding vectors for existing memories that don't have them yet. Useful when first enabling vector search on an existing database.
 
+### Vector storage format (v2.10)
+
+`memories.content_vector` holds a little-endian **Float32 BLOB**, one 4-byte lane per dimension (`vector-codec.mjs`). Before 2.10 it held a JSON array of the same numbers, which cost about 5x the bytes for no extra precision: the embedding API returns float32 values, and on a 10,920-row library every one of 2,048,000 sampled stored values was float32-exact. That library went from 226 MB of vector text to 43 MB of BLOBs.
+
+- **Migration 013** converts legacy rows in place. It runs as a background loop inside the long-lived MCP server (200-row transactions, yielding to requests between them) — never inside hook children or ordinary CLI calls, which run under a spawn timeout where bulk work gets killed half-done and taxes every turn. `PRAGMA user_version = 13` marks completion so a finished library never re-scans.
+- `node index.mjs --migrate-vectors` runs it to completion immediately (idempotent, resumable; safe to re-run after a `SQLITE_BUSY`). Run `VACUUM` afterwards to hand the freed pages back to the filesystem — the migration deliberately does not, since VACUUM takes an exclusive lock.
+- The completion marker is one-way. If legacy JSON rows reappear later (restoring a pre-2.10 backup, an import that bypasses the codec), the server's automatic loop will not notice. They still read fine, and `--migrate-vectors` ignores the marker and converts them.
+- Every reader decodes both formats, so a partially migrated library is fully functional.
+- Text that does not decode as a finite numeric array is set to NULL so the self-heal sweep re-embeds it, instead of the `!= ''` coverage checks counting it as vectorised forever.
+- The column keeps TEXT affinity; SQLite stores BLOBs verbatim in it, so no schema change was needed. `typeof(content_vector)` distinguishes the two eras.
+
 ### `migrate-claude-memories.mjs`
 
 Imports Claude Code's auto-memory `.md` files (`~/.claude/projects/*/memory/*.md`) into the SQLite database. Idempotent — safe to re-run. Does not delete original files.

backfill-embeddings.mjs

@@ -23,6 +23,7 @@ import { readFileSync, existsSync } from 'node:fs'
 import { resolve, dirname } from 'node:path'
 import { fileURLToPath } from 'node:url'
 import { createRequire } from 'node:module'
+import { encodeVector } from './vector-codec.mjs'
 
 const require = createRequire(import.meta.url)
 const __dirname = dirname(fileURLToPath(import.meta.url))
@@ -142,7 +143,7 @@ async function processOne(row) {
   try {
     const vec = await embed(row.content)
     if (!vec || vec.length !== EMBED_DIM) throw new Error(`invalid vector (len=${vec?.length})`)
-    writeTxn(row.rowid, JSON.stringify(vec), new Float32Array(vec))
+    writeTxn(row.rowid, encodeVector(vec), new Float32Array(vec))
     processed++
   } catch (e) {
     failed++

index.mjs

@@ -29,6 +29,7 @@ import { expandRecallQuery } from './query-rewrite.mjs'
 import { checkSupersedeShrink } from './high-signal-tokens.mjs'
 import { HOST_LABEL_RE } from './auth.mjs'
 import { normalizedFtsScore } from './recall-scoring.mjs'
+import { encodeVector, decodeVector } from './vector-codec.mjs'
 import {
   MAX_RECALL_CANDIDATES,
   MAX_RECALL_CONTEXT_CHARS,
@@ -513,6 +514,14 @@ export function initMemory() {
     log('Migration 012: memories_quarantine table ready')
   } catch (e) { log(`Migration 012 (quarantine) failed: ${e.message}`) }
 
+  // Migration 013 (memories.content_vector JSON -> Float32 BLOB) is deliberately
+  // NOT run here. initMemory() executes inside hook children that spawnSync
+  // kills at ~2.8 s, and inside every CLI invocation; bulk conversion in those
+  // processes gets SIGTERM'd half-done and taxes every turn until it drains.
+  // It runs as a background loop in the long-lived MCP server (mcp-server.mjs,
+  // beside the self-heal sweep) and on demand via `--migrate-vectors`.
+  // Readers accept both formats, so nothing depends on it having run.
+
   // Migration: memories.source CHECK constraint add 'compression'
   // SQLite doesn't support ALTER CHECK -> check if current CHECK includes 'compression', rebuild if not
   try {
@@ -906,7 +915,7 @@ export async function recordConversationAsync(msg) {
   if (embedding) {
     try {
       getDb().prepare(`UPDATE conversations SET content_vector = ? WHERE rowid = ?`)
-        .run(JSON.stringify(embedding), id)
+        .run(encodeVector(embedding), id)   // same Float32 BLOB codec as memories (v2.10)
     } catch {}
   }
   return id
@@ -1531,9 +1540,21 @@ function findNearDuplicates(db, embedding, excludeId, { topK = 5, threshold = 0.
         AND m.content_vector IS NOT NULL AND m.content_vector != ''
     `).all(new Float32Array(embedding), topK + 1, excludeId)
   } catch { return [] }
+  return rankNearDuplicateCandidates(embedding, cands, { threshold })
+}
+
+/**
+ * Pure half of findNearDuplicates: decode each candidate's stored vector
+ * (Float32 BLOB or legacy JSON — vector-codec.mjs) and rank by exact cosine.
+ * Split out so the dual-format read is testable without sqlite-vec loaded.
+ * @param {number[]} embedding
+ * @param {{id:number, content_vector:any, summary?:string, content?:string}[]} cands
+ */
+export function rankNearDuplicateCandidates(embedding, cands, { threshold = 0.92 } = {}) {
   const out = []
   for (const c of cands) {
-    let v; try { v = JSON.parse(c.content_vector) } catch { continue }
+    const v = decodeVector(c.content_vector)
+    if (!v) continue
     const cos = cosineSim(embedding, v)
     if (cos >= threshold) out.push({ id: c.id, cosine: +cos.toFixed(4), summary: c.summary || (c.content || '').slice(0, 60) })
   }
@@ -1553,9 +1574,13 @@ export async function storeMemoryAsync(mem, opts = {}) {
   if (embedding) {
     const db = getDb()
     try {
-      // Store JSON string in memories.content_vector (cross-tool visible + backup)
+      // Float32 BLOB (v2.10, vector-codec.mjs). This was a JSON string "for
+      // cross-tool visibility" — but no other process reads this database's
+      // column (a project-wide grep found only a sibling engine copy with its
+      // own DB, ad-hoc scripts that never parse it, and a worktree of this
+      // repo), and JSON cost 5x the bytes to spell out the same float32 values.
       db.prepare(`UPDATE memories SET content_vector = ? WHERE rowid = ?`)
-        .run(JSON.stringify(embedding), id)
+        .run(encodeVector(embedding), id)
     } catch {}
 
     // Sync to sqlite-vec virtual table (for KNN queries)
@@ -1579,6 +1604,80 @@ export async function storeMemoryAsync(mem, opts = {}) {
   return id
 }
 
+/**
+ * Migration 013 worker: convert legacy JSON-text vectors to Float32 BLOBs.
+ *
+ * Idempotent and resumable: converted rows become typeof 'blob' and drop out
+ * of the WHERE, so every batch makes progress and re-running is a no-op.
+ * Text that does not decode to a finite numeric array was never a usable
+ * vector; it is set to NULL so embedMissingVectors() re-embeds that row
+ * instead of the `!= ''` coverage checks counting it as vectorised forever.
+ *
+ * Never runs COUNT(*) unless asked: typeof() cannot use an index, so counting
+ * is a full table scan — the exact cost this function exists to bound. Whether
+ * the backlog is drained is known from the loop itself (a short batch).
+ *
+ * @param {object} [opts]
+ * @param {import('better-sqlite3').Database} [opts.db]   defaults to getDb()
+ * @param {number}  [opts.limit=Infinity]    max rows to scan this call
+ * @param {number}  [opts.budgetMs=Infinity] stop starting new batches after this
+ * @param {number}  [opts.batch=200]         rows per transaction
+ * @param {boolean} [opts.count=false]       if cut short, also COUNT what is left (full scan)
+ * @param {boolean} [opts.skipIfComplete=false]  honour the PRAGMA user_version marker: return
+ *   drained immediately, without any scan, once a previous run has drained. The
+ *   server's background loop passes this; the CLI does not, so an explicit run
+ *   still converts legacy rows that appear after completion (restored backup, import).
+ * @returns {{scanned:number, converted:number, skipped:number, drained:boolean, remaining:number|null, ms:number}}
+ *   remaining: 0 when drained; the exact count when `count` was requested; else null.
+ */
+export function migrateVectorsToBlob({ db: dbArg = null, limit = Infinity, budgetMs = Infinity, batch = 200, count = false, skipIfComplete = false } = {}) {
+  const db = dbArg || getDb()
+  const t0 = Date.now()
+  const out = { scanned: 0, converted: 0, skipped: 0, drained: false, remaining: null, ms: 0 }
+  if (skipIfComplete) {
+    let uv = 0
+    try { uv = db.pragma('user_version', { simple: true }) } catch {}
+    if (uv >= 13) { out.drained = true; out.remaining = 0; out.ms = Date.now() - t0; return out }
+  }
+  const selectBatch = db.prepare(`
+    SELECT rowid, content_vector AS v FROM memories
+    WHERE typeof(content_vector) = 'text' AND content_vector != ''
+    LIMIT ?
+  `)
+  // Both writes re-check typeof inside the transaction: a row another process
+  // converted (or re-embedded) between our SELECT and our UPDATE is left alone
+  // instead of being overwritten with a stale decode.
+  const update = db.prepare(`UPDATE memories SET content_vector = ? WHERE rowid = ? AND typeof(content_vector) = 'text'`)
+  const clear  = db.prepare(`UPDATE memories SET content_vector = NULL WHERE rowid = ? AND typeof(content_vector) = 'text'`)
+  const convertBatch = db.transaction((rows) => {
+    for (const r of rows) {
+      out.scanned++
+      const blob = encodeVector(decodeVector(r.v))   // null if undecodable OR non-finite
+      if (!blob) { if (clear.run(r.rowid).changes) out.skipped++; continue }
+      if (update.run(blob, r.rowid).changes) out.converted++
+    }
+  })
+  for (;;) {
+    if (out.scanned >= limit || (Date.now() - t0) >= budgetMs) break
+    const want = Math.max(1, Math.min(batch, limit - out.scanned))
+    const rows = selectBatch.all(want)
+    if (rows.length === 0) { out.drained = true; break }
+    convertBatch(rows)
+    if (rows.length < want) { out.drained = true; break }   // short batch: table exhausted
+  }
+  if (out.drained) {
+    out.remaining = 0
+    // Completion marker read by the MCP server's background runner.
+    try { if (db.pragma('user_version', { simple: true }) < 13) db.pragma('user_version = 13') } catch {}
+  } else if (count) {
+    out.remaining = db.prepare(
+      `SELECT COUNT(*) AS c FROM memories WHERE typeof(content_vector) = 'text' AND content_vector != ''`
+    ).get().c
+  }
+  out.ms = Date.now() - t0
+  return out
+}
+
 /**
  * Self-heal sweep: fill missing content_vector on active memories.
  * Covers writes that bypassed storeMemoryAsync (sync storeMemory / CLI / batch)
@@ -1604,7 +1703,7 @@ export async function embedMissingVectors(limit = 200) {
     try {
       const vec = await generateEmbedding(row.content)
       if (!vec || vec.length !== dim) { failed++; continue }  // 维度不符 → 拒绝写脏向量
-      updateStmt.run(JSON.stringify(vec), row.rowid)
+      updateStmt.run(encodeVector(vec), row.rowid)
       if (_vecLoaded) {
         try {
           db.prepare(`INSERT OR REPLACE INTO memories_vec(memory_rowid, embedding) VALUES (?, ?)`)
@@ -4106,6 +4205,23 @@ if (_isMain) {
         }
       }
 
+    } else if (hasFlag('--migrate-vectors')) {
+      // v2.10: run Migration 013 to completion in this process. (The MCP server
+      // also drains it in the background; hooks and other CLI calls never do.)
+      // Idempotent and resumable, so a SQLITE_BUSY from a concurrent writer is
+      // reported cleanly and the command is simply re-run. Follow with `VACUUM`
+      // to hand the freed pages back to the filesystem — the migration itself
+      // does not, because VACUUM takes an exclusive lock and that is an
+      // operator's call.
+      try {
+        const r = migrateVectorsToBlob({ count: true })
+        process.stdout.write(JSON.stringify(r) + '\n')
+        process.exitCode = r.drained ? 0 : 1
+      } catch (e) {
+        process.stderr.write(`--migrate-vectors failed: ${e.message} (safe to re-run; converted rows stay converted)\n`)
+        process.exitCode = 1
+      }
+
     } else if (getFlag('--context') !== null) {
       const query = getFlag('--context') || ''
       const ctx = await buildMemoryContext({ query, memoryLimit: 10 })

mcp-server.mjs

@@ -46,6 +46,7 @@ import {
   listLocations,
   deleteLocation,
 } from './index.mjs'
+import { migrateVectorsToBlob } from './index.mjs'   // Migration 013 background runner (below)
 import { parseHostTokens, resolveAuthMode, resolveHost } from './auth.mjs'
 import { recallClaudeMarkdownMemory } from './lib/claude-markdown-memory.mjs'
 
@@ -84,6 +85,31 @@ embedMissingVectors(500).then(r => {
   if (r.embedded || r.failed) console.error(`[mneme] startup self-heal: embedded ${r.embedded}, failed ${r.failed}, scanned ${r.scanned}`)
 }).catch(e => console.error(`[mneme] startup self-heal failed: ${e.message}`))
 
+// Migration 013 background runner: memories.content_vector JSON -> Float32 BLOB
+// (vector-codec.mjs). It lives here rather than in initMemory() because this is
+// the one long-lived process: hook children are spawnSync-killed at ~2.8 s, and
+// bulk work there gets SIGTERM'd half-done and taxes every turn until it drains.
+// 200-row transactions with a yield between them keep request latency flat; a
+// 10k-row backlog drains in a few seconds. skipIfComplete turns every later
+// start into a single PRAGMA read — no typeof() scan of a finished table.
+;(function runVectorMigration() {
+  let converted = 0, skipped = 0
+  const tick = () => {
+    try {
+      const r = migrateVectorsToBlob({ limit: 200, skipIfComplete: true })
+      converted += r.converted; skipped += r.skipped
+      if (r.drained) {
+        if (converted || skipped) console.error(`[mneme] Migration 013: ${converted} vector(s) -> Float32 BLOB, ${skipped} unparseable cleared — complete`)
+        return
+      }
+      setTimeout(tick, 25).unref()          // yield to in-flight requests between batches
+    } catch (e) {
+      console.error(`[mneme] Migration 013 paused: ${e.message} (resumes on next start; readers accept both formats)`)
+    }
+  }
+  setTimeout(tick, 1000).unref()            // let startup settle before the first batch
+})()
+
 const SERVER_NAME = 'mneme'
 const SERVER_VERSION = '2.8.0'
 

memory-health.mjs

@@ -43,6 +43,7 @@ import { dirname, resolve } from 'node:path'
 import { fileURLToPath, pathToFileURL } from 'node:url'
 import { createRequire } from 'node:module'
 import { extractHighSignalTokens, isStillCarried } from './high-signal-tokens.mjs'
+import { decodeVector } from './vector-codec.mjs'
 
 const require = createRequire(import.meta.url)
 const __dirname = dirname(fileURLToPath(import.meta.url))
@@ -149,8 +150,10 @@ export function isNoiseQuery(q) {
 }
 
 // ── Vector helpers (pre-normalized cosine = dot product) ──
-function parseVec(json) {
-  try { const v = JSON.parse(json); return Array.isArray(v) ? v : null } catch { return null }
+function parseVec(stored) {
+  // Float32 BLOB (v2.10+) or legacy JSON text — vector-codec.mjs reads both,
+  // so this scan is correct on a half-migrated library.
+  return decodeVector(stored)
 }
 function normalize(v) {
   let n = 0

package.json

@@ -1,6 +1,6 @@
 {
   "name": "mneme",
-  "version": "2.9.0",
+  "version": "2.10.0",
   "description": "Token-efficient persistent memory for AI agents — SQLite + FTS5 + sqlite-vec hybrid search + MCP on-demand recall. Save 80-90% memory-related token costs.",
   "type": "module",
   "main": "index.mjs",

schema.sql

@@ -55,7 +55,13 @@ CREATE TABLE IF NOT EXISTS memories (
   -- Extended metadata
   metadata TEXT DEFAULT '{}',
 
-  -- Vector (JSON array, optional, application-layer cosine similarity)
+  -- Vector: Float32 BLOB since v2.10 (host byte order; see vector-codec.mjs).
+  -- Column keeps TEXT affinity on purpose — SQLite stores BLOBs verbatim in a
+  -- TEXT column, so no schema change was needed; typeof() = 'text' rows are
+  -- pre-2.10 JSON arrays that Migration 013 converts in place. Readers accept
+  -- both. Optional. Read only by application-layer cosine (the near-duplicate
+  -- write gate and memory-health); sqlite-vec KNN uses the separate memories_vec
+  -- table, populated from the same embedding at write time, and never reads this.
   content_vector TEXT,
 
   -- Timestamps & access stats