diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6a7df6b..b4a6389 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -123,5 +123,10 @@ jobs: TOKENMEM_DB_PATH: ${{ runner.temp }}/mneme-ci-embedding-timeout.db run: node embedding-timeout.integration.test.mjs + - name: Integration test (vector storage — Float32 BLOB codec + Migration 013) + env: + TOKENMEM_DB_PATH: ${{ runner.temp }}/mneme-ci-vector-blob.db + run: node vector-blob.integration.test.mjs + - name: Adapter test (codex-recovery-proxy fault injection) run: node adapters/codex-recovery-proxy/proxy.test.mjs diff --git a/README.md b/README.md index 1f05f4c..e42d897 100644 --- a/README.md +++ b/README.md @@ -445,6 +445,9 @@ TOKENMEM_COMPACT_SUMMARY="..." node index.mjs --store-compact-summary # Backfill embeddings for existing memories node backfill-embeddings.mjs --concurrency 3 node backfill-embeddings.mjs --dry-run # count only + +# Convert pre-2.10 JSON vectors to Float32 BLOBs in one go (see "Vector storage format") +node index.mjs --migrate-vectors ``` --- @@ -455,6 +458,17 @@ node backfill-embeddings.mjs --dry-run # count only Batch-generates embedding vectors for existing memories that don't have them yet. Useful when first enabling vector search on an existing database. +### Vector storage format (v2.10) + +`memories.content_vector` holds a little-endian **Float32 BLOB**, one 4-byte lane per dimension (`vector-codec.mjs`). Before 2.10 it held a JSON array of the same numbers, which cost about 5x the bytes for no extra precision: the embedding API returns float32 values, and on a 10,920-row library every one of 2,048,000 sampled stored values was float32-exact. That library went from 226 MB of vector text to 43 MB of BLOBs. + +- **Migration 013** converts legacy rows in place. It runs as a background loop inside the long-lived MCP server (200-row transactions, yielding to requests between them) — never inside hook children or ordinary CLI calls, which run under a spawn timeout where bulk work gets killed half-done and taxes every turn. `PRAGMA user_version = 13` marks completion so a finished library never re-scans. +- `node index.mjs --migrate-vectors` runs it to completion immediately (idempotent, resumable; safe to re-run after a `SQLITE_BUSY`). Run `VACUUM` afterwards to hand the freed pages back to the filesystem — the migration deliberately does not, since VACUUM takes an exclusive lock. +- The completion marker is one-way. If legacy JSON rows reappear later (restoring a pre-2.10 backup, an import that bypasses the codec), the server's automatic loop will not notice. They still read fine, and `--migrate-vectors` ignores the marker and converts them. +- Every reader decodes both formats, so a partially migrated library is fully functional. +- Text that does not decode as a finite numeric array is set to NULL so the self-heal sweep re-embeds it, instead of the `!= ''` coverage checks counting it as vectorised forever. +- The column keeps TEXT affinity; SQLite stores BLOBs verbatim in it, so no schema change was needed. `typeof(content_vector)` distinguishes the two eras. + ### `migrate-claude-memories.mjs` Imports Claude Code's auto-memory `.md` files (`~/.claude/projects/*/memory/*.md`) into the SQLite database. Idempotent — safe to re-run. Does not delete original files. diff --git a/backfill-embeddings.mjs b/backfill-embeddings.mjs index ff53562..865d48a 100644 --- a/backfill-embeddings.mjs +++ b/backfill-embeddings.mjs @@ -23,6 +23,7 @@ import { readFileSync, existsSync } from 'node:fs' import { resolve, dirname } from 'node:path' import { fileURLToPath } from 'node:url' import { createRequire } from 'node:module' +import { encodeVector } from './vector-codec.mjs' const require = createRequire(import.meta.url) const __dirname = dirname(fileURLToPath(import.meta.url)) @@ -142,7 +143,7 @@ async function processOne(row) { try { const vec = await embed(row.content) if (!vec || vec.length !== EMBED_DIM) throw new Error(`invalid vector (len=${vec?.length})`) - writeTxn(row.rowid, JSON.stringify(vec), new Float32Array(vec)) + writeTxn(row.rowid, encodeVector(vec), new Float32Array(vec)) processed++ } catch (e) { failed++ diff --git a/index.mjs b/index.mjs index afac335..408a62d 100644 --- a/index.mjs +++ b/index.mjs @@ -29,6 +29,7 @@ import { expandRecallQuery } from './query-rewrite.mjs' import { checkSupersedeShrink } from './high-signal-tokens.mjs' import { HOST_LABEL_RE } from './auth.mjs' import { normalizedFtsScore } from './recall-scoring.mjs' +import { encodeVector, decodeVector } from './vector-codec.mjs' import { MAX_RECALL_CANDIDATES, MAX_RECALL_CONTEXT_CHARS, @@ -513,6 +514,14 @@ export function initMemory() { log('Migration 012: memories_quarantine table ready') } catch (e) { log(`Migration 012 (quarantine) failed: ${e.message}`) } + // Migration 013 (memories.content_vector JSON -> Float32 BLOB) is deliberately + // NOT run here. initMemory() executes inside hook children that spawnSync + // kills at ~2.8 s, and inside every CLI invocation; bulk conversion in those + // processes gets SIGTERM'd half-done and taxes every turn until it drains. + // It runs as a background loop in the long-lived MCP server (mcp-server.mjs, + // beside the self-heal sweep) and on demand via `--migrate-vectors`. + // Readers accept both formats, so nothing depends on it having run. + // Migration: memories.source CHECK constraint add 'compression' // SQLite doesn't support ALTER CHECK -> check if current CHECK includes 'compression', rebuild if not try { @@ -906,7 +915,7 @@ export async function recordConversationAsync(msg) { if (embedding) { try { getDb().prepare(`UPDATE conversations SET content_vector = ? WHERE rowid = ?`) - .run(JSON.stringify(embedding), id) + .run(encodeVector(embedding), id) // same Float32 BLOB codec as memories (v2.10) } catch {} } return id @@ -1531,9 +1540,21 @@ function findNearDuplicates(db, embedding, excludeId, { topK = 5, threshold = 0. AND m.content_vector IS NOT NULL AND m.content_vector != '' `).all(new Float32Array(embedding), topK + 1, excludeId) } catch { return [] } + return rankNearDuplicateCandidates(embedding, cands, { threshold }) +} + +/** + * Pure half of findNearDuplicates: decode each candidate's stored vector + * (Float32 BLOB or legacy JSON — vector-codec.mjs) and rank by exact cosine. + * Split out so the dual-format read is testable without sqlite-vec loaded. + * @param {number[]} embedding + * @param {{id:number, content_vector:any, summary?:string, content?:string}[]} cands + */ +export function rankNearDuplicateCandidates(embedding, cands, { threshold = 0.92 } = {}) { const out = [] for (const c of cands) { - let v; try { v = JSON.parse(c.content_vector) } catch { continue } + const v = decodeVector(c.content_vector) + if (!v) continue const cos = cosineSim(embedding, v) if (cos >= threshold) out.push({ id: c.id, cosine: +cos.toFixed(4), summary: c.summary || (c.content || '').slice(0, 60) }) } @@ -1553,9 +1574,13 @@ export async function storeMemoryAsync(mem, opts = {}) { if (embedding) { const db = getDb() try { - // Store JSON string in memories.content_vector (cross-tool visible + backup) + // Float32 BLOB (v2.10, vector-codec.mjs). This was a JSON string "for + // cross-tool visibility" — but no other process reads this database's + // column (a project-wide grep found only a sibling engine copy with its + // own DB, ad-hoc scripts that never parse it, and a worktree of this + // repo), and JSON cost 5x the bytes to spell out the same float32 values. db.prepare(`UPDATE memories SET content_vector = ? WHERE rowid = ?`) - .run(JSON.stringify(embedding), id) + .run(encodeVector(embedding), id) } catch {} // Sync to sqlite-vec virtual table (for KNN queries) @@ -1579,6 +1604,80 @@ export async function storeMemoryAsync(mem, opts = {}) { return id } +/** + * Migration 013 worker: convert legacy JSON-text vectors to Float32 BLOBs. + * + * Idempotent and resumable: converted rows become typeof 'blob' and drop out + * of the WHERE, so every batch makes progress and re-running is a no-op. + * Text that does not decode to a finite numeric array was never a usable + * vector; it is set to NULL so embedMissingVectors() re-embeds that row + * instead of the `!= ''` coverage checks counting it as vectorised forever. + * + * Never runs COUNT(*) unless asked: typeof() cannot use an index, so counting + * is a full table scan — the exact cost this function exists to bound. Whether + * the backlog is drained is known from the loop itself (a short batch). + * + * @param {object} [opts] + * @param {import('better-sqlite3').Database} [opts.db] defaults to getDb() + * @param {number} [opts.limit=Infinity] max rows to scan this call + * @param {number} [opts.budgetMs=Infinity] stop starting new batches after this + * @param {number} [opts.batch=200] rows per transaction + * @param {boolean} [opts.count=false] if cut short, also COUNT what is left (full scan) + * @param {boolean} [opts.skipIfComplete=false] honour the PRAGMA user_version marker: return + * drained immediately, without any scan, once a previous run has drained. The + * server's background loop passes this; the CLI does not, so an explicit run + * still converts legacy rows that appear after completion (restored backup, import). + * @returns {{scanned:number, converted:number, skipped:number, drained:boolean, remaining:number|null, ms:number}} + * remaining: 0 when drained; the exact count when `count` was requested; else null. + */ +export function migrateVectorsToBlob({ db: dbArg = null, limit = Infinity, budgetMs = Infinity, batch = 200, count = false, skipIfComplete = false } = {}) { + const db = dbArg || getDb() + const t0 = Date.now() + const out = { scanned: 0, converted: 0, skipped: 0, drained: false, remaining: null, ms: 0 } + if (skipIfComplete) { + let uv = 0 + try { uv = db.pragma('user_version', { simple: true }) } catch {} + if (uv >= 13) { out.drained = true; out.remaining = 0; out.ms = Date.now() - t0; return out } + } + const selectBatch = db.prepare(` + SELECT rowid, content_vector AS v FROM memories + WHERE typeof(content_vector) = 'text' AND content_vector != '' + LIMIT ? + `) + // Both writes re-check typeof inside the transaction: a row another process + // converted (or re-embedded) between our SELECT and our UPDATE is left alone + // instead of being overwritten with a stale decode. + const update = db.prepare(`UPDATE memories SET content_vector = ? WHERE rowid = ? AND typeof(content_vector) = 'text'`) + const clear = db.prepare(`UPDATE memories SET content_vector = NULL WHERE rowid = ? AND typeof(content_vector) = 'text'`) + const convertBatch = db.transaction((rows) => { + for (const r of rows) { + out.scanned++ + const blob = encodeVector(decodeVector(r.v)) // null if undecodable OR non-finite + if (!blob) { if (clear.run(r.rowid).changes) out.skipped++; continue } + if (update.run(blob, r.rowid).changes) out.converted++ + } + }) + for (;;) { + if (out.scanned >= limit || (Date.now() - t0) >= budgetMs) break + const want = Math.max(1, Math.min(batch, limit - out.scanned)) + const rows = selectBatch.all(want) + if (rows.length === 0) { out.drained = true; break } + convertBatch(rows) + if (rows.length < want) { out.drained = true; break } // short batch: table exhausted + } + if (out.drained) { + out.remaining = 0 + // Completion marker read by the MCP server's background runner. + try { if (db.pragma('user_version', { simple: true }) < 13) db.pragma('user_version = 13') } catch {} + } else if (count) { + out.remaining = db.prepare( + `SELECT COUNT(*) AS c FROM memories WHERE typeof(content_vector) = 'text' AND content_vector != ''` + ).get().c + } + out.ms = Date.now() - t0 + return out +} + /** * Self-heal sweep: fill missing content_vector on active memories. * Covers writes that bypassed storeMemoryAsync (sync storeMemory / CLI / batch) @@ -1604,7 +1703,7 @@ export async function embedMissingVectors(limit = 200) { try { const vec = await generateEmbedding(row.content) if (!vec || vec.length !== dim) { failed++; continue } // 维度不符 → 拒绝写脏向量 - updateStmt.run(JSON.stringify(vec), row.rowid) + updateStmt.run(encodeVector(vec), row.rowid) if (_vecLoaded) { try { db.prepare(`INSERT OR REPLACE INTO memories_vec(memory_rowid, embedding) VALUES (?, ?)`) @@ -4106,6 +4205,23 @@ if (_isMain) { } } + } else if (hasFlag('--migrate-vectors')) { + // v2.10: run Migration 013 to completion in this process. (The MCP server + // also drains it in the background; hooks and other CLI calls never do.) + // Idempotent and resumable, so a SQLITE_BUSY from a concurrent writer is + // reported cleanly and the command is simply re-run. Follow with `VACUUM` + // to hand the freed pages back to the filesystem — the migration itself + // does not, because VACUUM takes an exclusive lock and that is an + // operator's call. + try { + const r = migrateVectorsToBlob({ count: true }) + process.stdout.write(JSON.stringify(r) + '\n') + process.exitCode = r.drained ? 0 : 1 + } catch (e) { + process.stderr.write(`--migrate-vectors failed: ${e.message} (safe to re-run; converted rows stay converted)\n`) + process.exitCode = 1 + } + } else if (getFlag('--context') !== null) { const query = getFlag('--context') || '' const ctx = await buildMemoryContext({ query, memoryLimit: 10 }) diff --git a/mcp-server.mjs b/mcp-server.mjs index 38d3e2f..3c87534 100644 --- a/mcp-server.mjs +++ b/mcp-server.mjs @@ -46,6 +46,7 @@ import { listLocations, deleteLocation, } from './index.mjs' +import { migrateVectorsToBlob } from './index.mjs' // Migration 013 background runner (below) import { parseHostTokens, resolveAuthMode, resolveHost } from './auth.mjs' import { recallClaudeMarkdownMemory } from './lib/claude-markdown-memory.mjs' @@ -84,6 +85,31 @@ embedMissingVectors(500).then(r => { if (r.embedded || r.failed) console.error(`[mneme] startup self-heal: embedded ${r.embedded}, failed ${r.failed}, scanned ${r.scanned}`) }).catch(e => console.error(`[mneme] startup self-heal failed: ${e.message}`)) +// Migration 013 background runner: memories.content_vector JSON -> Float32 BLOB +// (vector-codec.mjs). It lives here rather than in initMemory() because this is +// the one long-lived process: hook children are spawnSync-killed at ~2.8 s, and +// bulk work there gets SIGTERM'd half-done and taxes every turn until it drains. +// 200-row transactions with a yield between them keep request latency flat; a +// 10k-row backlog drains in a few seconds. skipIfComplete turns every later +// start into a single PRAGMA read — no typeof() scan of a finished table. +;(function runVectorMigration() { + let converted = 0, skipped = 0 + const tick = () => { + try { + const r = migrateVectorsToBlob({ limit: 200, skipIfComplete: true }) + converted += r.converted; skipped += r.skipped + if (r.drained) { + if (converted || skipped) console.error(`[mneme] Migration 013: ${converted} vector(s) -> Float32 BLOB, ${skipped} unparseable cleared — complete`) + return + } + setTimeout(tick, 25).unref() // yield to in-flight requests between batches + } catch (e) { + console.error(`[mneme] Migration 013 paused: ${e.message} (resumes on next start; readers accept both formats)`) + } + } + setTimeout(tick, 1000).unref() // let startup settle before the first batch +})() + const SERVER_NAME = 'mneme' const SERVER_VERSION = '2.8.0' diff --git a/memory-health.mjs b/memory-health.mjs index 0ba28f0..53a10b0 100644 --- a/memory-health.mjs +++ b/memory-health.mjs @@ -43,6 +43,7 @@ import { dirname, resolve } from 'node:path' import { fileURLToPath, pathToFileURL } from 'node:url' import { createRequire } from 'node:module' import { extractHighSignalTokens, isStillCarried } from './high-signal-tokens.mjs' +import { decodeVector } from './vector-codec.mjs' const require = createRequire(import.meta.url) const __dirname = dirname(fileURLToPath(import.meta.url)) @@ -149,8 +150,10 @@ export function isNoiseQuery(q) { } // ── Vector helpers (pre-normalized cosine = dot product) ── -function parseVec(json) { - try { const v = JSON.parse(json); return Array.isArray(v) ? v : null } catch { return null } +function parseVec(stored) { + // Float32 BLOB (v2.10+) or legacy JSON text — vector-codec.mjs reads both, + // so this scan is correct on a half-migrated library. + return decodeVector(stored) } function normalize(v) { let n = 0 diff --git a/package.json b/package.json index e21e8fe..4abc6b0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "mneme", - "version": "2.9.0", + "version": "2.10.0", "description": "Token-efficient persistent memory for AI agents — SQLite + FTS5 + sqlite-vec hybrid search + MCP on-demand recall. Save 80-90% memory-related token costs.", "type": "module", "main": "index.mjs", diff --git a/schema.sql b/schema.sql index 2f9e475..45d7dc3 100644 --- a/schema.sql +++ b/schema.sql @@ -55,7 +55,13 @@ CREATE TABLE IF NOT EXISTS memories ( -- Extended metadata metadata TEXT DEFAULT '{}', - -- Vector (JSON array, optional, application-layer cosine similarity) + -- Vector: Float32 BLOB since v2.10 (host byte order; see vector-codec.mjs). + -- Column keeps TEXT affinity on purpose — SQLite stores BLOBs verbatim in a + -- TEXT column, so no schema change was needed; typeof() = 'text' rows are + -- pre-2.10 JSON arrays that Migration 013 converts in place. Readers accept + -- both. Optional. Read only by application-layer cosine (the near-duplicate + -- write gate and memory-health); sqlite-vec KNN uses the separate memories_vec + -- table, populated from the same embedding at write time, and never reads this. content_vector TEXT, -- Timestamps & access stats diff --git a/vector-blob.integration.test.mjs b/vector-blob.integration.test.mjs new file mode 100644 index 0000000..20f94d7 --- /dev/null +++ b/vector-blob.integration.test.mjs @@ -0,0 +1,216 @@ +// Integration: memories.content_vector as Float32 BLOB — codec + Migration 013 +// Run: node vector-blob.integration.test.mjs +// +// The column used to hold a JSON array. Measured on a 10,920-row library the +// JSON form was 226 MB for values that are all float32-exact (2,048,000 of +// 2,048,000 sampled satisfied Math.fround(x) === x), i.e. 5x the bytes to +// write the same 32-bit numbers in decimal. This test pins three properties: +// +// 1. The codec is lossless on float32 values and tolerant of everything a +// column can actually contain (legacy JSON, BLOB, '', NULL, junk). +// 2. Migration 013 is bounded, resumable, idempotent, converts in place, and +// NULLs unparseable text instead of leaving it to be counted as a vector +// forever by the `!= ''` coverage checks. +// 3. A reader that predates the change (memory-health's near-dup scan) sees +// BLOB rows and legacy rows as the same thing, so a half-migrated library +// is not a broken library. + +import { initMemory, closeMemory, migrateVectorsToBlob, rankNearDuplicateCandidates } from './index.mjs' +import { encodeVector, decodeVector, vectorStorageKind } from './vector-codec.mjs' +import { detectNearDup } from './memory-health.mjs' +import Database from 'better-sqlite3' +import { resolve, dirname } from 'node:path' +import { fileURLToPath } from 'node:url' +import { existsSync } from 'node:fs' + +initMemory() + +const __dirname_at = dirname(fileURLToPath(import.meta.url)) +const DB_PATH = process.env.TOKENMEM_DB_PATH + || (existsSync(resolve(__dirname_at, 'engram.db')) + ? resolve(__dirname_at, 'engram.db') + : resolve(__dirname_at, 'tokenmem.db')) + +const db = new Database(DB_PATH) + +let pass = 0, fail = 0 +const ok = (name, cond) => { + if (cond) { pass++; console.log(`✓ ${name}`) } + else { fail++; console.log(`✗ ${name}`) } +} +const near = (a, b, eps = 1e-6) => a.length === b.length && a.every((x, i) => Math.abs(x - b[i]) < eps) + +// ── 1. codec ──────────────────────────────────────────────────────────────── +{ + const v = [-0.07749947160482407, 0.007732080761343241, 0.5, 1, -1, 0] // all float32-exact + ok('input is float32-exact (test premise)', v.every(x => Math.fround(x) === x)) + const buf = encodeVector(v) + ok('encode → Buffer of 4 bytes per lane', Buffer.isBuffer(buf) && buf.byteLength === v.length * 4) + ok('decode(encode(v)) is exactly v', decodeVector(buf).every((x, i) => x === v[i])) + ok('storage kind of encoded value is blob', vectorStorageKind(buf) === 'blob') + + ok('decode legacy JSON array', near(decodeVector('[0.5, -1, 2]'), [0.5, -1, 2])) + ok('decode legacy JSON with surrounding whitespace', near(decodeVector(' [1,2]\n'), [1, 2])) + ok('storage kind of JSON text is json', vectorStorageKind('[1]') === 'json') + + ok('decode(null) → null', decodeVector(null) === null) + ok('decode(undefined) → null', decodeVector(undefined) === null) + ok("decode('') → null", decodeVector('') === null) + ok("storage kind of '' is empty", vectorStorageKind('') === 'empty') + ok('decode junk text → null', decodeVector('not a vector') === null) + ok('decode JSON that is not all finite numbers → null', decodeVector('[1,"a",2]') === null && decodeVector('[1,null]') === null) + ok('decode JSON object → null', decodeVector('{"a":1}') === null) + ok('decode empty JSON array → null', decodeVector('[]') === null) + ok('decode BLOB with byteLength not divisible by 4 → null', decodeVector(Buffer.from([1, 2, 3])) === null) + ok('decode empty BLOB → null', decodeVector(Buffer.alloc(0)) === null) + const nanBuf = Buffer.alloc(8); nanBuf.writeFloatLE(1, 0); nanBuf.writeFloatLE(NaN, 4) + ok('decode BLOB holding NaN → null', decodeVector(nanBuf) === null) + ok('encode rejects NaN / Infinity / float32 overflow (symmetric with decode)', + encodeVector([1, NaN]) === null && encodeVector([1, Infinity]) === null && encodeVector([1, 1e39]) === null) + + // Driver Buffers can be unaligned slices of a slab. A naive Float32Array view + // would throw RangeError; the codec must copy first. + const slab = Buffer.alloc(1 + 8) + slab.writeFloatLE(3.5, 1); slab.writeFloatLE(-2, 5) + const unaligned = slab.subarray(1) + ok('decode unaligned Buffer slice (byteOffset=1) works', near(decodeVector(unaligned), [3.5, -2])) + + // Float32Array input must be copied, not aliased. + const f = new Float32Array([1, 2]); const b2 = encodeVector(f); f[0] = 99 + ok('encode(Float32Array) does not alias the caller\'s buffer', decodeVector(b2)[0] === 1) + + ok('encode(null|[]) → null', encodeVector(null) === null && encodeVector([]) === null) +} + +// ── 2. Migration 013 ──────────────────────────────────────────────────────── +const now = Date.now() +const ins = db.prepare(` + INSERT INTO memories (id, content, summary, category, memory_level, memory_type, + importance, content_vector, created_at, updated_at, last_accessed, access_count) + VALUES (?, ?, ?, ?, ?, 'long_term', ?, ?, ?, ?, ?, 0) +`) +db.prepare(`DELETE FROM memories WHERE id LIKE 'vb-%'`).run() + +const legacy = { + 'vb-l1': [1, 0, 0], + 'vb-l2': [0.9995, 0.0316, 0], + 'vb-l3': [0, 1, 0], + 'vb-l4': [0.5, 0.5, 0.7071067690849304], + 'vb-l5': [-1, 0, 0], +} +for (const [id, vec] of Object.entries(legacy)) { + ins.run(id, `content ${id}`, `summary ${id}`, 'skill', 'semi_abstract', 6, JSON.stringify(vec), now, now, now) +} +ins.run('vb-blob', 'content vb-blob', 'summary vb-blob', 'skill', 'semi_abstract', 6, encodeVector([0, 0, 1]), now, now, now) +ins.run('vb-junk', 'content vb-junk', 'summary vb-junk', 'skill', 'semi_abstract', 6, 'not a vector', now, now, now) +ins.run('vb-empty', 'content vb-empty', 'summary vb-empty', 'skill', 'semi_abstract', 6, '', now, now, now) +ins.run('vb-null', 'content vb-null', 'summary vb-null', 'skill', 'semi_abstract', 6, null, now, now, now) + +const kindOf = (id) => db.prepare(`SELECT typeof(content_vector) AS t FROM memories WHERE id = ?`).get(id).t +const vecOf = (id) => decodeVector(db.prepare(`SELECT content_vector AS v FROM memories WHERE id = ?`).get(id).v) +const legacyLeft = () => db.prepare(`SELECT COUNT(*) AS c FROM memories WHERE id LIKE 'vb-%' AND typeof(content_vector)='text' AND content_vector != ''`).get().c + +ok('seed: legacy rows are typeof text', Object.keys(legacy).every(id => kindOf(id) === 'text')) +ok('seed: pre-encoded row is typeof blob', kindOf('vb-blob') === 'blob') +ok('seed: 6 legacy text rows pending (5 vectors + 1 junk)', legacyLeft() === 6) + +// Reset the completion marker so this test observes the migration itself, not +// the marker initMemory() may have set on the empty table at startup. +db.pragma('user_version = 0') + +// Bounded: limit=2 scans exactly 2; with count:true it also reports the rest. +const r1 = migrateVectorsToBlob({ db, limit: 2, count: true }) +ok('limit=2 scans exactly 2', r1.scanned === 2 && r1.converted + r1.skipped === 2) +ok('limit=2 is not drained and counts 4 remaining', r1.drained === false && r1.remaining === 4 && legacyLeft() === 4) +ok('completion marker NOT set while rows remain', db.pragma('user_version', { simple: true }) === 0) + +// A cut-short run without count:true reports remaining=null — it must not COUNT(*). +const r1b = migrateVectorsToBlob({ db, limit: 1 }) +ok('bounded run without count reports remaining=null, not a full-scan count', r1b.scanned === 1 && r1b.drained === false && r1b.remaining === null && legacyLeft() === 3) + +// budgetMs=0: no batch may start, nothing changes. +const r0 = migrateVectorsToBlob({ db, budgetMs: 0 }) +ok('budgetMs=0 scans nothing', r0.scanned === 0 && r0.drained === false && r0.remaining === null && legacyLeft() === 3) + +// Resume to completion. +const r2 = migrateVectorsToBlob({ db }) +ok('resume scans the remaining 3', r2.scanned === 3) +ok('across all runs: 5 vectors converted, 1 unparseable cleared', + r1.converted + r1b.converted + r2.converted === 5 && r1.skipped + r1b.skipped + r2.skipped === 1) +ok('drained, remaining 0 after completion', r2.drained === true && r2.remaining === 0 && legacyLeft() === 0) +ok('completion marker set (user_version = 13)', db.pragma('user_version', { simple: true }) === 13) + +ok('all legacy rows are now typeof blob', Object.keys(legacy).every(id => kindOf(id) === 'blob')) +ok('converted values are bit-identical to the originals', + Object.entries(legacy).every(([id, vec]) => { const d = vecOf(id); return d && d.every((x, i) => x === Math.fround(vec[i])) })) +ok('pre-encoded blob row untouched', kindOf('vb-blob') === 'blob' && near(vecOf('vb-blob'), [0, 0, 1])) +ok('junk text row became NULL (re-embeddable, no longer counted as a vector)', kindOf('vb-junk') === 'null') +ok("'' row left as-is (already excluded by != '' everywhere)", kindOf('vb-empty') === 'text') +ok('NULL row left as-is', kindOf('vb-null') === 'null') + +// Idempotent. +const r3 = migrateVectorsToBlob({ db }) +ok('second full run is a no-op', r3.scanned === 0 && r3.converted === 0 && r3.skipped === 0 && r3.drained === true && r3.remaining === 0) + +// The existing coverage predicate must still count BLOB rows as vectorised. +const covered = db.prepare(`SELECT COUNT(*) AS c FROM memories WHERE id LIKE 'vb-%' AND content_vector IS NOT NULL AND content_vector != ''`).get().c +ok("IS NOT NULL AND != '' counts the 6 BLOB rows and nothing else", covered === 6) + +// The completion marker is one-way and honoured only when asked (skipIfComplete): +// a legacy row appearing after completion is left alone by the server's loop and +// converted by an explicit run — exactly what README documents for operators. +ins.run('vb-late', 'content vb-late', 'summary vb-late', 'skill', 'semi_abstract', 6, JSON.stringify([0.5, 0.5, 0]), now, now, now) +const r4 = migrateVectorsToBlob({ db, skipIfComplete: true }) +ok('skipIfComplete with marker set → drained immediately, nothing scanned, late row untouched', + r4.scanned === 0 && r4.drained === true && r4.remaining === 0 && kindOf('vb-late') === 'text') +const r5 = migrateVectorsToBlob({ db }) +ok('explicit run ignores the marker and converts the late row', r5.converted === 1 && kindOf('vb-late') === 'blob') + +// ── 3. A pre-existing reader sees both formats as one ─────────────────────── +// memory-health's near-dup scan decodes whatever is in the column. Seed one +// legacy JSON row and one BLOB row that are near-identical and check the pair +// is found across the format boundary. +db.prepare(`DELETE FROM memories WHERE id LIKE 'vb-%'`).run() +db.pragma('user_version = 0') +// 'preference' is a real category (the column has a CHECK). On a scratch DB it +// holds only these two rows; on a shared DB it may not, so the count assertion +// tightens to exactly-one only when the bucket started empty. +const CROSS_CAT = 'preference' +const preexisting = db.prepare(`SELECT COUNT(*) AS c FROM memories WHERE category = ? AND deleted_at IS NULL`).get(CROSS_CAT).c +ins.run('vb-x-json', 'content x json', 'summary x json', CROSS_CAT, 'semi_abstract', 6, JSON.stringify([1, 0, 0]), now - 9 * 86400_000, now, now) +ins.run('vb-x-blob', 'content x blob', 'summary x blob', CROSS_CAT, 'semi_abstract', 6, encodeVector([0.9995, 0.0316, 0]), now - 8 * 86400_000, now, now) +{ + const nd = detectNearDup(db, { simFloor: 0.95, simDup: 0.97, simSupersede: 0.85 }) + const inCat = (list, cat) => (list || []).filter(c => c.cat === cat) + const dups = inCat(nd.dup_candidates, CROSS_CAT).length + // cos([1,0,0], [0.9995,0.0316,0]) = 0.9995 ≥ simDup. + ok('memory-health dup scan pairs a legacy JSON row with a BLOB row (dual-format read)', + preexisting === 0 ? dups === 1 : dups >= 1) + ok('…and does not also file that pair as a supersede candidate', + preexisting === 0 ? inCat(nd.supersede_candidates, CROSS_CAT).length === 0 : true) +} + +// ── 4. The write-gate's ranker reads both formats too ──────────────────────── +// findNearDuplicates = sqlite-vec shortlist + rankNearDuplicateCandidates. The +// shortlist needs the vec extension (absent in CI); the ranker is pure and is +// where the decode happens, so it is tested directly with mixed-format rows. +{ + const cands = [ + { id: 11, content_vector: JSON.stringify([1, 0, 0]), summary: 'json twin' }, + { id: 12, content_vector: encodeVector([0.9995, 0.0316, 0]), summary: 'blob near-twin' }, + { id: 13, content_vector: 'junk', summary: 'undecodable' }, + { id: 14, content_vector: encodeVector([0, 1, 0]), summary: 'orthogonal' }, + ] + const ranked = rankNearDuplicateCandidates([1, 0, 0], cands, { threshold: 0.92 }) + ok('ranker returns the JSON row and the BLOB row, best first', ranked.length === 2 && ranked[0].id === 11 && ranked[1].id === 12) + ok('ranker skips undecodable and below-threshold rows', !ranked.some(r => r.id === 13 || r.id === 14)) + ok('ranker cosines are exact (1.0 for the identical JSON row, 0.9995 for the BLOB twin)', ranked[0].cosine === 1 && ranked[1].cosine === 0.9995) +} + +// cleanup +db.prepare(`DELETE FROM memories WHERE id LIKE 'vb-%'`).run() +db.close() +closeMemory() + +console.log(`\n${fail === 0 ? 'PASS' : 'FAIL'}: ${pass} passed / ${fail} failed`) +process.exit(fail === 0 ? 0 : 1) diff --git a/vector-codec.mjs b/vector-codec.mjs new file mode 100644 index 0000000..473eded --- /dev/null +++ b/vector-codec.mjs @@ -0,0 +1,77 @@ +// vector-codec.mjs — storage codec for memories.content_vector +// +// Storage format is Float32, one 4-byte lane per dimension, in the host's +// native byte order (Float32Array; not an explicitly serialised endianness). +// Every supported target — x64, arm64 — is little-endian, and sqlite-vec's +// own index makes the identical assumption, so a DB file carried to a +// big-endian host would mis-decode both columns alike. Held as a BLOB in +// the existing TEXT-affinity column. SQLite never coerces +// BLOB values on the way in, so `typeof(content_vector)` reliably reports +// 'blob' for encoded rows and 'text' for legacy JSON rows — that is what the +// migration keys on, and it needs no schema change. +// +// Why Float32 instead of the JSON array this column used to hold: measured on +// a 10,920-row library (2026-09-03), the JSON form averaged 21,723 bytes per +// 1024-dim vector against 4,096 for Float32 — 226 MB against 43 MB. It is +// lossless, not approximate: every one of 2,048,000 sampled stored values +// satisfied Math.fround(x) === x. The embedding API already returns float32; +// JSON was spending 5x the bytes to spell the same 32-bit values out in +// decimal. sqlite-vec's KNN index is Float32 as well, so recall has always +// run at this precision. +// +// decodeVector accepts both formats so no reader needs to know which era a +// row is from, and the migration can be interrupted anywhere and resumed. + +/** + * @param {number[]|Float32Array|null|undefined} vec + * @returns {Buffer|null} little-endian Float32 bytes, or null for empty input + */ +export function encodeVector(vec) { + if (!vec || typeof vec.length !== 'number' || vec.length === 0) return null + const f32 = vec instanceof Float32Array ? vec : Float32Array.from(vec) + // Symmetric with decodeVector: a NaN, an Infinity, or a double beyond + // float32 range (which Float32Array.from silently turns into ±Infinity) + // must not be written. Stored, it would count as "covered" for every + // `!= ''` check while decoding to null forever — a vector no sweep repairs. + if (!f32.every(Number.isFinite)) return null + // Copy out of any shared ArrayBuffer so the stored bytes cannot alias a + // caller's typed array that is later mutated. + return Buffer.from(f32.buffer.slice(f32.byteOffset, f32.byteOffset + f32.byteLength)) +} + +/** + * @param {Buffer|Uint8Array|string|null|undefined} stored + * @returns {number[]|null} plain array of finite numbers, or null if unusable + */ +export function decodeVector(stored) { + if (stored == null) return null + if (stored instanceof Uint8Array) { // Buffer is a Uint8Array subclass + if (stored.byteLength === 0 || stored.byteLength % 4 !== 0) return null + // A Buffer handed back by the driver may sit at an unaligned offset inside + // a slab; Float32Array views require 4-byte alignment, so copy first. + const aligned = new Uint8Array(stored.byteLength) + aligned.set(stored) + const out = Array.from(new Float32Array(aligned.buffer)) + return out.every(Number.isFinite) ? out : null + } + if (typeof stored === 'string') { + const s = stored.trim() + if (!s.startsWith('[')) return null + try { + const v = JSON.parse(s) + return Array.isArray(v) && v.length > 0 && v.every(x => typeof x === 'number' && Number.isFinite(x)) ? v : null + } catch { return null } + } + return null +} + +/** + * Classify a stored value without decoding it. Used for reporting and tests. + * @returns {'blob'|'json'|'empty'|'unknown'} + */ +export function vectorStorageKind(stored) { + if (stored == null || stored === '') return 'empty' + if (stored instanceof Uint8Array) return 'blob' + if (typeof stored === 'string') return 'json' + return 'unknown' +}