From 200058937f9ad9bbcfba5474300eed2add087982 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste THERY Date: Sat, 4 Jul 2026 14:55:15 +0700 Subject: [PATCH] feat(core): detect stale index and surface vector index warnings (#51) * feat(core): detect stale index and surface vector index warnings Audit of all RAG library usage confirmed every dependency ( @huggingface/transformers, @lancedb/lancedb, @modelcontextprotocol/sdk, zod, unpdf, mammoth, fast-glob ) is used according to its current API. No library needed replacement or upgrade. Three quality and DX gaps fixed: - store.ts no longer swallows createIndex errors silently. writeRows now returns vectorIndexWarning, propagated into IngestResult and printed by the ingest CLI, so a fallback to flat scan is visible to the operator. - A new index-manifest.json (schemaVersion, embedding provider/model, chunk size/overlap, ragmir version) is written next to the LanceDB table at each ingest. getIndexFreshnessWarning detects a stale or incompatible index cheaply. ask() returns staleWarning, doctor reports indexFreshness with a rebuild next step, and the search/ask/doctor CLI surfaces the warning on stderr without breaking JSON output. - The hybrid lexical scan limit is now configurable (hybridTextScanLimit, default 5000, RAGMIR_HYBRID_TEXT_SCAN_LIMIT env). getLexicalScanWarning flags when the corpus exceeds it so BM25 recall loss is no longer silent. Also includes repository metadata and docs updates: npm package metadata in the root package.json, sponsor badge and issue reporting guidance in the README/CONTRIBUTING, a feature-request issue template, and conventional- commit/branch checkboxes in the PR template. * fix(doctor): drop manual backtick escaping flagged by CodeQL The freshness warning comes from controlled getIndexFreshnessWarning output, not user input, so the manual backtick escaping was unnecessary and incomplete (did not escape backslashes). CodeQL flagged it as an incomplete string-escaping pattern; removing it resolves the alert. --- .github/ISSUE_TEMPLATE/feature_request.yml | 57 +++++++++ .github/PULL_REQUEST_TEMPLATE.md | 7 +- CONTRIBUTING.md | 17 +++ README.md | 1 + package.json | 27 ++++ packages/ragmir-core/src/cli.ts | 27 ++++ packages/ragmir-core/src/config.test.ts | 21 ++++ packages/ragmir-core/src/config.ts | 7 ++ packages/ragmir-core/src/defaults.ts | 1 + packages/ragmir-core/src/doctor.test.ts | 2 + packages/ragmir-core/src/doctor.ts | 40 ++++-- .../ragmir-core/src/index-diagnostics.test.ts | 118 ++++++++++++++++++ packages/ragmir-core/src/index-diagnostics.ts | 58 +++++++++ packages/ragmir-core/src/index.ts | 6 + packages/ragmir-core/src/ingest.test.ts | 18 +++ packages/ragmir-core/src/ingest.ts | 22 +++- packages/ragmir-core/src/query.test.ts | 1 + packages/ragmir-core/src/query.ts | 7 +- packages/ragmir-core/src/store.test.ts | 76 ++++++++++- packages/ragmir-core/src/store.ts | 88 +++++++++++-- .../ragmir-core/src/test-support/config.ts | 1 + packages/ragmir-core/src/types.ts | 25 ++++ 22 files changed, 606 insertions(+), 21 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yml create mode 100644 packages/ragmir-core/src/index-diagnostics.test.ts create mode 100644 packages/ragmir-core/src/index-diagnostics.ts diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..81cee5c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,57 @@ +name: Feature request +description: Suggest a new Ragmir feature or improvement +title: "feat: " +labels: ["enhancement"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to suggest a feature. Ragmir focuses on + sovereign local RAG for confidential datasets and AI agents. Features + that add cloud dependencies, telemetry, or hosted services are out of + scope. + - type: textarea + id: problem + attributes: + label: Problem + description: What problem does this feature solve? What are you trying to do that Ragmir cannot do today? + validations: + required: true + - type: textarea + id: solution + attributes: + label: Proposed solution + description: Describe the feature or change you would like to see. + validations: + required: true + - type: textarea + id: alternatives + attributes: + label: Alternatives considered + description: Have you tried any workarounds or existing tools? + validations: + required: false + - type: dropdown + id: area + attributes: + label: Area + description: Which part of Ragmir does this concern? + options: + - Ingestion (parsing, chunking, redaction) + - Retrieval (search, ask, research, evaluate) + - MCP server / agent integration + - CLI + - Desktop app + - Skills (audio, report, legal) + - Documentation + - Other + validations: + required: true + - type: checkboxes + id: sovereign + attributes: + label: Sovereignty check + description: Confirm this feature is compatible with Ragmir's local-first, zero-telemetry posture. + options: + - label: This feature does not require sending documents, queries, or telemetry to a hosted service. + required: true diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index b908830..fe74b91 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -8,7 +8,12 @@ - [ ] `pnpm test` - [ ] `pnpm build` +## Commits + +- [ ] Conventional Commits format (`feat:`, `fix:`, `docs:`, `chore:`, `test:`...). +- [ ] Branch follows Git Flow (`feature/*`, `fix/*`, `chore/*` from `develop`). + ## Security -- [ ] No secrets, private documents, `.env` files, generated `.kb/` or generated `.ragmir/` state are included. +- [ ] No secrets, private documents, `.env` files, or generated `.ragmir/` state are included. - [ ] Public documentation does not expose private project, customer, pricing, or validation details. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6e57fac..e0ce3ee 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,6 +2,23 @@ Ragmir is an open-source project under the MIT License. Issues and pull requests are welcome. +Ragmir is maintained by a single developer ([Jean-Baptiste Thery](https://github.com/jb-thery)). +Be kind, be specific, and keep the scope of each contribution focused so it can be reviewed +within a reasonable time. + +## Reporting Issues + +Use the [issue templates](https://github.com/jcode-works/jcode-ragmir/issues/new/choose): + +- **Bug report**: include the Ragmir version, a minimal reproduction (commands + sample files, + no private documents or secrets), and the expected behavior. +- **Feature request**: describe the problem you are trying to solve and the proposed solution. + Confirm the feature stays compatible with Ragmir's local-first, zero-telemetry posture. + +Before opening a new issue, search [existing issues](https://github.com/jcode-works/jcode-ragmir/issues) +to avoid duplicates. Security vulnerabilities must not be reported through public issues — follow +[`SECURITY.md`](./SECURITY.md). + ## Development This repo pins its Node.js and Rust versions with [mise](https://mise.jdx.dev/) (see `mise.toml`), diff --git a/README.md b/README.md index 92b420b..d3cf160 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ [![npm](https://img.shields.io/npm/v/@jcode.labs/ragmir)](https://www.npmjs.com/package/@jcode.labs/ragmir) [![npm downloads](https://img.shields.io/npm/dm/@jcode.labs/ragmir?label=downloads%2Fmonth)](https://www.npmjs.com/package/@jcode.labs/ragmir) [![license: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/jcode-works/jcode-ragmir/blob/main/LICENSE) +[![GitHub Sponsors](https://img.shields.io/badge/sponsor-JCode%20Labs-ea4aaa.svg)](https://github.com/sponsors/jb-thery) Open-source local RAG library, CLI, and MCP server. Ragmir indexes your specs, docs, and code locally and gives your AI agents only the useful cited passages, over MCP, without burning tokens on diff --git a/package.json b/package.json index e3f9d87..b211a45 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,33 @@ "description": "Monorepo for Ragmir Core and open-source Ragmir add-ons.", "type": "module", "license": "MIT", + "author": { + "name": "Jean-Baptiste Thery", + "url": "https://github.com/jb-thery" + }, + "homepage": "https://github.com/jcode-works/jcode-ragmir#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/jcode-works/jcode-ragmir.git" + }, + "bugs": { + "url": "https://github.com/jcode-works/jcode-ragmir/issues" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/jb-thery" + }, + "keywords": [ + "ragmir", + "rag", + "local-rag", + "sovereign-rag", + "mcp", + "ai-agents", + "private-ai", + "local-first", + "knowledge-base" + ], "packageManager": "pnpm@11.9.0", "scripts": { "bootstrap": "mise install && pnpm install", diff --git a/packages/ragmir-core/src/cli.ts b/packages/ragmir-core/src/cli.ts index e215725..4a293e2 100644 --- a/packages/ragmir-core/src/cli.ts +++ b/packages/ragmir-core/src/cli.ts @@ -21,6 +21,7 @@ import { doctor } from "./doctor.js" import { pullEmbeddingModel } from "./embeddings.js" import { evaluateGoldenQueries } from "./evaluate.js" import { countSkippedByReason } from "./files.js" +import { getIndexFreshnessWarning, getLexicalScanWarning } from "./index-diagnostics.js" import { audit, ingest } from "./ingest.js" import { initProject } from "./init.js" import { serveMcp } from "./mcp.js" @@ -279,6 +280,9 @@ program ) printUnsupportedSummary(result.unsupportedExtensions) printEmptyTextFiles(result.emptyTextFiles) + if (result.vectorIndexWarning) { + console.log(pc.yellow(result.vectorIndexWarning)) + } if (result.unsupportedFiles > 0 || result.oversizedFiles > 0 || result.sensitiveFiles > 0) { const auditCommand = await ragmirCommand(cwd, ["audit", "--unsupported"]) console.log( @@ -324,6 +328,8 @@ program return } + await printStaleIndexWarnings(cwd) + for (const [index, result] of outputResults.entries()) { const distance = result.distance === null ? "n/a" : result.distance.toFixed(4) console.log( @@ -354,6 +360,9 @@ program } console.log(`\n${result.answer}\n`) + if (result.staleWarning) { + console.error(pc.yellow(result.staleWarning)) + } if (result.sources.length > 0) { console.log(pc.dim("Sources:")) for (const [index, source] of result.sources.entries()) { @@ -938,6 +947,20 @@ function isTtsModule(value: unknown): value is TtsModule { ) } +async function printStaleIndexWarnings(cwd: string): Promise { + const config = await loadConfig(cwd) + const freshnessWarning = await getIndexFreshnessWarning(config) + if (freshnessWarning) { + console.error(pc.yellow(freshnessWarning)) + return + } + const chunkCount = await countRows(config) + const lexicalScanWarning = getLexicalScanWarning(config, chunkCount) + if (lexicalScanWarning) { + console.error(pc.yellow(lexicalScanWarning)) + } +} + function printDoctor(report: Awaited>): void { console.log(`projectRoot=${report.projectRoot}`) console.log(`initialized=${report.initialized}`) @@ -962,6 +985,10 @@ function printDoctor(report: Awaited>): void { console.log(pc.yellow(`warning: ${warning}`)) } } + console.log(`indexFreshness.manifestFound=${report.indexFreshness.manifestFound}`) + if (report.indexFreshness.warning) { + console.log(pc.yellow(`indexFreshness: ${report.indexFreshness.warning}`)) + } console.log("nextSteps:") for (const step of report.nextSteps) { console.log(` - ${step}`) diff --git a/packages/ragmir-core/src/config.test.ts b/packages/ragmir-core/src/config.test.ts index 45204aa..96580bb 100644 --- a/packages/ragmir-core/src/config.test.ts +++ b/packages/ragmir-core/src/config.test.ts @@ -304,6 +304,27 @@ describe("loadConfig", () => { } }) + it("defaults hybridTextScanLimit and overrides it from env", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "jcode-kb-scan-limit-")) + tempDirs.push(root) + await mkdir(path.join(root, ".ragmir"), { recursive: true }) + await writeFile(path.join(root, ".ragmir/config.json"), "{}\n", "utf8") + + expect((await loadConfig(root)).hybridTextScanLimit).toBe(5000) + + const original = process.env.RAGMIR_HYBRID_TEXT_SCAN_LIMIT + process.env.RAGMIR_HYBRID_TEXT_SCAN_LIMIT = "2000" + try { + expect((await loadConfig(root)).hybridTextScanLimit).toBe(2000) + } finally { + if (original === undefined) { + delete process.env.RAGMIR_HYBRID_TEXT_SCAN_LIMIT + } else { + process.env.RAGMIR_HYBRID_TEXT_SCAN_LIMIT = original + } + } + }) + it("rejects unknown config keys so typos surface instead of being ignored", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "jcode-kb-strict-")) tempDirs.push(root) diff --git a/packages/ragmir-core/src/config.ts b/packages/ragmir-core/src/config.ts index 90c2e81..ba5cdbd 100644 --- a/packages/ragmir-core/src/config.ts +++ b/packages/ragmir-core/src/config.ts @@ -52,6 +52,7 @@ const rawConfigSchema = z maxFileBytes: z.number().int().positive().default(DEFAULT_CONFIG.maxFileBytes), ingestConcurrency: z.number().int().positive().default(DEFAULT_CONFIG.ingestConcurrency), embeddingBatchSize: z.number().int().positive().default(DEFAULT_CONFIG.embeddingBatchSize), + hybridTextScanLimit: z.number().int().positive().default(DEFAULT_CONFIG.hybridTextScanLimit), includeExtensions: z.array(z.string().min(1)).default(DEFAULT_CONFIG.includeExtensions), pdfOcrCommand: z.array(z.string().min(1)).default(DEFAULT_CONFIG.pdfOcrCommand), pdfOcrTimeoutMs: z.number().int().positive().default(DEFAULT_CONFIG.pdfOcrTimeoutMs), @@ -146,6 +147,7 @@ export async function loadConfig(start = process.cwd()): Promise { maxFileBytes: withEnv.maxFileBytes, ingestConcurrency: withEnv.ingestConcurrency, embeddingBatchSize: withEnv.embeddingBatchSize, + hybridTextScanLimit: withEnv.hybridTextScanLimit, includeExtensions: normalizeExtensions(withEnv.includeExtensions), pdfOcrCommand: withEnv.pdfOcrCommand, pdfOcrTimeoutMs: withEnv.pdfOcrTimeoutMs, @@ -228,6 +230,11 @@ function applyEnv(config: RawConfig): RawConfig { "KB_EMBEDDING_BATCH_SIZE", config.embeddingBatchSize, ), + hybridTextScanLimit: readPositiveIntEnv( + "RAGMIR_HYBRID_TEXT_SCAN_LIMIT", + "KB_HYBRID_TEXT_SCAN_LIMIT", + config.hybridTextScanLimit, + ), includeExtensions: readExtensionsEnv( "RAGMIR_INCLUDE_EXTENSIONS", "KB_INCLUDE_EXTENSIONS", diff --git a/packages/ragmir-core/src/defaults.ts b/packages/ragmir-core/src/defaults.ts index 7046620..c01c8b5 100644 --- a/packages/ragmir-core/src/defaults.ts +++ b/packages/ragmir-core/src/defaults.ts @@ -39,6 +39,7 @@ export const DEFAULT_CONFIG: Omit = { maxFileBytes: 50_000_000, ingestConcurrency: 4, embeddingBatchSize: 32, + hybridTextScanLimit: 5_000, includeExtensions: [], pdfOcrCommand: [], pdfOcrTimeoutMs: 120_000, diff --git a/packages/ragmir-core/src/doctor.test.ts b/packages/ragmir-core/src/doctor.test.ts index cff65f4..dfddfea 100644 --- a/packages/ragmir-core/src/doctor.test.ts +++ b/packages/ragmir-core/src/doctor.test.ts @@ -59,6 +59,8 @@ describe("doctor", () => { expect(ready.nextSteps).toContain( "For natural-language Q&A, run `pnpm exec ragmir models pull --enable`, then run `pnpm exec ragmir ingest --rebuild`.", ) + expect(ready.indexFreshness.manifestFound).toBe(true) + expect(ready.indexFreshness.warning).toBeNull() }) it("detects an installed agent kit from the files installSkill writes", async () => { diff --git a/packages/ragmir-core/src/doctor.ts b/packages/ragmir-core/src/doctor.ts index 53d630f..6527020 100644 --- a/packages/ragmir-core/src/doctor.ts +++ b/packages/ragmir-core/src/doctor.ts @@ -3,6 +3,7 @@ import path from "node:path" import { findProjectConfig, loadConfig } from "./config.js" import { RAGMIR_DIR } from "./defaults.js" import { countSkippedByReason } from "./files.js" +import { getIndexFreshnessWarning, getLexicalScanWarning } from "./index-diagnostics.js" import { audit } from "./ingest.js" import { ragmirCommand } from "./package-manager.js" import { securityAudit } from "./security.js" @@ -12,7 +13,7 @@ import { MCP_CONFIG_FILENAME, SKILL_NAMES, } from "./skill.js" -import { countRows } from "./store.js" +import { countRows, readIndexManifest } from "./store.js" import type { DoctorReport } from "./types.js" export async function doctor(cwd = process.cwd()): Promise { @@ -21,11 +22,20 @@ export async function doctor(cwd = process.cwd()): Promise { const config = await loadConfig(cwd) const command = await ragmirCommand(config.projectRoot, []) const agentKitInstalled = isAgentKitInstalled(config.projectRoot) - const [auditReport, securityReport, chunksIndexed] = await Promise.all([ - audit(config.projectRoot), - securityAudit(config.projectRoot), - countRows(config), - ]) + const [auditReport, securityReport, chunksIndexed, manifest, freshnessWarning] = + await Promise.all([ + audit(config.projectRoot), + securityAudit(config.projectRoot), + countRows(config), + readIndexManifest(config), + getIndexFreshnessWarning(config), + ]) + + const lexicalScanWarning = chunksIndexed > 0 ? getLexicalScanWarning(config, chunksIndexed) : null + const indexFreshness = { + manifestFound: manifest !== null, + warning: freshnessWarning, + } const nextSteps = nextActions({ initialized, @@ -38,6 +48,8 @@ export async function doctor(cwd = process.cwd()): Promise { warnings: securityReport.warnings.length, embeddingProvider: config.embeddingProvider, agentKitInstalled, + freshnessWarning, + lexicalScanWarning, run: (args) => command.display + (args.length > 0 ? ` ${args.join(" ")}` : ""), }) @@ -61,12 +73,14 @@ export async function doctor(cwd = process.cwd()): Promise { missingFromIndex: auditReport.missingFromIndex.length, staleInIndex: auditReport.staleInIndex.length, securityWarnings: securityReport.warnings, + indexFreshness, ready: initialized && chunksIndexed > 0 && auditReport.missingFromIndex.length === 0 && auditReport.staleInIndex.length === 0 && - securityReport.warnings.length === 0, + securityReport.warnings.length === 0 && + freshnessWarning === null, nextSteps, } } @@ -82,6 +96,8 @@ interface NextActionInput { warnings: number embeddingProvider: string agentKitInstalled: boolean + freshnessWarning: string | null + lexicalScanWarning: string | null run: (args: string[]) => string } @@ -114,6 +130,16 @@ function nextActions(input: NextActionInput): string[] { ) } + if (input.freshnessWarning) { + steps.push( + `${input.freshnessWarning} Run \`${input.run(["ingest", "--rebuild"])}\` to align the index with the active configuration.`, + ) + } + + if (input.lexicalScanWarning) { + steps.push(input.lexicalScanWarning) + } + if (input.warnings > 0) { steps.push( `Run \`${input.run(["security-audit", "--strict"])}\` and fix the reported warnings.`, diff --git a/packages/ragmir-core/src/index-diagnostics.test.ts b/packages/ragmir-core/src/index-diagnostics.test.ts new file mode 100644 index 0000000..7f30d7d --- /dev/null +++ b/packages/ragmir-core/src/index-diagnostics.test.ts @@ -0,0 +1,118 @@ +import { mkdtemp, rm } from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { afterEach, describe, expect, it } from "vitest" +import { + getIndexFreshnessWarning, + getLexicalScanWarning, + INDEX_SCHEMA_VERSION, +} from "./index-diagnostics.js" +import { writeIndexManifest } from "./store.js" +import { testConfig } from "./test-support/config.js" +import type { IndexManifest } from "./types.js" + +const tempDirs: string[] = [] + +afterEach(async () => { + for (const dir of tempDirs.splice(0)) { + await rm(dir, { recursive: true, force: true }) + } +}) + +function baseManifest(overrides: Partial = {}): IndexManifest { + return { + schemaVersion: INDEX_SCHEMA_VERSION, + createdAt: "2026-01-01T00:00:00.000Z", + ragmirVersion: "0.4.12", + embeddingProvider: "local-hash", + embeddingModel: "mixedbread-ai/mxbai-embed-xsmall-v1", + chunkSize: 1200, + chunkOverlap: 200, + fileCount: 1, + chunkCount: 1, + ...overrides, + } +} + +describe("getIndexFreshnessWarning", () => { + it("returns null when the index manifest is absent", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "ragmir-freshness-missing-")) + tempDirs.push(root) + const config = testConfig(root) + + expect(await getIndexFreshnessWarning(config)).toBeNull() + }) + + it("returns null when the manifest matches the active config", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "ragmir-freshness-match-")) + tempDirs.push(root) + const config = testConfig(root) + await writeIndexManifest(baseManifest(), config) + + expect(await getIndexFreshnessWarning(config)).toBeNull() + }) + + it("warns when the embedding provider differs", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "ragmir-freshness-provider-")) + tempDirs.push(root) + const config = testConfig(root, { embeddingProvider: "transformers" }) + await writeIndexManifest(baseManifest({ embeddingProvider: "local-hash" }), config) + + const warning = await getIndexFreshnessWarning(config) + expect(warning).not.toBeNull() + expect(warning).toContain('"local-hash"') + expect(warning).toContain('"transformers"') + }) + + it("warns when the embedding model differs", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "ragmir-freshness-model-")) + tempDirs.push(root) + const config = testConfig(root, { embeddingModel: "Xenova/all-MiniLM-L6-v2" }) + await writeIndexManifest( + baseManifest({ embeddingModel: "mixedbread-ai/mxbai-embed-xsmall-v1" }), + config, + ) + + const warning = await getIndexFreshnessWarning(config) + expect(warning).not.toBeNull() + expect(warning).toContain("Xenova/all-MiniLM-L6-v2") + }) + + it("warns when the stored schema version is older than the current one", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "ragmir-freshness-schema-")) + tempDirs.push(root) + const config = testConfig(root) + await writeIndexManifest(baseManifest({ schemaVersion: 0 }), config) + + const warning = await getIndexFreshnessWarning(config) + expect(warning).not.toBeNull() + expect(warning).toContain("schema is outdated") + }) + + it("warns when the chunk size differs", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "ragmir-freshness-chunksize-")) + tempDirs.push(root) + const config = testConfig(root, { chunkSize: 800 }) + await writeIndexManifest(baseManifest({ chunkSize: 1200 }), config) + + const warning = await getIndexFreshnessWarning(config) + expect(warning).not.toBeNull() + expect(warning).toContain("chunkSize") + }) +}) + +describe("getLexicalScanWarning", () => { + it("returns null when the chunk count is within the limit", () => { + const config = testConfig({ hybridTextScanLimit: 5000 }) + expect(getLexicalScanWarning(config, 5000)).toBeNull() + }) + + it("warns when the chunk count exceeds the limit", () => { + const config = testConfig({ hybridTextScanLimit: 5000 }) + const warning = getLexicalScanWarning(config, 8000) + expect(warning).not.toBeNull() + expect(warning).toContain("5000") + expect(warning).toContain("8000") + expect(warning).toContain("hybridTextScanLimit") + }) +}) diff --git a/packages/ragmir-core/src/index-diagnostics.ts b/packages/ragmir-core/src/index-diagnostics.ts new file mode 100644 index 0000000..cfbe738 --- /dev/null +++ b/packages/ragmir-core/src/index-diagnostics.ts @@ -0,0 +1,58 @@ +import { readIndexManifest } from "./store.js" +import type { Config } from "./types.js" + +/** + * Bumped when the index layout or manifest meaning changes in a way that + * invalidates previously indexed vectors (new columns, changed semantics, new + * required metadata). A stored manifest with a lower schemaVersion means the + * index predates the current code and should be rebuilt. + */ +export const INDEX_SCHEMA_VERSION = 1 + +/** + * Detect a stale or incompatible index without re-scanning every source file. + * Returns a human-readable warning when the stored manifest is missing, was + * built with a different embedding provider/model, predates the current index + * schema, or used different chunking settings. Returns `null` when the index is + * fresh, so callers can treat the warning as purely informational. + * + * The hybrid lexical scan limit is also surfaced here so callers know when BM25 + * retrieval is truncating a large corpus and losing recall. + */ +export async function getIndexFreshnessWarning(config: Config): Promise { + const manifest = await readIndexManifest(config) + + if (!manifest) { + return null + } + + if (manifest.schemaVersion < INDEX_SCHEMA_VERSION) { + return `Index schema is outdated (stored v${manifest.schemaVersion}, current v${INDEX_SCHEMA_VERSION}). Rebuild with \`ragmir ingest --rebuild\` to use the latest index format.` + } + + if (manifest.embeddingProvider !== config.embeddingProvider) { + return `Index was built with embedding provider "${manifest.embeddingProvider}" but the active config uses "${config.embeddingProvider}". Rebuild with \`ragmir ingest --rebuild\`.` + } + + if (manifest.embeddingModel !== config.embeddingModel) { + return `Index was built with embedding model "${manifest.embeddingModel}" but the active config uses "${config.embeddingModel}". Rebuild with \`ragmir ingest --rebuild\` to refresh vectors.` + } + + if (manifest.chunkSize !== config.chunkSize || manifest.chunkOverlap !== config.chunkOverlap) { + return `Index was built with chunkSize=${manifest.chunkSize}/chunkOverlap=${manifest.chunkOverlap} but the active config uses chunkSize=${config.chunkSize}/chunkOverlap=${config.chunkOverlap}. Rebuild with \`ragmir ingest --rebuild\`.` + } + + return null +} + +/** + * Warn when the indexed corpus exceeds the hybrid lexical scan limit, because + * BM25 retrieval then scans only the first N chunks and silently loses recall. + * Independent of model freshness so callers can surface both diagnostics. + */ +export function getLexicalScanWarning(config: Config, chunkCount: number): string | null { + if (chunkCount <= config.hybridTextScanLimit) { + return null + } + return `Lexical search scans only the first ${config.hybridTextScanLimit} of ${chunkCount} chunks. Raise \`hybridTextScanLimit\` in .ragmir/config.json for better keyword recall.` +} diff --git a/packages/ragmir-core/src/index.ts b/packages/ragmir-core/src/index.ts index adfdf94..e87d25b 100644 --- a/packages/ragmir-core/src/index.ts +++ b/packages/ragmir-core/src/index.ts @@ -4,6 +4,11 @@ export { destroyIndex } from "./destroy.js" export { doctor } from "./doctor.js" export { clearTransformersCache, pullEmbeddingModel } from "./embeddings.js" export { evaluateGoldenQueries } from "./evaluate.js" +export { + getIndexFreshnessWarning, + getLexicalScanWarning, + INDEX_SCHEMA_VERSION, +} from "./index-diagnostics.js" export { audit, ingest } from "./ingest.js" export { initProject } from "./init.js" export { serveMcp } from "./mcp.js" @@ -55,6 +60,7 @@ export type { EvaluationOptions, EvaluationResult, GoldenQuery, + IndexManifest, IngestResult, ResearchEvidence, ResearchOptions, diff --git a/packages/ragmir-core/src/ingest.test.ts b/packages/ragmir-core/src/ingest.test.ts index f1b8fbb..4d6bed3 100644 --- a/packages/ragmir-core/src/ingest.test.ts +++ b/packages/ragmir-core/src/ingest.test.ts @@ -2,8 +2,10 @@ import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" import os from "node:os" import path from "node:path" import { afterEach, describe, expect, it } from "vitest" +import { loadConfig } from "./config.js" import { audit, ingest } from "./ingest.js" import { initProject } from "./init.js" +import { readIndexManifest } from "./store.js" const tempDirs: string[] = [] @@ -66,6 +68,22 @@ describe("ingest", () => { expect(second.reusedFiles).toBe(1) }) + it("writes an index manifest after ingest and reports a null vectorIndexWarning", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "ragmir-manifest-write-")) + tempDirs.push(root) + await initProject(root) + await mkdir(path.join(root, ".ragmir", "raw"), { recursive: true }) + await writeFile(path.join(root, ".ragmir", "raw", "alpha.md"), "Alpha evidence.\n", "utf8") + + const result = await ingest({ cwd: root }) + + expect(result.vectorIndexWarning).toBeNull() + const manifest = await readIndexManifest(await loadConfig(root)) + expect(manifest).not.toBeNull() + expect(manifest?.embeddingProvider).toBe("local-hash") + expect(manifest?.chunkCount).toBe(result.chunks) + }) + it("forces a full re-index of every file when rebuild is requested", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "ragmir-rebuild-")) tempDirs.push(root) diff --git a/packages/ragmir-core/src/ingest.ts b/packages/ragmir-core/src/ingest.ts index deab75d..54cbc08 100644 --- a/packages/ragmir-core/src/ingest.ts +++ b/packages/ragmir-core/src/ingest.ts @@ -8,6 +8,7 @@ import { inventorySourceFiles, summarizeUnsupportedExtensions, } from "./files.js" +import { INDEX_SCHEMA_VERSION } from "./index-diagnostics.js" import { parseFile } from "./parsing.js" import { redactText, totalRedactions } from "./redaction.js" import { @@ -15,6 +16,7 @@ import { readEmptyTextFiles, readRows, writeEmptyTextFiles, + writeIndexManifest, writeRows, } from "./store.js" import { normalizeForMatch } from "./text.js" @@ -28,6 +30,7 @@ import type { TextChunk, VectorRow, } from "./types.js" +import { VERSION } from "./version.js" const MAX_AUDIT_ROWS = 100_000 const MAX_SOURCE_DIAGNOSTIC_ITEMS = 20 @@ -119,7 +122,23 @@ export async function ingest(options: IngestOptions = {}): Promise } const indexRows = [...reusableRows, ...rows] - await writeRows(indexRows, config) + const writeResult = await writeRows(indexRows, config) + if (indexRows.length > 0) { + await writeIndexManifest( + { + schemaVersion: INDEX_SCHEMA_VERSION, + createdAt: new Date().toISOString(), + ragmirVersion: VERSION, + embeddingProvider: config.embeddingProvider, + embeddingModel: config.embeddingModel, + chunkSize: config.chunkSize, + chunkOverlap: config.chunkOverlap, + fileCount: new Set(indexRows.map((row) => row.relativePath)).size, + chunkCount: indexRows.length, + }, + config, + ) + } await writeEmptyTextFiles( emptyTextFiles.flatMap((relativePath) => { const file = currentFiles.get(relativePath) @@ -147,6 +166,7 @@ export async function ingest(options: IngestOptions = {}): Promise emptyTextFiles, unsupportedExtensions: summarizeUnsupportedExtensions(inventory.skippedFiles), redactions: totalRedactions(redactionCounts), + vectorIndexWarning: writeResult.vectorIndexWarning, errors, } } diff --git a/packages/ragmir-core/src/query.test.ts b/packages/ragmir-core/src/query.test.ts index bea0e89..419d9e6 100644 --- a/packages/ragmir-core/src/query.test.ts +++ b/packages/ragmir-core/src/query.test.ts @@ -101,6 +101,7 @@ describe("ask", () => { expect(result.answer).toContain("retrieval context only") expect(result.answer).toContain("[1]") expect(result.answer).toContain("policy.md#0") + expect(result.staleWarning).toBeNull() }) }) diff --git a/packages/ragmir-core/src/query.ts b/packages/ragmir-core/src/query.ts index 010eece..aa4de03 100644 --- a/packages/ragmir-core/src/query.ts +++ b/packages/ragmir-core/src/query.ts @@ -1,6 +1,7 @@ import { recordAccess } from "./access-log.js" import { loadConfig } from "./config.js" import { embedText } from "./embeddings.js" +import { getIndexFreshnessWarning } from "./index-diagnostics.js" import { openRowsTable } from "./store.js" import { tokenize } from "./text.js" import type { AskResult, SearchOptions, SearchResult } from "./types.js" @@ -22,7 +23,6 @@ interface RankedRow { const MIN_VECTOR_CANDIDATES = 80 const VECTOR_CANDIDATE_MULTIPLIER = 4 -const HYBRID_TEXT_SCAN_LIMIT = 5_000 /** * Reciprocal Rank Fusion (Cormack et al. 2009). Each candidate scores * `weight / (RRF_K + rank)` per retriever it appears in, summed across @@ -55,7 +55,7 @@ export async function search(query: string, options: SearchOptions = {}): Promis .vectorSearch(vector) .limit(vectorCandidateLimit(topK)) .toArray()) as SearchRow[] - const textRows = (await table.query().limit(HYBRID_TEXT_SCAN_LIMIT).toArray()) as SearchRow[] + const textRows = (await table.query().limit(config.hybridTextScanLimit).toArray()) as SearchRow[] const rows = rankHybridRows(query, vectorRows, textRows).slice(0, topK) const results = rows.map((row) => ({ @@ -81,11 +81,13 @@ export function vectorCandidateLimit(topK: number): number { export async function ask(query: string, options: SearchOptions = {}): Promise { const config = await loadConfig(String(options.cwd ?? process.cwd())) const sources = await search(query, options) + const staleWarning = await getIndexFreshnessWarning(config) if (sources.length === 0) { return { answer: "No relevant passages were found. Add documents and run `ragmir doctor --fix` first.", sources, + staleWarning, } } @@ -99,6 +101,7 @@ export async function ask(query: string, options: SearchOptions = {}): Promise { const rows = await readRows(config) expect(rows.map((row) => row.relativePath)).toEqual([".ragmir/raw/b.md", ".ragmir/raw/b.md"]) }) + + it("returns a null vectorIndexWarning on a nominal small write (flat scan)", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "ragmir-store-warning-")) + tempDirs.push(root) + const config = testConfig(root) + + const result = await writeRows([sampleRow(".ragmir/raw/a.md", 0, [0.1, 0.2], config)], config) + expect(result.vectorIndexWarning).toBeNull() + }) }) describe("empty-text-files manifest", () => { @@ -158,3 +176,59 @@ describe("empty-text-files manifest", () => { expect(records[0]?.relativePath).toBe("good.pdf") }) }) + +describe("index manifest", () => { + const sampleManifest: IndexManifest = { + schemaVersion: 1, + createdAt: "2026-01-01T00:00:00.000Z", + ragmirVersion: "0.4.12", + embeddingProvider: "local-hash", + embeddingModel: "mixedbread-ai/mxbai-embed-xsmall-v1", + chunkSize: 1200, + chunkOverlap: 200, + fileCount: 3, + chunkCount: 12, + } + + it("round-trips the index manifest", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "ragmir-index-manifest-")) + tempDirs.push(root) + const config = testConfig(root) + + await writeIndexManifest(sampleManifest, config) + const manifest = await readIndexManifest(config) + + expect(manifest).toEqual(sampleManifest) + }) + + it("returns null when the manifest is missing", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "ragmir-index-manifest-missing-")) + tempDirs.push(root) + + expect(await readIndexManifest(testConfig(root))).toBeNull() + }) + + it("returns null when the manifest is malformed", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "ragmir-index-manifest-bad-")) + tempDirs.push(root) + const config = testConfig(root) + await mkdir(config.storageDir, { recursive: true }) + await writeFile(path.join(config.storageDir, "index-manifest.json"), "{not valid json", "utf8") + + expect(await readIndexManifest(config)).toBeNull() + }) + + it("returns null when the manifest has the wrong shape", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "ragmir-index-manifest-shape-")) + tempDirs.push(root) + const config = testConfig(root) + await mkdir(config.storageDir, { recursive: true }) + await writeFile( + path.join(config.storageDir, "index-manifest.json"), + JSON.stringify({ schemaVersion: 1, createdAt: "x" }), + "utf8", + ) + + expect(await readIndexManifest(config)).toBeNull() + }) +}) diff --git a/packages/ragmir-core/src/store.ts b/packages/ragmir-core/src/store.ts index da3e50a..d1b218d 100644 --- a/packages/ragmir-core/src/store.ts +++ b/packages/ragmir-core/src/store.ts @@ -2,9 +2,10 @@ import { mkdir, readFile, rm, writeFile } from "node:fs/promises" import path from "node:path" import * as lancedb from "@lancedb/lancedb" import { isRecord } from "./guards.js" -import type { Config, VectorRow } from "./types.js" +import type { Config, IndexManifest, VectorRow } from "./types.js" const EMPTY_TEXT_FILES_MANIFEST = "empty-text-files.json" +const INDEX_MANIFEST = "index-manifest.json" /** * LanceDB requires a minimum number of rows to train an IVF_PQ vector index. * Below this threshold, brute-force (flat) scan is used instead, which is @@ -24,7 +25,11 @@ export interface EmptyTextFileRecord { checksum: string } -export async function writeRows(rows: VectorRow[], config: Config): Promise { +export interface IndexWriteResult { + vectorIndexWarning: string | null +} + +export async function writeRows(rows: VectorRow[], config: Config): Promise { await mkdir(config.storageDir, { recursive: true }) const db = await lancedb.connect(config.storageDir) @@ -33,7 +38,8 @@ export async function writeRows(rows: VectorRow[], config: Config): Promise ({ ...row })) @@ -45,16 +51,27 @@ export async function writeRows(rows: VectorRow[], config: Config): Promise= MIN_INDEX_ROWS) { - await ensureVectorIndex(table, rows.length) + if (rows.length < MIN_INDEX_ROWS) { + return { vectorIndexWarning: null } } + + const result = await ensureVectorIndex(table, rows.length) + return { vectorIndexWarning: result.warning } } -async function ensureVectorIndex(table: lancedb.Table, rowCount: number): Promise { +interface EnsureVectorIndexResult { + created: boolean + warning: string | null +} + +async function ensureVectorIndex( + table: lancedb.Table, + rowCount: number, +): Promise { const existing = await table.listIndices() const hasVectorIndex = existing.some((index) => index.name === "vector_idx") if (hasVectorIndex) { - return + return { created: false, warning: null } } // numSubVectors must divide the vector dimension evenly. 16 is a safe default @@ -67,9 +84,16 @@ async function ensureVectorIndex(table: lancedb.Table, rowCount: number): Promis await table.createIndex("vector", { config: lancedb.Index.ivfPq({ numPartitions, numSubVectors }), }) - } catch { + return { created: true, warning: null } + } catch (error) { // Index training can fail on edge-case dimensionality or tiny effective - // corpora; the table remains usable via flat scan. Do not fail the ingest. + // corpora; the table remains usable via flat scan, but the operator + // deserves to know queries will be slower than expected. + const detail = error instanceof Error ? error.message : String(error) + return { + created: false, + warning: `Vector index training failed (${detail}). Falling back to flat scan; queries will be slower on large corpora.`, + } } } @@ -77,6 +101,52 @@ function clampIvfPartitions(value: number): number { return Math.min(MAX_IVF_PARTITIONS, Math.max(MIN_IVF_PARTITIONS, value)) } +export async function writeIndexManifest(manifest: IndexManifest, config: Config): Promise { + await mkdir(config.storageDir, { recursive: true }) + const manifestPath = path.join(config.storageDir, INDEX_MANIFEST) + await writeFile(manifestPath, JSON.stringify(manifest, null, 2), "utf8") +} + +export async function readIndexManifest(config: Config): Promise { + try { + const raw = JSON.parse( + await readFile(path.join(config.storageDir, INDEX_MANIFEST), "utf8"), + ) as unknown + if (!isRecord(raw)) { + return null + } + return isIndexManifest(raw) ? raw : null + } catch (error) { + // The manifest is purely diagnostic. A missing file (pre-manifest index) or + // a corrupt/unreadable file should surface as "no freshness info" rather + // than fail the caller. + if (error instanceof SyntaxError) { + return null + } + if (isNodeError(error) && error.code === "ENOENT") { + return null + } + throw error + } +} + +function isIndexManifest(value: unknown): value is IndexManifest { + if (!isRecord(value)) { + return false + } + return ( + typeof value.schemaVersion === "number" && + typeof value.createdAt === "string" && + typeof value.ragmirVersion === "string" && + (value.embeddingProvider === "local-hash" || value.embeddingProvider === "transformers") && + typeof value.embeddingModel === "string" && + typeof value.chunkSize === "number" && + typeof value.chunkOverlap === "number" && + typeof value.fileCount === "number" && + typeof value.chunkCount === "number" + ) +} + export async function writeEmptyTextFiles( records: EmptyTextFileRecord[], config: Config, diff --git a/packages/ragmir-core/src/test-support/config.ts b/packages/ragmir-core/src/test-support/config.ts index 6c90b7f..d59691c 100644 --- a/packages/ragmir-core/src/test-support/config.ts +++ b/packages/ragmir-core/src/test-support/config.ts @@ -37,6 +37,7 @@ export function testConfig( maxFileBytes: DEFAULT_CONFIG.maxFileBytes, ingestConcurrency: DEFAULT_CONFIG.ingestConcurrency, embeddingBatchSize: DEFAULT_CONFIG.embeddingBatchSize, + hybridTextScanLimit: DEFAULT_CONFIG.hybridTextScanLimit, includeExtensions: [...DEFAULT_CONFIG.includeExtensions], pdfOcrCommand: [...DEFAULT_CONFIG.pdfOcrCommand], pdfOcrTimeoutMs: DEFAULT_CONFIG.pdfOcrTimeoutMs, diff --git a/packages/ragmir-core/src/types.ts b/packages/ragmir-core/src/types.ts index ef0b444..5d96919 100644 --- a/packages/ragmir-core/src/types.ts +++ b/packages/ragmir-core/src/types.ts @@ -22,6 +22,7 @@ export interface Config { maxFileBytes: number ingestConcurrency: number embeddingBatchSize: number + hybridTextScanLimit: number includeExtensions: string[] pdfOcrCommand: string[] pdfOcrTimeoutMs: number @@ -58,6 +59,24 @@ export interface AccessLogUsageReport { export type EmbeddingProvider = "local-hash" | "transformers" +/** + * Manifest written next to the LanceDB table at each ingest. It captures the + * configuration that produced the indexed vectors so callers can detect a + * stale index cheaply (without re-scanning every file's checksum) when the + * embedding model, provider, chunking, or Ragmir schema has changed. + */ +export interface IndexManifest { + schemaVersion: number + createdAt: string + ragmirVersion: string + embeddingProvider: EmbeddingProvider + embeddingModel: string + chunkSize: number + chunkOverlap: number + fileCount: number + chunkCount: number +} + export interface RedactionConfig { enabled: boolean builtIn: boolean @@ -146,6 +165,7 @@ export interface IngestResult { emptyTextFiles: string[] unsupportedExtensions: Array<{ extension: string; count: number }> redactions: number + vectorIndexWarning: string | null errors: Array<{ path: string; message: string }> } @@ -270,6 +290,7 @@ export interface EvaluationResult { export interface AskResult { answer: string sources: SearchResult[] + staleWarning: string | null } export interface AuditReport { @@ -310,6 +331,10 @@ export interface DoctorReport { missingFromIndex: number staleInIndex: number securityWarnings: string[] + indexFreshness: { + manifestFound: boolean + warning: string | null + } ready: boolean nextSteps: string[] }