release: stale index detection, vector index warnings, configurable BM25 scan

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,57 @@
+name: Feature request
+description: Suggest a new Ragmir feature or improvement
+title: "feat: "
+labels: ["enhancement"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to suggest a feature. Ragmir focuses on
+        sovereign local RAG for confidential datasets and AI agents. Features
+        that add cloud dependencies, telemetry, or hosted services are out of
+        scope.
+  - type: textarea
+    id: problem
+    attributes:
+      label: Problem
+      description: What problem does this feature solve? What are you trying to do that Ragmir cannot do today?
+    validations:
+      required: true
+  - type: textarea
+    id: solution
+    attributes:
+      label: Proposed solution
+      description: Describe the feature or change you would like to see.
+    validations:
+      required: true
+  - type: textarea
+    id: alternatives
+    attributes:
+      label: Alternatives considered
+      description: Have you tried any workarounds or existing tools?
+    validations:
+      required: false
+  - type: dropdown
+    id: area
+    attributes:
+      label: Area
+      description: Which part of Ragmir does this concern?
+      options:
+        - Ingestion (parsing, chunking, redaction)
+        - Retrieval (search, ask, research, evaluate)
+        - MCP server / agent integration
+        - CLI
+        - Desktop app
+        - Skills (audio, report, legal)
+        - Documentation
+        - Other
+    validations:
+      required: true
+  - type: checkboxes
+    id: sovereign
+    attributes:
+      label: Sovereignty check
+      description: Confirm this feature is compatible with Ragmir's local-first, zero-telemetry posture.
+      options:
+        - label: This feature does not require sending documents, queries, or telemetry to a hosted service.
+          required: true

.github/PULL_REQUEST_TEMPLATE.md

@@ -8,7 +8,12 @@
 - [ ] `pnpm test`
 - [ ] `pnpm build`
 
+## Commits
+
+- [ ] Conventional Commits format (`feat:`, `fix:`, `docs:`, `chore:`, `test:`...).
+- [ ] Branch follows Git Flow (`feature/*`, `fix/*`, `chore/*` from `develop`).
+
 ## Security
 
-- [ ] No secrets, private documents, `.env` files, generated `.kb/` or generated `.ragmir/` state are included.
+- [ ] No secrets, private documents, `.env` files, or generated `.ragmir/` state are included.
 - [ ] Public documentation does not expose private project, customer, pricing, or validation details.

CONTRIBUTING.md

@@ -2,6 +2,23 @@
 
 Ragmir is an open-source project under the MIT License. Issues and pull requests are welcome.
 
+Ragmir is maintained by a single developer ([Jean-Baptiste Thery](https://github.com/jb-thery)).
+Be kind, be specific, and keep the scope of each contribution focused so it can be reviewed
+within a reasonable time.
+
+## Reporting Issues
+
+Use the [issue templates](https://github.com/jcode-works/jcode-ragmir/issues/new/choose):
+
+- **Bug report**: include the Ragmir version, a minimal reproduction (commands + sample files,
+  no private documents or secrets), and the expected behavior.
+- **Feature request**: describe the problem you are trying to solve and the proposed solution.
+  Confirm the feature stays compatible with Ragmir's local-first, zero-telemetry posture.
+
+Before opening a new issue, search [existing issues](https://github.com/jcode-works/jcode-ragmir/issues)
+to avoid duplicates. Security vulnerabilities must not be reported through public issues — follow
+[`SECURITY.md`](./SECURITY.md).
+
 ## Development
 
 This repo pins its Node.js and Rust versions with [mise](https://mise.jdx.dev/) (see `mise.toml`),

README.md

@@ -5,6 +5,7 @@
 [![npm](https://img.shields.io/npm/v/@jcode.labs/ragmir)](https://www.npmjs.com/package/@jcode.labs/ragmir)
 [![npm downloads](https://img.shields.io/npm/dm/@jcode.labs/ragmir?label=downloads%2Fmonth)](https://www.npmjs.com/package/@jcode.labs/ragmir)
 [![license: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/jcode-works/jcode-ragmir/blob/main/LICENSE)
+[![GitHub Sponsors](https://img.shields.io/badge/sponsor-JCode%20Labs-ea4aaa.svg)](https://github.com/sponsors/jb-thery)
 
 Open-source local RAG library, CLI, and MCP server. Ragmir indexes your specs, docs, and code
 locally and gives your AI agents only the useful cited passages, over MCP, without burning tokens on

package.json

@@ -5,6 +5,33 @@
   "description": "Monorepo for Ragmir Core and open-source Ragmir add-ons.",
   "type": "module",
   "license": "MIT",
+  "author": {
+    "name": "Jean-Baptiste Thery",
+    "url": "https://github.com/jb-thery"
+  },
+  "homepage": "https://github.com/jcode-works/jcode-ragmir#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/jcode-works/jcode-ragmir.git"
+  },
+  "bugs": {
+    "url": "https://github.com/jcode-works/jcode-ragmir/issues"
+  },
+  "funding": {
+    "type": "github",
+    "url": "https://github.com/sponsors/jb-thery"
+  },
+  "keywords": [
+    "ragmir",
+    "rag",
+    "local-rag",
+    "sovereign-rag",
+    "mcp",
+    "ai-agents",
+    "private-ai",
+    "local-first",
+    "knowledge-base"
+  ],
   "packageManager": "pnpm@11.9.0",
   "scripts": {
     "bootstrap": "mise install && pnpm install",

packages/ragmir-core/src/cli.ts

@@ -21,6 +21,7 @@ import { doctor } from "./doctor.js"
 import { pullEmbeddingModel } from "./embeddings.js"
 import { evaluateGoldenQueries } from "./evaluate.js"
 import { countSkippedByReason } from "./files.js"
+import { getIndexFreshnessWarning, getLexicalScanWarning } from "./index-diagnostics.js"
 import { audit, ingest } from "./ingest.js"
 import { initProject } from "./init.js"
 import { serveMcp } from "./mcp.js"
@@ -279,6 +280,9 @@ program
     )
     printUnsupportedSummary(result.unsupportedExtensions)
     printEmptyTextFiles(result.emptyTextFiles)
+    if (result.vectorIndexWarning) {
+      console.log(pc.yellow(result.vectorIndexWarning))
+    }
     if (result.unsupportedFiles > 0 || result.oversizedFiles > 0 || result.sensitiveFiles > 0) {
       const auditCommand = await ragmirCommand(cwd, ["audit", "--unsupported"])
       console.log(
@@ -324,6 +328,8 @@ program
         return
       }
 
+      await printStaleIndexWarnings(cwd)
+
       for (const [index, result] of outputResults.entries()) {
         const distance = result.distance === null ? "n/a" : result.distance.toFixed(4)
         console.log(
@@ -354,6 +360,9 @@ program
     }
 
     console.log(`\n${result.answer}\n`)
+    if (result.staleWarning) {
+      console.error(pc.yellow(result.staleWarning))
+    }
     if (result.sources.length > 0) {
       console.log(pc.dim("Sources:"))
       for (const [index, source] of result.sources.entries()) {
@@ -938,6 +947,20 @@ function isTtsModule(value: unknown): value is TtsModule {
   )
 }
 
+async function printStaleIndexWarnings(cwd: string): Promise<void> {
+  const config = await loadConfig(cwd)
+  const freshnessWarning = await getIndexFreshnessWarning(config)
+  if (freshnessWarning) {
+    console.error(pc.yellow(freshnessWarning))
+    return
+  }
+  const chunkCount = await countRows(config)
+  const lexicalScanWarning = getLexicalScanWarning(config, chunkCount)
+  if (lexicalScanWarning) {
+    console.error(pc.yellow(lexicalScanWarning))
+  }
+}
+
 function printDoctor(report: Awaited<ReturnType<typeof doctor>>): void {
   console.log(`projectRoot=${report.projectRoot}`)
   console.log(`initialized=${report.initialized}`)
@@ -962,6 +985,10 @@ function printDoctor(report: Awaited<ReturnType<typeof doctor>>): void {
       console.log(pc.yellow(`warning: ${warning}`))
     }
   }
+  console.log(`indexFreshness.manifestFound=${report.indexFreshness.manifestFound}`)
+  if (report.indexFreshness.warning) {
+    console.log(pc.yellow(`indexFreshness: ${report.indexFreshness.warning}`))
+  }
   console.log("nextSteps:")
   for (const step of report.nextSteps) {
     console.log(`  - ${step}`)

packages/ragmir-core/src/config.test.ts

@@ -304,6 +304,27 @@ describe("loadConfig", () => {
     }
   })
 
+  it("defaults hybridTextScanLimit and overrides it from env", async () => {
+    const root = await mkdtemp(path.join(os.tmpdir(), "jcode-kb-scan-limit-"))
+    tempDirs.push(root)
+    await mkdir(path.join(root, ".ragmir"), { recursive: true })
+    await writeFile(path.join(root, ".ragmir/config.json"), "{}\n", "utf8")
+
+    expect((await loadConfig(root)).hybridTextScanLimit).toBe(5000)
+
+    const original = process.env.RAGMIR_HYBRID_TEXT_SCAN_LIMIT
+    process.env.RAGMIR_HYBRID_TEXT_SCAN_LIMIT = "2000"
+    try {
+      expect((await loadConfig(root)).hybridTextScanLimit).toBe(2000)
+    } finally {
+      if (original === undefined) {
+        delete process.env.RAGMIR_HYBRID_TEXT_SCAN_LIMIT
+      } else {
+        process.env.RAGMIR_HYBRID_TEXT_SCAN_LIMIT = original
+      }
+    }
+  })
+
   it("rejects unknown config keys so typos surface instead of being ignored", async () => {
     const root = await mkdtemp(path.join(os.tmpdir(), "jcode-kb-strict-"))
     tempDirs.push(root)

packages/ragmir-core/src/config.ts

@@ -52,6 +52,7 @@ const rawConfigSchema = z
     maxFileBytes: z.number().int().positive().default(DEFAULT_CONFIG.maxFileBytes),
     ingestConcurrency: z.number().int().positive().default(DEFAULT_CONFIG.ingestConcurrency),
     embeddingBatchSize: z.number().int().positive().default(DEFAULT_CONFIG.embeddingBatchSize),
+    hybridTextScanLimit: z.number().int().positive().default(DEFAULT_CONFIG.hybridTextScanLimit),
     includeExtensions: z.array(z.string().min(1)).default(DEFAULT_CONFIG.includeExtensions),
     pdfOcrCommand: z.array(z.string().min(1)).default(DEFAULT_CONFIG.pdfOcrCommand),
     pdfOcrTimeoutMs: z.number().int().positive().default(DEFAULT_CONFIG.pdfOcrTimeoutMs),
@@ -146,6 +147,7 @@ export async function loadConfig(start = process.cwd()): Promise<Config> {
     maxFileBytes: withEnv.maxFileBytes,
     ingestConcurrency: withEnv.ingestConcurrency,
     embeddingBatchSize: withEnv.embeddingBatchSize,
+    hybridTextScanLimit: withEnv.hybridTextScanLimit,
     includeExtensions: normalizeExtensions(withEnv.includeExtensions),
     pdfOcrCommand: withEnv.pdfOcrCommand,
     pdfOcrTimeoutMs: withEnv.pdfOcrTimeoutMs,
@@ -228,6 +230,11 @@ function applyEnv(config: RawConfig): RawConfig {
       "KB_EMBEDDING_BATCH_SIZE",
       config.embeddingBatchSize,
     ),
+    hybridTextScanLimit: readPositiveIntEnv(
+      "RAGMIR_HYBRID_TEXT_SCAN_LIMIT",
+      "KB_HYBRID_TEXT_SCAN_LIMIT",
+      config.hybridTextScanLimit,
+    ),
     includeExtensions: readExtensionsEnv(
       "RAGMIR_INCLUDE_EXTENSIONS",
       "KB_INCLUDE_EXTENSIONS",

packages/ragmir-core/src/defaults.ts

@@ -39,6 +39,7 @@ export const DEFAULT_CONFIG: Omit<Config, "projectRoot"> = {
   maxFileBytes: 50_000_000,
   ingestConcurrency: 4,
   embeddingBatchSize: 32,
+  hybridTextScanLimit: 5_000,
   includeExtensions: [],
   pdfOcrCommand: [],
   pdfOcrTimeoutMs: 120_000,

packages/ragmir-core/src/doctor.test.ts

@@ -59,6 +59,8 @@ describe("doctor", () => {
     expect(ready.nextSteps).toContain(
       "For natural-language Q&A, run `pnpm exec ragmir models pull --enable`, then run `pnpm exec ragmir ingest --rebuild`.",
     )
+    expect(ready.indexFreshness.manifestFound).toBe(true)
+    expect(ready.indexFreshness.warning).toBeNull()
   })
 
   it("detects an installed agent kit from the files installSkill writes", async () => {

packages/ragmir-core/src/doctor.ts

@@ -3,6 +3,7 @@ import path from "node:path"
 import { findProjectConfig, loadConfig } from "./config.js"
 import { RAGMIR_DIR } from "./defaults.js"
 import { countSkippedByReason } from "./files.js"
+import { getIndexFreshnessWarning, getLexicalScanWarning } from "./index-diagnostics.js"
 import { audit } from "./ingest.js"
 import { ragmirCommand } from "./package-manager.js"
 import { securityAudit } from "./security.js"
@@ -12,7 +13,7 @@ import {
   MCP_CONFIG_FILENAME,
   SKILL_NAMES,
 } from "./skill.js"
-import { countRows } from "./store.js"
+import { countRows, readIndexManifest } from "./store.js"
 import type { DoctorReport } from "./types.js"
 
 export async function doctor(cwd = process.cwd()): Promise<DoctorReport> {
@@ -21,11 +22,20 @@ export async function doctor(cwd = process.cwd()): Promise<DoctorReport> {
   const config = await loadConfig(cwd)
   const command = await ragmirCommand(config.projectRoot, [])
   const agentKitInstalled = isAgentKitInstalled(config.projectRoot)
-  const [auditReport, securityReport, chunksIndexed] = await Promise.all([
-    audit(config.projectRoot),
-    securityAudit(config.projectRoot),
-    countRows(config),
-  ])
+  const [auditReport, securityReport, chunksIndexed, manifest, freshnessWarning] =
+    await Promise.all([
+      audit(config.projectRoot),
+      securityAudit(config.projectRoot),
+      countRows(config),
+      readIndexManifest(config),
+      getIndexFreshnessWarning(config),
+    ])
+
+  const lexicalScanWarning = chunksIndexed > 0 ? getLexicalScanWarning(config, chunksIndexed) : null
+  const indexFreshness = {
+    manifestFound: manifest !== null,
+    warning: freshnessWarning,
+  }
 
   const nextSteps = nextActions({
     initialized,
@@ -38,6 +48,8 @@ export async function doctor(cwd = process.cwd()): Promise<DoctorReport> {
     warnings: securityReport.warnings.length,
     embeddingProvider: config.embeddingProvider,
     agentKitInstalled,
+    freshnessWarning,
+    lexicalScanWarning,
     run: (args) => command.display + (args.length > 0 ? ` ${args.join(" ")}` : ""),
   })
 
@@ -61,12 +73,14 @@ export async function doctor(cwd = process.cwd()): Promise<DoctorReport> {
     missingFromIndex: auditReport.missingFromIndex.length,
     staleInIndex: auditReport.staleInIndex.length,
     securityWarnings: securityReport.warnings,
+    indexFreshness,
     ready:
       initialized &&
       chunksIndexed > 0 &&
       auditReport.missingFromIndex.length === 0 &&
       auditReport.staleInIndex.length === 0 &&
-      securityReport.warnings.length === 0,
+      securityReport.warnings.length === 0 &&
+      freshnessWarning === null,
     nextSteps,
   }
 }
@@ -82,6 +96,8 @@ interface NextActionInput {
   warnings: number
   embeddingProvider: string
   agentKitInstalled: boolean
+  freshnessWarning: string | null
+  lexicalScanWarning: string | null
   run: (args: string[]) => string
 }
 
@@ -114,6 +130,16 @@ function nextActions(input: NextActionInput): string[] {
     )
   }
 
+  if (input.freshnessWarning) {
+    steps.push(
+      `${input.freshnessWarning} Run \`${input.run(["ingest", "--rebuild"])}\` to align the index with the active configuration.`,
+    )
+  }
+
+  if (input.lexicalScanWarning) {
+    steps.push(input.lexicalScanWarning)
+  }
+
   if (input.warnings > 0) {
     steps.push(
       `Run \`${input.run(["security-audit", "--strict"])}\` and fix the reported warnings.`,