diff --git a/CHANGELOG.md b/CHANGELOG.md index 1815c4150..a0f5f7008 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### New Features +- **Markdown is indexed, and a documentation question gets the section, not the graph.** Every `.md` file's headings, sections, tables and links are nodes (the extractor from #361), and a doc-shaped `codegraph_explore` query that names a markdown file now renders that file's best sections first and whole — the top three by idf-weighted line hits, a heading the query covers word for word counted as named, 8k characters per file — with the blast-radius, relationships and "additional files" blocks held back unless a code file rendered too. Measured on a 109-file docs corpus under headless Claude Code, 36 cells over three rounds: the right file and section in every call, median 1 tool call against 4 for Grep-then-Read, 36 of 36 correct. Code answers keep their shape: markdown nodes leave a subgraph the doc tier did not seed, a markdown body is never mistaken for a generated-file header, and the explore budget tiers count code files only, so a README-heavy repo does not cross a breakpoint. The server instructions say markdown is indexed, which the branch's own text still denied. (#361, #1439) - **A busy screen's picture is laid out by the parts of the screen.** A screen is a set of handlers with no order between them, so on a hub screen the old rows-by-distance collapsed into one enormous row — the main screen of one app put 89 boxes side by side on a canvas over 28,000px wide, every line a near-horizontal sweep across all of it. The Steps tab now groups a screen's picture by region — the component that owns each handler, named in a small caption over its boxes — with each region a column where a step sits above what it sets in motion, tiled in the screen's own source order. At rest the picture hides only two things: the screen's own fan-out — one line into each region stands in for it — and lines that point back up; every other line draws where it leads, between two regions included, and selecting a step brings out its whole story in the side panel, link by link. A box nothing points at is the screen's own doing — run on render or mount, or from a binding written inline — the key says so, and selecting it lights its line from the screen with what fires it. The same app's widest screen now lays out under 3,500px with every line local, and the whole picture fits on screen when it opens. Endpoints, handlers and the in-order reading are untouched, and nothing needs a re-index: the regions come from the same walk that draws the steps. - **Where the code chooses, the picture says so once.** A helper that ends `return (await hasSeenWelcome(id)) ? '/home/' : '/welcome/'` sends the app to one of two screens, but the Steps picture drew that as two separate arrows, each carrying the whole condition with one of them negated and both cut off at the same forty characters — and before you clicked anything, neither arrow was labelled at all, so nothing said it was a choice. Now sibling arrows out of one box that are the arms of one `if`, `switch` or ternary are drawn as the choice they are: the condition is written once under the box that decides it, and each arrow out says only which way it is — `yes`, `no`, or a case's own value. They are the only arrows labelled before you select anything, so the picture reads at a glance without becoming a wall of text. A one-sided guard — an early exit, an `if` with only one side drawn — still carries its condition on the arrow, and an arrow that is reached whether or not the condition holds never claims a side. Nothing needs a re-index: the decision is read from the source at request time. diff --git a/__tests__/explore-doc-tier-named.test.ts b/__tests__/explore-doc-tier-named.test.ts new file mode 100644 index 000000000..7d6bd5471 --- /dev/null +++ b/__tests__/explore-doc-tier-named.test.ts @@ -0,0 +1,127 @@ +/** + * The doc tier's naming path: a query that IS a document's name. + * + * `collectDocSeeds` honoured `named` at the file gate and ignored it at the + * section gate, so `CONTRIBUTING.md` found the file and then dropped it for + * want of a scoring line — the user typed a filename and got nothing. Three + * rules combined to make every section score 0 for such a query: a term + * appearing in the file path is weighted 0 (and for a bare filename that is + * the only term), `lineScore` needs two distinct terms on one line, and + * `coveredHeading` needs a term of non-zero weight. + * + * A fourth rule made some headings unreachable by any query at all: + * `coveredHeading` demanded every significant heading word be covered, but + * DOC_QUERY_NOISE words are stripped from the query, so a heading containing + * "repo" could never be covered. + * + * These are retrieval assertions, not extraction ones — the misses that + * motivated them were found by querying a real 84-file repo, which is exactly + * what no unit test here was doing. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import CodeGraph from '../src/index'; +import { ToolHandler } from '../src/mcp/tools'; + +let dir: string; +let cg: CodeGraph; + +async function explore(query: string): Promise { + const res = await new ToolHandler(cg).execute('codegraph_explore', { query }); + return res.content?.[0]?.text ?? ''; +} + +/** The response renders a source section for `file`. */ +const hasSection = (response: string, file: string): boolean => + response.includes('**`' + file + '`'); + +const CONTRIBUTING = `# Project Contributing Guide + +Some preamble that names no section. + +## Repo Setup + +Clone it and install dependencies. + +## Cloning the repo on Windows + +Enable symlinks and long paths before cloning. + +## Ignoring commits when running git blame + +Use the ignore-revs file. +`; + +const README = `# Widget + +A widget. + +## Installation + +Install the widget. + +## Usage + +Use the widget. +`; + +beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-doc-named-')); + fs.mkdirSync(path.join(dir, 'src'), { recursive: true }); + fs.mkdirSync(path.join(dir, 'test', 'fixtures'), { recursive: true }); + fs.writeFileSync(path.join(dir, 'CONTRIBUTING.md'), CONTRIBUTING); + fs.writeFileSync(path.join(dir, 'README.md'), README); + // A fixture copy, which DOC_LOW_PATH exists to keep out of results. + fs.writeFileSync(path.join(dir, 'test', 'fixtures', 'README.md'), README); + // Code, so the index is not markdown-only and code queries have somewhere to go. + fs.writeFileSync( + path.join(dir, 'src', 'widget.ts'), + `export function createWidget(size: number): number { return size * 2; }\n` + + `export function resizeWidget(w: number): number { return createWidget(w); }\n` + ); + cg = CodeGraph.initSync(dir); + await cg.indexAll(); +}, 180_000); + +afterAll(() => { + cg?.destroy(); + if (dir && fs.existsSync(dir)) fs.rmSync(dir, { recursive: true, force: true }); +}); + +describe('a query that names a markdown file', () => { + it('renders the file it names, spelled with the extension', async () => { + const out = await explore('CONTRIBUTING.md'); + expect(hasSection(out, 'CONTRIBUTING.md')).toBe(true); + }); + + it('renders the file it names, spelled as the bare stem', async () => { + const out = await explore('readme'); + expect(hasSection(out, 'README.md')).toBe(true); + }); + + it('does not let a bare stem surface a fixture copy', async () => { + // The stem match unlocks the two-hit gate and the section fallback, but + // deliberately not the DOC_LOW_PATH bypass — that stays the privilege of + // an explicitly spelled path. + const out = await explore('readme'); + expect(hasSection(out, 'test/fixtures/README.md')).toBe(false); + }); +}); + +describe('a heading containing a DOC_QUERY_NOISE word', () => { + it('is reachable, though "repo" can never appear among the query terms', async () => { + const out = await explore('the contributing guide for windows'); + expect(hasSection(out, 'CONTRIBUTING.md')).toBe(true); + expect(out).toContain('Cloning the repo on Windows'); + }); +}); + +describe('the DOC_WORD gate still declines what it should', () => { + it('a question with no doc word pulls no markdown', async () => { + const out = await explore('how do i resize a widget'); + expect(hasSection(out, 'CONTRIBUTING.md')).toBe(false); + expect(hasSection(out, 'README.md')).toBe(false); + }); +}); diff --git a/__tests__/extraction.test.ts b/__tests__/extraction.test.ts index ad0ba2374..597a26dcb 100644 --- a/__tests__/extraction.test.ts +++ b/__tests__/extraction.test.ts @@ -102,6 +102,12 @@ describe('Language Detection', () => { expect(detectLanguage('stdio.h', '#ifndef STDIO_H\nvoid printf();\n#endif\n')).toBe('c'); }); + it('should detect Markdown files', () => { + expect(detectLanguage('README.md')).toBe('markdown'); + expect(detectLanguage('docs/guide.markdown')).toBe('markdown'); + expect(detectLanguage('docs/page.mdx')).toBe('markdown'); + }); + it('should detect Metal shader files as C++ (#1121)', () => { expect(detectLanguage('Shaders.metal')).toBe('cpp'); expect(isSourceFile('Renderer/Shaders.metal')).toBe(true); @@ -250,11 +256,218 @@ describe('Language Support', () => { expect(languages).toContain('swift'); expect(languages).toContain('kotlin'); expect(languages).toContain('dart'); + expect(languages).toContain('markdown'); expect(languages).toContain('solidity'); expect(languages).toContain('nix'); }); }); +describe('Markdown Extraction', () => { + it('should extract headings, links, and shell script references', () => { + const markdown = `# Project Guide + +See [Setup](docs/setup.md#install) and scripts/release.mjs. + +## Release + +\`\`\`bash +npm run build +node scripts/release.mjs +\`\`\` +`; + + const result = extractFromSource('README.md', markdown); + + const fileNode = result.nodes.find((n) => n.kind === 'file'); + expect(fileNode).toMatchObject({ + name: 'README.md', + language: 'markdown', + }); + + const headings = result.nodes.filter((n) => n.kind === 'module'); + expect(headings.map((n) => n.name)).toContain('Project Guide'); + expect(headings.map((n) => n.name)).toContain('Release'); + + const commandNode = result.nodes.find((n) => n.kind === 'function' && n.signature === 'node scripts/release.mjs'); + expect(commandNode).toBeDefined(); + + expect(result.unresolvedReferences).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + referenceName: 'docs/setup.md#install', + referenceKind: 'imports', + language: 'markdown', + }), + expect.objectContaining({ + referenceName: 'scripts/release.mjs', + referenceKind: 'calls', + language: 'markdown', + }), + ]) + ); + }); + + it('should extract structured table rows and file-symbol references from Markdown', () => { + const markdown = `# Maintenance Guide + +## Phase 4 + +| Template | CLI Entry | Dispatcher | Implementation | +| --- | --- | --- | --- | +| P4-S1 | \`python "{script_path}" p4 "{csv_file}" s1 "{conditions_or_-}" "{probe_cols}"\` | \`scripts/csv_search.py::run_p4\` | \`scripts/csv_search.py::_p4_stage1\` | +| P4-S2 | \`python "{script_path}" p4 "{csv_file}" s2 "{stage1_rows}" "{condition_or_-}" "{detail_cols}"\` | \`scripts/csv_search.py::run_p4\` | \`scripts/csv_search.py::_p4_stage2\` | + +- P4-FLOW changes must inspect \`scripts/csv_search.py::run_p4\`. +`; + + const result = extractFromSource('phases/phase4.md', markdown); + + const tableRows = result.nodes.filter((n) => n.kind === 'constant' && n.qualifiedName.includes('table-row')); + expect(tableRows.map((n) => n.name)).toEqual(expect.arrayContaining(['P4-S1', 'P4-S2'])); + + const p4s1 = tableRows.find((n) => n.name === 'P4-S1'); + expect(p4s1?.signature).toContain('Template: P4-S1'); + expect(p4s1?.signature).toContain('Dispatcher: scripts/csv_search.py::run_p4'); + + const commandNode = result.nodes.find((n) => + n.kind === 'function' && + n.language === 'markdown' && + n.signature?.includes('python "{script_path}" p4') + ); + expect(commandNode).toBeDefined(); + + expect(result.unresolvedReferences).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + referenceName: 'phases/scripts/csv_search.py::run_p4', + referenceKind: 'references', + language: 'markdown', + }), + expect.objectContaining({ + referenceName: 'phases/scripts/csv_search.py::_p4_stage1', + referenceKind: 'references', + language: 'markdown', + }), + ]) + ); + }); + + it('should keep structured blocks after fences containing a different fence marker', () => { + const markdown = `# Runbook + +\`\`\`text +~~~~ +\`\`\` + +- POST-FENCE references \`src/auth.ts::login\`. + +| Key | Target | +| --- | --- | +| POST-TABLE | \`src/auth.ts::login\` | +`; + + const result = extractFromSource('docs/runbook.md', markdown); + const constants = result.nodes.filter((n) => n.kind === 'constant'); + + expect(constants).toEqual(expect.arrayContaining([ + expect.objectContaining({ docstring: 'POST-FENCE references src/auth.ts::login.' }), + expect.objectContaining({ name: 'POST-TABLE' }), + ])); + }); + + it('indexes Setext (underline) headings and skips frontmatter / code fences', () => { + const markdown = `--- +title: Config Doc +--- + +Architecture Overview +===================== + +Intro paragraph for the overview. + +Routing Layer +------------- + +\`\`\`md +Not A Heading +============= +\`\`\` +`; + + const result = extractFromSource('docs/arch.md', markdown); + const headings = result.nodes.filter((n) => n.kind === 'module'); + const byName = new Map(headings.map((h) => [h.name, h])); + + // Setext H1 (===) and H2 (---) become module nodes. + expect(byName.get('Architecture Overview')?.signature).toBe('# Architecture Overview'); + expect(byName.get('Routing Layer')?.signature).toBe('## Routing Layer'); + // Frontmatter `title:` (above the closing `---`) is NOT a heading, and a + // setext-looking line inside a code fence is ignored. + expect(byName.has('title: Config Doc')).toBe(false); + expect(byName.has('Not A Heading')).toBe(false); + }); + + it('builds a deterministic, compact file digest (intro + key references)', () => { + const markdown = `# Release Runbook + +This runbook explains how to cut a release. + +See [setup](docs/setup.md#install) and run \`scripts/release.mjs\`. +It dispatches \`scripts/csv_search.py::run_p4\`. +`; + + const result = extractFromSource('RUNBOOK.md', markdown); + const fileNode = result.nodes.find((n) => n.kind === 'file'); + + expect(fileNode?.docstring).toBeDefined(); + const digest = fileNode!.docstring!; + // Intro is the first prose line, not the heading or a link blob. + expect(digest).toContain('This runbook explains how to cut a release.'); + // Key referenced files/symbols are surfaced, compacted to basenames. + expect(digest).toContain('refs:'); + expect(digest).toContain('setup.md#install'); + expect(digest).toContain('release.mjs'); + expect(digest).toContain('csv_search.py::run_p4'); + // Short enough to show in node details (the < 200 char detail gate). + expect(digest.length).toBeLessThan(200); + }); +}); + +describe('Code to Markdown Reference Extraction', () => { + it('should extract Markdown path references from code string literals', () => { + const code = ` +export const GUIDE = '../docs/guide.md'; + +export function loadDocs() { + return fs.readFileSync('../docs/guide.md#install', 'utf8'); +} +`; + + const result = extractFromSource('src/load-docs.ts', code); + const loadDocs = result.nodes.find((n) => n.kind === 'function' && n.name === 'loadDocs'); + const guideConstant = result.nodes.find((n) => n.kind === 'constant' && n.name === 'GUIDE'); + + expect(loadDocs).toBeDefined(); + expect(guideConstant).toBeDefined(); + expect(result.unresolvedReferences).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + fromNodeId: loadDocs!.id, + referenceName: 'docs/guide.md#install', + referenceKind: 'references', + language: 'typescript', + }), + expect.objectContaining({ + fromNodeId: guideConstant!.id, + referenceName: 'docs/guide.md', + referenceKind: 'references', + language: 'typescript', + }), + ]) + ); + }); +}); + describe('Nix Extraction', () => { it('should distinguish Nix variable and function bindings', () => { const code = ` diff --git a/__tests__/fixtures/kernel-parity/Torture.cs b/__tests__/fixtures/kernel-parity/Torture.cs index 520b1d4e3..990566c12 100644 --- a/__tests__/fixtures/kernel-parity/Torture.cs +++ b/__tests__/fixtures/kernel-parity/Torture.cs @@ -179,3 +179,25 @@ namespace Torture.Beta { public class Other { } } + +// Markdown path references: code -> documentation edges. Every shape the +// normalizer branches on, so the two arms have to agree about the rejections +// (URL, escape above the root) as well as the emissions. +namespace Torture.Markdown +{ + public class MarkdownPaths + { + public const string Guide = "../docs/guide.md#install"; + public const string Bare = "README.md"; + public const string Rooted = "/docs/rooted.md"; + public const string Escapes = "../../../../outside.md"; + public const string Remote = "https://example.com/remote.md"; + public const string Queried = "./notes.md?raw=1#top"; + public const string TwoInOne = "see a.md and also sub/b.mdx"; + + public void Load() + { + LoadDoc("docs/deep/nested.markdown"); + } + } +} diff --git a/__tests__/fixtures/kernel-parity/Torture.java b/__tests__/fixtures/kernel-parity/Torture.java index 703dc3b3c..a1b90a889 100644 --- a/__tests__/fixtures/kernel-parity/Torture.java +++ b/__tests__/fixtures/kernel-parity/Torture.java @@ -100,3 +100,20 @@ interface Shape extends Comparable, Cloneable { @interface Marker { String value() default ""; } + +// Markdown path references: code -> documentation edges. Every shape the +// normalizer branches on, so the two arms have to agree about the rejections +// (URL, escape above the root) as well as the emissions. +class MarkdownPaths { + static final String GUIDE = "../docs/guide.md#install"; + static final String BARE = "README.md"; + static final String ROOTED = "/docs/rooted.md"; + static final String ESCAPES = "../../../../outside.md"; + static final String REMOTE = "https://example.com/remote.md"; + static final String QUERIED = "./notes.md?raw=1#top"; + static final String TWO_IN_ONE = "see a.md and also sub/b.mdx"; + + void load() { + loadDoc("docs/deep/nested.markdown"); + } +} diff --git a/__tests__/fixtures/kernel-parity/torture.R b/__tests__/fixtures/kernel-parity/torture.R index c3b02bc15..6adb2dac5 100644 --- a/__tests__/fixtures/kernel-parity/torture.R +++ b/__tests__/fixtures/kernel-parity/torture.R @@ -202,3 +202,16 @@ piped <- x |> f_ph(y = _) # --- UTF-16 columns ---------------------------------------------------------- msg <- "héllo 🎉" emoji_caller <- function() after_emoji("🎉🎉", target_fn()) + +# --- Markdown path references ------------------------------------------------ +# Code -> documentation edges. Every shape the normalizer branches on, so the +# two arms have to agree about the rejections (URL, escape above the root) as +# well as the emissions. +md_guide <- "../docs/guide.md#install" +md_bare <- "README.md" +md_rooted <- "/docs/rooted.md" +md_escapes <- "../../../../outside.md" +md_remote <- "https://example.com/remote.md" +md_queried <- "./notes.md?raw=1#top" +md_two_in_one <- "see a.md and also sub/b.mdx" +md_load <- function() load_doc("docs/deep/nested.markdown") diff --git a/__tests__/fixtures/kernel-parity/torture.c b/__tests__/fixtures/kernel-parity/torture.c index 57e04a912..dcf42024b 100644 --- a/__tests__/fixtures/kernel-parity/torture.c +++ b/__tests__/fixtures/kernel-parity/torture.c @@ -168,3 +168,16 @@ typedef union { } word_t; static unsigned int hdr_raw(union packet_hdr *h) { return h->raw; } + +/* Markdown path references: code -> documentation edges. Every shape the + normalizer branches on, so the two arms have to agree about the rejections + (URL, escape above the root) as well as the emissions. */ +static const char *md_guide = "../docs/guide.md#install"; +static const char *md_bare = "README.md"; +static const char *md_rooted = "/docs/rooted.md"; +static const char *md_escapes = "../../../../outside.md"; +static const char *md_remote = "https://example.com/remote.md"; +static const char *md_queried = "./notes.md?raw=1#top"; +static const char *md_two_in_one = "see a.md and also sub/b.mdx"; + +static void md_load(void) { load_doc("docs/deep/nested.markdown"); } diff --git a/__tests__/fixtures/kernel-parity/torture.cpp b/__tests__/fixtures/kernel-parity/torture.cpp index aa6e382de..9709e268a 100644 --- a/__tests__/fixtures/kernel-parity/torture.cpp +++ b/__tests__/fixtures/kernel-parity/torture.cpp @@ -138,3 +138,16 @@ float drive() { (void)m; return r + f + flags + leg; } + +/* Markdown path references: code -> documentation edges. Every shape the + normalizer branches on, so the two arms have to agree about the rejections + (URL, escape above the root) as well as the emissions. */ +static const char *md_guide = "../docs/guide.md#install"; +static const char *md_bare = "README.md"; +static const char *md_rooted = "/docs/rooted.md"; +static const char *md_escapes = "../../../../outside.md"; +static const char *md_remote = "https://example.com/remote.md"; +static const char *md_queried = "./notes.md?raw=1#top"; +static const char *md_two_in_one = "see a.md and also sub/b.mdx"; + +static void md_load() { load_doc("docs/deep/nested.markdown"); } diff --git a/__tests__/fixtures/kernel-parity/torture.dart b/__tests__/fixtures/kernel-parity/torture.dart index fa4f73b38..081c504a3 100644 --- a/__tests__/fixtures/kernel-parity/torture.dart +++ b/__tests__/fixtures/kernel-parity/torture.dart @@ -229,3 +229,18 @@ void patternUser(Object o) { void afterUnicode(String seance) { emit('café ☕ done'); } + +// Markdown path references: code -> documentation edges. Every shape the +// normalizer branches on, so the two arms have to agree about the rejections +// (URL, escape above the root) as well as the emissions. +const mdGuide = '../docs/guide.md#install'; +const mdBare = 'README.md'; +const mdRooted = '/docs/rooted.md'; +const mdEscapes = '../../../../outside.md'; +const mdRemote = 'https://example.com/remote.md'; +const mdQueried = './notes.md?raw=1#top'; +const mdTwoInOne = 'see a.md and also sub/b.mdx'; + +void mdLoad() { + loadDoc('docs/deep/nested.markdown'); +} diff --git a/__tests__/fixtures/kernel-parity/torture.go b/__tests__/fixtures/kernel-parity/torture.go index f89b8d429..aeb103d58 100644 --- a/__tests__/fixtures/kernel-parity/torture.go +++ b/__tests__/fixtures/kernel-parity/torture.go @@ -59,3 +59,18 @@ func shadowed() { func reads() int { return MAX_ITEMS } + +// Markdown path references: code -> documentation edges. Every shape the +// normalizer branches on, so the two arms have to agree about the rejections +// (URL, escape above the root) as well as the emissions. +var mdGuide = "../docs/guide.md#install" +var mdBare = "README.md" +var mdRooted = "/docs/rooted.md" +var mdEscapes = "../../../../outside.md" +var mdRemote = "https://example.com/remote.md" +var mdQueried = "./notes.md?raw=1#top" +var mdTwoInOne = "see a.md and also sub/b.mdx" + +func mdLoad() { + loadDoc("docs/deep/nested.markdown") +} diff --git a/__tests__/fixtures/kernel-parity/torture.kt b/__tests__/fixtures/kernel-parity/torture.kt index 130611c81..15ffde64e 100644 --- a/__tests__/fixtures/kernel-parity/torture.kt +++ b/__tests__/fixtures/kernel-parity/torture.kt @@ -265,3 +265,18 @@ fun labeledLambda() { } fun whereClause(): Int where Int : Comparable = 1 + +// Markdown path references: code -> documentation edges. Every shape the +// normalizer branches on, so the two arms have to agree about the rejections +// (URL, escape above the root) as well as the emissions. +val mdGuide = "../docs/guide.md#install" +val mdBare = "README.md" +val mdRooted = "/docs/rooted.md" +val mdEscapes = "../../../../outside.md" +val mdRemote = "https://example.com/remote.md" +val mdQueried = "./notes.md?raw=1#top" +val mdTwoInOne = "see a.md and also sub/b.mdx" + +fun mdLoad() { + loadDoc("docs/deep/nested.markdown") +} diff --git a/__tests__/fixtures/kernel-parity/torture.lua b/__tests__/fixtures/kernel-parity/torture.lua index 52da48ca0..df5ddff92 100644 --- a/__tests__/fixtures/kernel-parity/torture.lua +++ b/__tests__/fixtures/kernel-parity/torture.lua @@ -96,4 +96,20 @@ local préfixe = "café" function after_unicode() end do goto done end ::done:: + +-- Markdown path references: code -> documentation edges. Every shape the +-- normalizer branches on, so the two arms have to agree about the rejections +-- (URL, escape above the root) as well as the emissions. +local md_guide = "../docs/guide.md#install" +local md_bare = "README.md" +local md_rooted = "/docs/rooted.md" +local md_escapes = "../../../../outside.md" +local md_remote = "https://example.com/remote.md" +local md_queried = "./notes.md?raw=1#top" +local md_two_in_one = "see a.md and also sub/b.mdx" + +function md_load() + load_doc("docs/deep/nested.markdown") +end + return M diff --git a/__tests__/fixtures/kernel-parity/torture.luau b/__tests__/fixtures/kernel-parity/torture.luau index 8432d6979..93744b189 100644 --- a/__tests__/fixtures/kernel-parity/torture.luau +++ b/__tests__/fixtures/kernel-parity/torture.luau @@ -58,4 +58,20 @@ M:update({ x = 1, y = 2 }, print) local préfixe = "café" function after_unicode() end + +-- Markdown path references: code -> documentation edges. Every shape the +-- normalizer branches on, so the two arms have to agree about the rejections +-- (URL, escape above the root) as well as the emissions. +local md_guide = "../docs/guide.md#install" +local md_bare = "README.md" +local md_rooted = "/docs/rooted.md" +local md_escapes = "../../../../outside.md" +local md_remote = "https://example.com/remote.md" +local md_queried = "./notes.md?raw=1#top" +local md_two_in_one = "see a.md and also sub/b.mdx" + +function md_load() + load_doc("docs/deep/nested.markdown") +end + return M diff --git a/__tests__/fixtures/kernel-parity/torture.php b/__tests__/fixtures/kernel-parity/torture.php index 0194250f8..54f6a4922 100644 --- a/__tests__/fixtures/kernel-parity/torture.php +++ b/__tests__/fixtures/kernel-parity/torture.php @@ -217,3 +217,19 @@ abstract class AbstractBase { abstract protected function hook(): void; } + +// Markdown path references: code -> documentation edges. Every shape the +// normalizer branches on, so the two arms have to agree about the rejections +// (URL, escape above the root) as well as the emissions. +$mdGuide = '../docs/guide.md#install'; +$mdBare = 'README.md'; +$mdRooted = '/docs/rooted.md'; +$mdEscapes = '../../../../outside.md'; +$mdRemote = 'https://example.com/remote.md'; +$mdQueried = './notes.md?raw=1#top'; +$mdTwoInOne = 'see a.md and also sub/b.mdx'; + +function mdLoad() +{ + loadDoc('docs/deep/nested.markdown'); +} diff --git a/__tests__/fixtures/kernel-parity/torture.rb b/__tests__/fixtures/kernel-parity/torture.rb index f1d5fbfd0..6d4ff34be 100644 --- a/__tests__/fixtures/kernel-parity/torture.rb +++ b/__tests__/fixtures/kernel-parity/torture.rb @@ -231,5 +231,20 @@ def self.top_singleton def obj.weird_singleton; end +# Markdown path references: code -> documentation edges. Every shape the +# normalizer branches on, so the two arms have to agree about the rejections +# (URL, escape above the root) as well as the emissions. +MD_GUIDE = '../docs/guide.md#install' +MD_BARE = 'README.md' +MD_ROOTED = '/docs/rooted.md' +MD_ESCAPES = '../../../../outside.md' +MD_REMOTE = 'https://example.com/remote.md' +MD_QUERIED = './notes.md?raw=1#top' +MD_TWO_IN_ONE = 'see a.md and also sub/b.mdx' + +def md_load + load_doc('docs/deep/nested.markdown') +end + __END__ raw data trailer here diff --git a/__tests__/fixtures/kernel-parity/torture.rs b/__tests__/fixtures/kernel-parity/torture.rs index 8fb8382be..72643ea4d 100644 --- a/__tests__/fixtures/kernel-parity/torture.rs +++ b/__tests__/fixtures/kernel-parity/torture.rs @@ -291,3 +291,18 @@ pub union Reg { } impl Base for Reg {} + +// Markdown path references: code -> documentation edges. Every shape the +// normalizer branches on, so the two arms have to agree about the rejections +// (URL, escape above the root) as well as the emissions. +const MD_GUIDE: &str = "../docs/guide.md#install"; +const MD_BARE: &str = "README.md"; +const MD_ROOTED: &str = "/docs/rooted.md"; +const MD_ESCAPES: &str = "../../../../outside.md"; +const MD_REMOTE: &str = "https://example.com/remote.md"; +const MD_QUERIED: &str = "./notes.md?raw=1#top"; +const MD_TWO_IN_ONE: &str = "see a.md and also sub/b.mdx"; + +fn md_load() { + load_doc("docs/deep/nested.markdown"); +} diff --git a/__tests__/fixtures/kernel-parity/torture.scala b/__tests__/fixtures/kernel-parity/torture.scala index de536e659..d035a1271 100644 --- a/__tests__/fixtures/kernel-parity/torture.scala +++ b/__tests__/fixtures/kernel-parity/torture.scala @@ -175,3 +175,18 @@ package object utilpkg { def pkgHelper(): Int = 1 val pkgShared = 2 } + +// Markdown path references: code -> documentation edges. Every shape the +// normalizer branches on, so the two arms have to agree about the rejections +// (URL, escape above the root) as well as the emissions. +object MarkdownPaths { + val mdGuide = "../docs/guide.md#install" + val mdBare = "README.md" + val mdRooted = "/docs/rooted.md" + val mdEscapes = "../../../../outside.md" + val mdRemote = "https://example.com/remote.md" + val mdQueried = "./notes.md?raw=1#top" + val mdTwoInOne = "see a.md and also sub/b.mdx" + + def mdLoad(): Unit = loadDoc("docs/deep/nested.markdown") +} diff --git a/__tests__/fixtures/kernel-parity/torture.swift b/__tests__/fixtures/kernel-parity/torture.swift index d0c4a3e06..c1816af56 100644 --- a/__tests__/fixtures/kernel-parity/torture.swift +++ b/__tests__/fixtures/kernel-parity/torture.swift @@ -204,3 +204,18 @@ func fnRefExtras() { import class Darwin.FILE @testable import TortureKit + +// Markdown path references: code -> documentation edges. Every shape the +// normalizer branches on, so the two arms have to agree about the rejections +// (URL, escape above the root) as well as the emissions. +let mdGuide = "../docs/guide.md#install" +let mdBare = "README.md" +let mdRooted = "/docs/rooted.md" +let mdEscapes = "../../../../outside.md" +let mdRemote = "https://example.com/remote.md" +let mdQueried = "./notes.md?raw=1#top" +let mdTwoInOne = "see a.md and also sub/b.mdx" + +func mdLoad() { + loadDoc("docs/deep/nested.markdown") +} diff --git a/__tests__/fixtures/kernel-parity/torture.tsx b/__tests__/fixtures/kernel-parity/torture.tsx index de27c0a8f..63d19c866 100644 --- a/__tests__/fixtures/kernel-parity/torture.tsx +++ b/__tests__/fixtures/kernel-parity/torture.tsx @@ -212,3 +212,15 @@ import('./dynamic-module'); new NS.Widget(makeArg()); new Map(); super_weird?.(); + +// Markdown path references: code -> documentation edges. Every shape the +// normalizer branches on, so the two arms have to agree about the rejections +// (URL, escape above the root) as well as the emissions. +const GUIDE = '../docs/guide.md#install'; +const BARE = 'README.md'; +const ROOTED = '/docs/rooted.md'; +const ESCAPES = '../../../../outside.md'; +const REMOTE = 'https://example.com/remote.md'; +const QUERIED = './notes.md?raw=1#top'; +const TWO_IN_ONE = 'see a.md and also sub/b.mdx'; +loadDoc('docs/deep/nested.markdown'); diff --git a/__tests__/integration/full-pipeline.test.ts b/__tests__/integration/full-pipeline.test.ts index 5b551c136..fc3aab2e7 100644 --- a/__tests__/integration/full-pipeline.test.ts +++ b/__tests__/integration/full-pipeline.test.ts @@ -82,6 +82,97 @@ describe('Integration: full pipeline', () => { cleanupTempDir(tempDir); }); + it('indexes Markdown headings and resolves Markdown links to script files', async () => { + fs.mkdirSync(path.join(tempDir, 'docs'), { recursive: true }); + fs.mkdirSync(path.join(tempDir, 'scripts'), { recursive: true }); + fs.writeFileSync( + path.join(tempDir, 'README.md'), + `# Project Guide + +See [Setup](docs/setup.md#install). + +## Release + +\`\`\`bash +node scripts/release.mjs +\`\`\` +` + ); + fs.writeFileSync(path.join(tempDir, 'docs', 'setup.md'), '# Install\n'); + fs.writeFileSync(path.join(tempDir, 'scripts', 'release.mjs'), 'export function release() { return true; }\n'); + + const cg = await CodeGraph.init(tempDir); + try { + await cg.indexAll(); + + const guide = cg.searchNodes('Project Guide').find((r) => r.node.language === 'markdown'); + expect(guide).toBeDefined(); + + const releaseCommand = cg + .searchNodes('release.mjs') + .find((r) => r.node.language === 'markdown' && r.node.kind === 'function'); + expect(releaseCommand).toBeDefined(); + + const guideEdges = cg.getOutgoingEdges(guide!.node.id).filter((e) => e.kind === 'imports'); + const guideTargets = guideEdges.map((e) => cg.getNode(e.target)); + const setupHeading = guideTargets.find((n) => n?.qualifiedName === 'docs/setup.md#install'); + expect(setupHeading).toMatchObject({ + kind: 'module', + name: 'Install', + filePath: 'docs/setup.md', + startLine: 1, + }); + + const commandEdges = cg.getOutgoingEdges(releaseCommand!.node.id).filter((e) => e.kind === 'calls'); + const commandTargets = commandEdges.map((e) => cg.getNode(e.target)?.filePath); + expect(commandTargets).toContain('scripts/release.mjs'); + } finally { + cg.destroy(); + } + }); + + it('indexes Markdown template tables and resolves file-symbol references to implementation functions', async () => { + fs.mkdirSync(path.join(tempDir, 'phases'), { recursive: true }); + fs.mkdirSync(path.join(tempDir, 'scripts'), { recursive: true }); + fs.writeFileSync( + path.join(tempDir, 'phases', 'phase4.md'), + `# Phase 4 + +## Fixed Script Templates + +| Template | CLI Entry | Dispatcher | Implementation | +| --- | --- | --- | --- | +| P4-S1 | \`python "{script_path}" p4 "{csv_file}" s1 "{conditions_or_-}" "{probe_cols}"\` | \`scripts/csv_search.py::run_p4\` | \`scripts/csv_search.py::_p4_stage1\` | +| P4-S2 | \`python "{script_path}" p4 "{csv_file}" s2 "{stage1_rows}" "{condition_or_-}" "{detail_cols}"\` | \`scripts/csv_search.py::run_p4\` | \`scripts/csv_search.py::_p4_stage2\` | +` + ); + fs.writeFileSync( + path.join(tempDir, 'scripts', 'csv_search.py'), + `def _p4_stage1(filepath, condition_spec, probe_cols_spec):\n return 's1'\n\n` + + `def _p4_stage2(filepath, stage1_rows, condition_spec, detail_cols_spec):\n return 's2'\n\n` + + `def run_p4(filepath, args):\n return _p4_stage1(filepath, '-', 'MPN')\n` + ); + + const cg = await CodeGraph.init(tempDir); + try { + await cg.indexAll(); + + const p4s1Row = cg.searchNodes('P4-S1').find((r) => r.node.language === 'markdown'); + expect(p4s1Row?.node.kind).toBe('constant'); + + const edges = cg.getOutgoingEdges(p4s1Row!.node.id).filter((e) => e.kind === 'references'); + const targets = edges.map((e) => cg.getNode(e.target)); + expect(targets).toEqual( + expect.arrayContaining([ + expect.objectContaining({ name: 'run_p4', filePath: 'scripts/csv_search.py' }), + expect.objectContaining({ name: '_p4_stage1', filePath: 'scripts/csv_search.py' }), + ]) + ); + } finally { + cg.destroy(); + } + }); + it('runs init → index → resolve → search → callers → context → sync', async () => { const MODULE_COUNT = 120; generateSyntheticProject(tempDir, MODULE_COUNT); @@ -269,4 +360,46 @@ describe('Integration: full pipeline', () => { cg.destroy(); } }, 30_000); + + it('resolves code string references to Markdown headings', async () => { + fs.mkdirSync(path.join(tempDir, 'docs'), { recursive: true }); + fs.mkdirSync(path.join(tempDir, 'scripts'), { recursive: true }); + fs.writeFileSync( + path.join(tempDir, 'docs', 'guide.md'), + `# Guide + +## Install + +Run the setup command. +` + ); + fs.writeFileSync( + path.join(tempDir, 'scripts', 'load_docs.py'), + `GUIDE = "docs/guide.md"\n\n` + + `def load_docs():\n` + + ` return open("docs/guide.md#install", encoding="utf-8").read()\n` + ); + + const cg = await CodeGraph.init(tempDir); + try { + await cg.indexAll(); + + const loadDocs = cg.searchNodes('load_docs').find((r) => r.node.language === 'python'); + expect(loadDocs).toBeDefined(); + + const edges = cg.getOutgoingEdges(loadDocs!.node.id).filter((e) => e.kind === 'references'); + const targets = edges.map((e) => cg.getNode(e.target)); + expect(targets).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + kind: 'module', + name: 'Install', + qualifiedName: 'docs/guide.md#install', + }), + ]) + ); + } finally { + cg.destroy(); + } + }); }); diff --git a/__tests__/resolution.test.ts b/__tests__/resolution.test.ts index decaadee5..794c8a1a0 100644 --- a/__tests__/resolution.test.ts +++ b/__tests__/resolution.test.ts @@ -358,6 +358,169 @@ describe('Resolution Module', () => { expect(result).not.toBeNull(); expect(result?.targetNodeId).toBe('method:user.ts:User.save:15'); }); + + it('should resolve Markdown file references by filename, path, and anchor suffix', () => { + const mockNodes: Node[] = [ + { + id: 'file:README.md', + kind: 'file', + name: 'README.md', + qualifiedName: 'README.md', + filePath: 'README.md', + language: 'markdown', + startLine: 1, + endLine: 10, + startColumn: 0, + endColumn: 0, + updatedAt: Date.now(), + }, + { + id: 'file:docs/setup.md', + kind: 'file', + name: 'setup.md', + qualifiedName: 'docs/setup.md', + filePath: 'docs/setup.md', + language: 'markdown', + startLine: 1, + endLine: 10, + startColumn: 0, + endColumn: 0, + updatedAt: Date.now(), + }, + { + id: 'module:docs/setup.md:install:1', + kind: 'module', + name: 'Install', + qualifiedName: 'docs/setup.md#install', + filePath: 'docs/setup.md', + language: 'markdown', + startLine: 1, + endLine: 10, + startColumn: 0, + endColumn: 9, + updatedAt: Date.now(), + }, + { + id: 'file:GUIDE.markdown', + kind: 'file', + name: 'GUIDE.markdown', + qualifiedName: 'GUIDE.markdown', + filePath: 'GUIDE.markdown', + language: 'markdown', + startLine: 1, + endLine: 10, + startColumn: 0, + endColumn: 0, + updatedAt: Date.now(), + }, + { + id: 'module:GUIDE.markdown:install:1', + kind: 'module', + name: 'Install', + qualifiedName: 'GUIDE.markdown#install', + filePath: 'GUIDE.markdown', + language: 'markdown', + startLine: 1, + endLine: 10, + startColumn: 0, + endColumn: 9, + updatedAt: Date.now(), + }, + ]; + + const context: ResolutionContext = { + getNodesInFile: () => mockNodes, + getNodesByName: (name) => mockNodes.filter((n) => n.name === name), + getNodesByQualifiedName: (qualifiedName) => mockNodes.filter((n) => n.qualifiedName === qualifiedName), + getNodesByKind: () => [], + fileExists: () => true, + readFile: () => null, + getProjectRoot: () => '/test', + getAllFiles: () => ['README.md', 'docs/setup.md', 'GUIDE.markdown'], + getNodesByLowerName: () => [], + getImportMappings: () => [], + }; + + const readmeRef = { + fromNodeId: 'module:docs/setup.md:install:1', + referenceName: 'README.md', + referenceKind: 'imports' as const, + line: 1, + column: 0, + filePath: 'docs/setup.md', + language: 'markdown' as const, + }; + const setupRef = { + ...readmeRef, + referenceName: 'docs/setup.md#install', + filePath: 'README.md', + }; + const markdownRef = { + ...readmeRef, + referenceName: 'GUIDE.markdown#install', + filePath: 'README.md', + }; + + expect(matchReference(readmeRef, context)?.targetNodeId).toBe('file:README.md'); + expect(matchReference(setupRef, context)?.targetNodeId).toBe('module:docs/setup.md:install:1'); + expect(matchReference(markdownRef, context)?.targetNodeId).toBe('module:GUIDE.markdown:install:1'); + }); + + it('should resolve Markdown file-symbol references to symbols in the referenced file', () => { + const mockNodes: Node[] = [ + { + id: 'file:scripts/csv_search.py', + kind: 'file', + name: 'csv_search.py', + qualifiedName: 'scripts/csv_search.py', + filePath: 'scripts/csv_search.py', + language: 'python', + startLine: 1, + endLine: 100, + startColumn: 0, + endColumn: 0, + updatedAt: Date.now(), + }, + { + id: 'function:scripts/csv_search.py:run_p4:40', + kind: 'function', + name: 'run_p4', + qualifiedName: 'scripts/csv_search.py::run_p4', + filePath: 'scripts/csv_search.py', + language: 'python', + startLine: 40, + endLine: 55, + startColumn: 0, + endColumn: 0, + updatedAt: Date.now(), + }, + ]; + + const context: ResolutionContext = { + getNodesInFile: (filePath) => mockNodes.filter((n) => n.filePath === filePath), + getNodesByName: (name) => mockNodes.filter((n) => n.name === name), + getNodesByQualifiedName: (qualifiedName) => mockNodes.filter((n) => n.qualifiedName === qualifiedName), + getNodesByKind: () => [], + fileExists: () => true, + readFile: () => null, + getProjectRoot: () => '/test', + getAllFiles: () => ['scripts/csv_search.py'], + getNodesByLowerName: () => [], + getImportMappings: () => [], + }; + + const ref = { + fromNodeId: 'constant:phases/phase4.md:P4-S1:10', + referenceName: 'phases/scripts/csv_search.py::run_p4', + referenceKind: 'references' as const, + line: 10, + column: 20, + filePath: 'phases/phase4.md', + language: 'markdown' as const, + }; + + expect(matchReference(ref, context)?.targetNodeId).toBe('function:scripts/csv_search.py:run_p4:40'); + }); }); describe('Ubiquitous-name ceiling (#999)', () => { diff --git a/__tests__/security.test.ts b/__tests__/security.test.ts index 9aa3c9597..ccda3671b 100644 --- a/__tests__/security.test.ts +++ b/__tests__/security.test.ts @@ -513,11 +513,12 @@ describe('Source file detection (isSourceFile)', () => { expect(isSourceFile('src/component.tsx')).toBe(true); expect(isSourceFile('lib/util.js')).toBe(true); expect(isSourceFile('src/main.py')).toBe(true); + // Markdown documentation is indexed as a source language. + expect(isSourceFile('README.md')).toBe(true); }); it('rejects unsupported extensions and extensionless files', () => { expect(isSourceFile('src/component.css')).toBe(false); - expect(isSourceFile('README.md')).toBe(false); expect(isSourceFile('Makefile')).toBe(false); expect(isSourceFile('.gitignore')).toBe(false); }); diff --git a/__tests__/watcher.test.ts b/__tests__/watcher.test.ts index 942fd5bcd..067f70f22 100644 --- a/__tests__/watcher.test.ts +++ b/__tests__/watcher.test.ts @@ -442,8 +442,8 @@ describe('FileWatcher', () => { // gate must drop it before scheduling sync. (It must exist on disk: // a VANISHED non-source path is the deleted-directory shape, which // deliberately schedules a sync — #1285.) - fs.writeFileSync(path.join(testDir, 'src', 'readme.md'), '# docs\n'); - __emitWatchEventForTests(testDir, 'src/readme.md'); + fs.writeFileSync(path.join(testDir, 'src', 'styles.css'), 'body {}\n'); + __emitWatchEventForTests(testDir, 'src/styles.css'); // Wait a bit longer than debounce — sync should NOT trigger. await new Promise((r) => setTimeout(r, 400)); diff --git a/codegraph-kernel/src/buffers.rs b/codegraph-kernel/src/buffers.rs index f9dd70d5b..80b27b817 100644 --- a/codegraph-kernel/src/buffers.rs +++ b/codegraph-kernel/src/buffers.rs @@ -125,6 +125,10 @@ pub const FUNCTION_REF_CODE: u8 = 200; /// Ref-row flag bit 0: the ref carries `filePath` = the extracted file. pub const REF_FLAG_FILE_PATH: u8 = 1; +/// The ref carries the file's language, as `addReference` (tree-sitter.ts) +/// emits it. Ordinary refs must NOT set this: their wasm counterparts have no +/// `language` field and parity compares the objects whole. +pub const REF_FLAG_LANGUAGE: u8 = 2; pub fn node_kind_index(kind: &str) -> Option { NODE_KINDS.iter().position(|k| *k == kind).map(|i| i as u8) diff --git a/codegraph-kernel/src/ccpp/mod.rs b/codegraph-kernel/src/ccpp/mod.rs index 78877d7da..84de0a13d 100644 --- a/codegraph-kernel/src/ccpp/mod.rs +++ b/codegraph-kernel/src/ccpp/mod.rs @@ -437,6 +437,8 @@ pub fn extract(file_path: &str, source: &str, language: &str) -> Result Walker<'t> { + markdown_refs_impl!(); + fn text(&self, node: Node) -> &'t str { &self.src[node.byte_range()] } @@ -800,6 +802,8 @@ impl<'t> Walker<'t> { } self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "function_definition" { // functionTypes for both; cpp's methodTypes also lists it, so @@ -1517,6 +1521,8 @@ impl<'t> Walker<'t> { stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "call_expression" { self.extract_call(node); diff --git a/codegraph-kernel/src/csharp.rs b/codegraph-kernel/src/csharp.rs index f370a0e54..7f5b1725d 100644 --- a/codegraph-kernel/src/csharp.rs +++ b/codegraph-kernel/src/csharp.rs @@ -264,6 +264,8 @@ fn record_is_struct(node: Node) -> bool { } impl<'t> Walker<'t> { + markdown_refs_impl!(); + fn text(&self, node: Node) -> &'t str { &self.src[node.byte_range()] } @@ -505,6 +507,8 @@ impl<'t> Walker<'t> { let mut skip_children = false; self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "class_declaration" || kind == "record_declaration" { // classifyClassNode: `record struct` → extractStruct, else class. @@ -580,6 +584,8 @@ impl<'t> Walker<'t> { stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "invocation_expression" { self.extract_call(node); @@ -873,6 +879,9 @@ impl<'t> Walker<'t> { // multi-declarator fields emit the type refs once PER // declarator, each from its own field node. self.extract_csharp_type_refs(node, row); + // The ladder skips a field's children, so the declarator's + // string literals are reached here, owned by the field. + self.markdown_refs_from_subtree(decl, row); } } } else { @@ -884,12 +893,15 @@ impl<'t> Walker<'t> { }); if let Some(name_node) = name_node { let name = self.text(name_node).to_string(); - self.create_node( + let row = self.create_node( field_kind, &name, node, Extra { docstring, visibility, is_static, ..Extra::default() }, ); + if let Some(row) = row { + self.markdown_refs_from_subtree(node, row); + } } } } diff --git a/codegraph-kernel/src/dart.rs b/codegraph-kernel/src/dart.rs index 748410b10..bddae38dc 100644 --- a/codegraph-kernel/src/dart.rs +++ b/codegraph-kernel/src/dart.rs @@ -213,6 +213,8 @@ pub fn extract(file_path: &str, source: &str) -> Result { } impl<'t> Walker<'t> { + markdown_refs_impl!(); + fn text(&self, node: Node) -> &'t str { &self.src[node.byte_range()] } @@ -621,6 +623,8 @@ impl<'t> Walker<'t> { // maybeCaptureFnRefs (:990) — the double-walk fn-ref twin source. self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); match node.kind() { "function_signature" => { @@ -1246,6 +1250,8 @@ impl<'t> Walker<'t> { fn visit_body(&mut self, node: Node<'t>) { stack_guard!(); self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); let kind = node.kind(); if kind == "new_expression" { diff --git a/codegraph-kernel/src/go.rs b/codegraph-kernel/src/go.rs index 66bb352f1..7a7a0a898 100644 --- a/codegraph-kernel/src/go.rs +++ b/codegraph-kernel/src/go.rs @@ -177,6 +177,8 @@ pub fn extract(file_path: &str, source: &str) -> Result { } impl<'t> Walker<'t> { + markdown_refs_impl!(); + fn text(&self, node: Node) -> &'t str { &self.src[node.byte_range()] } @@ -385,6 +387,8 @@ impl<'t> Walker<'t> { let mut skip_children = false; self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "function_declaration" { self.extract_function(node); @@ -426,6 +430,8 @@ impl<'t> Walker<'t> { stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "call_expression" { self.extract_call(node); diff --git a/codegraph-kernel/src/java.rs b/codegraph-kernel/src/java.rs index 7065ab565..71fa816be 100644 --- a/codegraph-kernel/src/java.rs +++ b/codegraph-kernel/src/java.rs @@ -247,6 +247,8 @@ pub fn extract(file_path: &str, source: &str) -> Result { } impl<'t> Walker<'t> { + markdown_refs_impl!(); + fn text(&self, node: Node) -> &'t str { &self.src[node.byte_range()] } @@ -490,6 +492,8 @@ impl<'t> Walker<'t> { let mut skip_children = false; self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "class_declaration" { self.extract_class(node); @@ -543,6 +547,8 @@ impl<'t> Walker<'t> { stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "method_invocation" { self.extract_call(node); @@ -759,6 +765,9 @@ impl<'t> Walker<'t> { if let Some(row) = row { self.extract_decorators_for(node, row); self.extract_type_annotations(node, row); + // The ladder skips a field's children, so the declarator's + // string literals are reached here, owned by the field. + self.markdown_refs_from_subtree(decl, row); } } } else { @@ -769,12 +778,15 @@ impl<'t> Walker<'t> { }); if let Some(name_node) = name_node { let name = self.text(name_node).to_string(); - self.create_node( + let row = self.create_node( field_kind, &name, node, Extra { docstring, visibility, is_static, ..Extra::default() }, ); + if let Some(row) = row { + self.markdown_refs_from_subtree(node, row); + } } } } diff --git a/codegraph-kernel/src/kotlin.rs b/codegraph-kernel/src/kotlin.rs index 58fde4285..7b6b6ae94 100644 --- a/codegraph-kernel/src/kotlin.rs +++ b/codegraph-kernel/src/kotlin.rs @@ -250,6 +250,8 @@ pub fn extract(file_path: &str, source: &str) -> Result { } impl<'t> Walker<'t> { + markdown_refs_impl!(); + fn text(&self, node: Node) -> &'t str { &self.src[node.byte_range()] } @@ -658,6 +660,8 @@ impl<'t> Walker<'t> { let mut skip_children = false; self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "function_declaration" { if self.inside_class_like() { @@ -728,6 +732,8 @@ impl<'t> Walker<'t> { stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "call_expression" { self.extract_call(node); diff --git a/codegraph-kernel/src/lib.rs b/codegraph-kernel/src/lib.rs index 7411fce0f..122406e90 100644 --- a/codegraph-kernel/src/lib.rs +++ b/codegraph-kernel/src/lib.rs @@ -30,6 +30,68 @@ macro_rules! stack_guard { }; } +/// The markdown path-reference pair, for a walker with the usual shape +/// (`text`, `line_of`, `col_of`, `arena`, `tables`, `file_path`). Every routed +/// language needs the same two methods, and the wasm arm they must match is +/// one implementation, so this is one implementation too — see markdown.rs for +/// what it mirrors and why the two ref flags are set only here. +macro_rules! markdown_refs_impl { + () => { + /// extractMarkdownPathReferencesFromStringNode. + fn markdown_refs_from_string(&mut self, node: Node<'t>, owner_row: u32) { + if !$crate::markdown::is_markdown_path_string_kind(node.kind()) { + return; + } + let found = $crate::markdown::markdown_path_refs(self.text(node), self.file_path); + if found.is_empty() { + return; + } + let kind_code = $crate::buffers::edge_kind_index("references").unwrap(); + let line = self.line_of(node); + let column = self.col_of(node); + for (name, offset) in found { + let name_ref = self.arena.put(&name); + // addReference denormalizes filePath and language onto the ref + // where the ordinary ref path does not, so these two flags are + // set here and nowhere else. + self.tables.push_ref_flagged( + &$crate::buffers::RefRow { + from_idx: owner_row, + kind: kind_code, + line, + column: column + offset as u32, + reference_name: name_ref, + candidates: $crate::buffers::NONE_STR, + from_id_str: $crate::buffers::NONE_STR, + }, + $crate::buffers::REF_FLAG_FILE_PATH | $crate::buffers::REF_FLAG_LANGUAGE, + ); + } + } + + /// extractMarkdownPathReferencesFromSubtree — for constructs whose walk + /// stops before their value, where the owner is the declared symbol + /// rather than the enclosing scope. + /// + /// Only some walkers stop that way (tsjs, python, java, csharp, ruby, + /// lua); in the rest a declaration's children are walked normally and + /// the string method above already covers them, so the pair is one + /// macro and this half goes unused there. The parity fixtures decide + /// which is which — every language's torture file carries the same + /// eight markdown shapes. + #[allow(dead_code)] + fn markdown_refs_from_subtree(&mut self, node: Node<'t>, owner_row: u32) { + stack_guard!(); + self.markdown_refs_from_string(node, owner_row); + for i in 0..node.named_child_count() { + if let Some(c) = node.named_child(i) { + self.markdown_refs_from_subtree(c, owner_row); + } + } + } + }; +} + mod buffers; mod ccpp; mod cfnptr; @@ -42,6 +104,7 @@ mod java; mod kotlin; mod langs; mod lua; +mod markdown; mod php; mod rlang; mod ruby; diff --git a/codegraph-kernel/src/lua.rs b/codegraph-kernel/src/lua.rs index b3cd32a76..8fb069fb1 100644 --- a/codegraph-kernel/src/lua.rs +++ b/codegraph-kernel/src/lua.rs @@ -157,6 +157,8 @@ pub fn extract(file_path: &str, source: &str, language: &str) -> Result Walker<'t> { + markdown_refs_impl!(); + fn text(&self, node: Node) -> &'t str { &self.src[node.byte_range()] } @@ -454,6 +456,8 @@ impl<'t> Walker<'t> { // maybeCaptureFnRefs (tree-sitter.ts:990). self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); // The dispatch ladder — lua/luau rows only. if kind == "function_declaration" { @@ -593,7 +597,7 @@ impl<'t> Walker<'t> { // Positional value pairing; a missing value → NO signature key. let signature = values.get(i).map(|v| util::init_signature(self.text(*v))); let name = name.to_string(); - self.create_node( + let row = self.create_node( "variable", &name, *name_node, // positioned at the IDENTIFIER @@ -604,6 +608,12 @@ impl<'t> Walker<'t> { ..Default::default() }, ); + // The ladder skips a declaration's children, so the positionally + // paired value's string literals are reached here. A name with no + // value contributes nothing, as the wasm arm's undefined does. + if let (Some(row), Some(v)) = (row, values.get(i)) { + self.markdown_refs_from_subtree(*v, row); + } } } @@ -660,6 +670,8 @@ impl<'t> Walker<'t> { stack_guard!(); // maybeCaptureFnRefs (5137) fires in the body walker too. self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); let kind = node.kind(); if kind == "function_call" { diff --git a/codegraph-kernel/src/markdown.rs b/codegraph-kernel/src/markdown.rs new file mode 100644 index 000000000..7e507745b --- /dev/null +++ b/codegraph-kernel/src/markdown.rs @@ -0,0 +1,204 @@ +//! Markdown path references from code string literals — the kernel half of the +//! code → documentation edge (`open("docs/guide.md#install")`). +//! +//! The wasm arm builds these in `TreeSitterExtractor`; a routed language never +//! reaches it, so without this the edge is absent for every kernel language. +//! `kernel-tsjs-parity` compares refs byte for byte, so the candidate regex and +//! the normalizer below mirror `extractMarkdownPathCandidates` and +//! `normalizeMarkdownPathReference` (src/extraction/tree-sitter.ts) exactly — +//! including the rejections. Change one side and the parity gate fails. + +use regex::Regex; +use std::sync::OnceLock; + +/// MARKDOWN_PATH_STRING_NODE_TYPES. +pub fn is_markdown_path_string_kind(kind: &str) -> bool { + matches!( + kind, + "string" + | "string_literal" + | "template_string" + | "raw_string_literal" + | "interpreted_string_literal" + | "interpolated_string_expression" + ) +} + +fn candidate_re() -> &'static Regex { + static RE: OnceLock = OnceLock::new(); + RE.get_or_init(|| { + Regex::new( + r#"(?i)((?:\.{1,2}[\\/]+|[A-Za-z0-9_.@-]+[\\/]+|[\\/]+)?(?:[A-Za-z0-9_.@-]+[\\/]+)*[A-Za-z0-9_.@-]+\.(?:md|mdx|markdown)(?:\?[^'"`\s)>,;]*)?(?:#[^'"`\s)>,;]*)?)"#, + ) + .expect("markdown path candidate regex") + }) +} + +fn scheme_re() -> &'static Regex { + static RE: OnceLock = OnceLock::new(); + RE.get_or_init(|| Regex::new(r"(?i)^[a-z][a-z0-9+.-]*://").expect("scheme regex")) +} + +/// `decodeURIComponent`, falling back to the raw value when it would throw. +fn decode_path(value: &str) -> String { + let bytes = value.as_bytes(); + let mut out: Vec = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'%' { + if i + 2 >= bytes.len() { + return value.to_string(); + } + let hex = match std::str::from_utf8(&bytes[i + 1..i + 3]) { + Ok(h) => h, + Err(_) => return value.to_string(), + }; + match u8::from_str_radix(hex, 16) { + Ok(b) => out.push(b), + Err(_) => return value.to_string(), + } + i += 3; + } else { + out.push(bytes[i]); + i += 1; + } + } + String::from_utf8(out).unwrap_or_else(|_| value.to_string()) +} + +/// `path.posix.normalize` for the shapes a doc path takes. +fn posix_normalize(p: &str) -> String { + let is_abs = p.starts_with('/'); + let mut out: Vec<&str> = Vec::new(); + for seg in p.split('/') { + match seg { + "" | "." => {} + ".." => { + if matches!(out.last(), Some(&last) if last != "..") { + out.pop(); + } else if !is_abs { + out.push(".."); + } + } + s => out.push(s), + } + } + let joined = out.join("/"); + if is_abs { + return format!("/{joined}"); + } + if joined.is_empty() { + return ".".to_string(); + } + joined +} + +fn ends_with_md(path: &str) -> bool { + let lower = path.to_ascii_lowercase(); + lower.ends_with(".md") || lower.ends_with(".mdx") || lower.ends_with(".markdown") +} + +/// `normalizeMarkdownPathReference` — None where the TS returns null. +fn normalize(reference_name: &str, file_path: &str) -> Option { + let trimmed = reference_name.trim().replace('\\', "/"); + if trimmed.is_empty() || scheme_re().is_match(&trimmed) { + return None; + } + + let (path_with_query, anchor) = match trimmed.find('#') { + Some(i) => (&trimmed[..i], &trimmed[i..]), + None => (trimmed.as_str(), ""), + }; + let raw_path = match path_with_query.find('?') { + Some(i) => &path_with_query[..i], + None => path_with_query, + }; + let clean_path = decode_path(raw_path); + + if !ends_with_md(&clean_path) { + return None; + } + + let file_posix = file_path.replace('\\', "/"); + let base_dir = match file_posix.rfind('/') { + Some(0) => "/".to_string(), + Some(i) => file_posix[..i].to_string(), + None => ".".to_string(), + }; + + let normalized_path = if let Some(rest) = clean_path.strip_prefix('/') { + posix_normalize(rest.trim_start_matches('/')) + } else if clean_path.starts_with("./") || clean_path.starts_with("../") { + let joined = if base_dir == "." { + clean_path.clone() + } else { + format!("{base_dir}/{clean_path}") + }; + posix_normalize(&joined) + } else { + posix_normalize(&clean_path) + }; + + if normalized_path.is_empty() + || normalized_path == "." + || normalized_path == ".." + || normalized_path.starts_with("../") + { + return None; + } + + Some(format!("{normalized_path}{anchor}")) +} + +/// Every markdown path reference in one string literal's text, as +/// (normalized name, byte offset of the match within `text`). The offset is +/// added to the literal's own column, mirroring the wasm arm. +pub fn markdown_path_refs(text: &str, file_path: &str) -> Vec<(String, usize)> { + let mut refs = Vec::new(); + for m in candidate_re().find_iter(text) { + // A `scheme://host/x.md` URL is not a repo path. The wasm arm looks + // back 16 chars for the `://` rather than matching it, because the + // candidate pattern starts after the scheme. + let start = m.start(); + let prefix = &text[start.saturating_sub(16)..start]; + if prefix.ends_with("://") { + continue; + } + if let Some(name) = normalize(m.as_str(), file_path) { + refs.push((name, start)); + } + } + refs +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn resolves_relative_paths_against_the_file() { + let refs = markdown_path_refs("'../docs/guide.md#install'", "src/load-docs.ts"); + assert_eq!(refs.len(), 1); + assert_eq!(refs[0].0, "docs/guide.md#install"); + } + + #[test] + fn rejects_an_escape_above_the_repo_root() { + assert!(markdown_path_refs("'../../../up.md'", "src/x.ts").is_empty()); + } + + #[test] + fn a_scheme_prefixed_path_is_whatever_the_wasm_arm_makes_of_it() { + // Not asserted either way here on purpose. The candidate pattern can + // start inside a URL (at the `//`), so the `://` look-back does not + // always fire, and the wasm arm — not a guess about it — is the spec. + // `kernel-tsjs-parity` pins this against a torture fixture instead. + let _ = markdown_path_refs("'https://example.com/a.md'", "src/x.ts"); + } + + #[test] + fn keeps_a_bare_name_and_an_anchor() { + let refs = markdown_path_refs("'README.md'", "src/x.ts"); + assert_eq!(refs[0].0, "README.md"); + } +} diff --git a/codegraph-kernel/src/php.rs b/codegraph-kernel/src/php.rs index 13a8626b6..a6b92c3ff 100644 --- a/codegraph-kernel/src/php.rs +++ b/codegraph-kernel/src/php.rs @@ -263,6 +263,8 @@ pub fn extract(file_path: &str, source: &str) -> Result { } impl<'t> Walker<'t> { + markdown_refs_impl!(); + fn text(&self, node: Node) -> &'t str { &self.src[node.byte_range()] } @@ -540,6 +542,8 @@ impl<'t> Walker<'t> { let mut skip_children = false; self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "function_definition" { // functionTypes; method_declaration is not in it, so this is @@ -618,6 +622,8 @@ impl<'t> Walker<'t> { stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if matches!( kind, diff --git a/codegraph-kernel/src/python.rs b/codegraph-kernel/src/python.rs index b2397facd..b596bbeed 100644 --- a/codegraph-kernel/src/python.rs +++ b/codegraph-kernel/src/python.rs @@ -325,6 +325,8 @@ impl<'t> Walker<'t> { let mut skip_children = false; self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "function_definition" { // functionTypes ∩ methodTypes: inside a class-like ⇒ method. @@ -365,6 +367,8 @@ impl<'t> Walker<'t> { stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "call" { self.extract_call(node); @@ -470,6 +474,8 @@ impl<'t> Walker<'t> { self.stack.pop(); } + markdown_refs_impl!(); + /// extractVariable's python branch: `left = right` at module scope. fn extract_variable(&mut self, node: Node<'t>) { let docstring = preceding_docstring(node, self.src); @@ -482,7 +488,12 @@ impl<'t> Walker<'t> { let name = self.text(left).to_string(); let signature = right.map(|r| util::init_signature(self.text(r))); // No isConst hook ⇒ always `variable` (UPPER_CASE constants included). - self.create_node("variable", &name, node, Extra { docstring, signature, ..Extra::default() }); + let row = self.create_node("variable", &name, node, Extra { docstring, signature, ..Extra::default() }); + // visit_node skips an assignment's children, so the right-hand side's + // string literals are reached here, owned by the assigned name. + if let (Some(row), Some(r)) = (row, right) { + self.markdown_refs_from_subtree(r, row); + } } fn extract_import(&mut self, node: Node<'t>) { diff --git a/codegraph-kernel/src/rlang.rs b/codegraph-kernel/src/rlang.rs index 9c85f0cf2..c15bd4bca 100644 --- a/codegraph-kernel/src/rlang.rs +++ b/codegraph-kernel/src/rlang.rs @@ -141,6 +141,8 @@ pub fn extract(file_path: &str, source: &str) -> Result { } impl<'t> Walker<'t> { + markdown_refs_impl!(); + fn text(&self, node: Node) -> &'t str { &self.src[node.byte_range()] } @@ -302,6 +304,8 @@ impl<'t> Walker<'t> { if self.hook(node) { return; } + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if node.kind() == "call" { self.extract_call(node); } diff --git a/codegraph-kernel/src/ruby.rs b/codegraph-kernel/src/ruby.rs index 35cd2a713..0368e51e2 100644 --- a/codegraph-kernel/src/ruby.rs +++ b/codegraph-kernel/src/ruby.rs @@ -204,6 +204,8 @@ pub fn extract(file_path: &str, source: &str) -> Result { } impl<'t> Walker<'t> { + markdown_refs_impl!(); + fn text(&self, node: Node) -> &'t str { &self.src[node.byte_range()] } @@ -465,6 +467,8 @@ impl<'t> Walker<'t> { let mut skip_children = false; self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "method" { // functionTypes ∩ methodTypes: inside class-like (module counts!) @@ -532,6 +536,8 @@ impl<'t> Walker<'t> { stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "call" { self.extract_call(node); @@ -689,7 +695,13 @@ impl<'t> Walker<'t> { } let name = self.text(left).to_string(); let signature = right.map(|r| util::init_signature(self.text(r))); - self.create_node("variable", &name, node, Extra { docstring, signature, visibility: None }); + let row = + self.create_node("variable", &name, node, Extra { docstring, signature, visibility: None }); + // visit_node skips an assignment's children, so the right-hand side's + // string literals are reached here, owned by the assigned name. + if let (Some(row), Some(r)) = (row, right) { + self.markdown_refs_from_subtree(r, row); + } } /// extractImport — every non-body `call` lands here (importTypes:['call']). diff --git a/codegraph-kernel/src/rustlang.rs b/codegraph-kernel/src/rustlang.rs index 3e880fa11..f3530a6f3 100644 --- a/codegraph-kernel/src/rustlang.rs +++ b/codegraph-kernel/src/rustlang.rs @@ -201,6 +201,8 @@ pub fn extract(file_path: &str, source: &str) -> Result { } impl<'t> Walker<'t> { + markdown_refs_impl!(); + fn text(&self, node: Node) -> &'t str { &self.src[node.byte_range()] } @@ -445,6 +447,8 @@ impl<'t> Walker<'t> { let mut skip_children = false; self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if matches!(kind, "function_item" | "function_signature_item") { self.extract_fn_or_method(node); @@ -1128,6 +1132,8 @@ impl<'t> Walker<'t> { stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); // Rocket route macros: handler paths live in a raw token tree. if kind == "macro_invocation" { diff --git a/codegraph-kernel/src/scala.rs b/codegraph-kernel/src/scala.rs index fc80feae7..d7bad35d8 100644 --- a/codegraph-kernel/src/scala.rs +++ b/codegraph-kernel/src/scala.rs @@ -245,6 +245,8 @@ pub fn extract(file_path: &str, source: &str) -> Result { } impl<'t> Walker<'t> { + markdown_refs_impl!(); + fn text(&self, node: Node) -> &'t str { &self.src[node.byte_range()] } @@ -540,6 +542,8 @@ impl<'t> Walker<'t> { // maybeCaptureFnRefs (:990). self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); let kind = node.kind(); match kind { @@ -1162,6 +1166,8 @@ impl<'t> Walker<'t> { fn visit_body(&mut self, node: Node<'t>) { stack_guard!(); self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); let kind = node.kind(); if kind == "call_expression" { diff --git a/codegraph-kernel/src/swift.rs b/codegraph-kernel/src/swift.rs index 7d46c4922..17993a958 100644 --- a/codegraph-kernel/src/swift.rs +++ b/codegraph-kernel/src/swift.rs @@ -279,6 +279,8 @@ fn last_simple_identifier<'t>(node: Node<'t>) -> Option> { } impl<'t> Walker<'t> { + markdown_refs_impl!(); + fn text(&self, node: Node) -> &'t str { &self.src[node.byte_range()] } @@ -566,6 +568,8 @@ impl<'t> Walker<'t> { let mut skip_children = false; self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "function_declaration" { if self.inside_class_like() { @@ -730,6 +734,8 @@ impl<'t> Walker<'t> { stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "call_expression" { self.extract_call(node); diff --git a/codegraph-kernel/src/tsjs/extractors.rs b/codegraph-kernel/src/tsjs/extractors.rs index f240c5e79..46833cf75 100644 --- a/codegraph-kernel/src/tsjs/extractors.rs +++ b/codegraph-kernel/src/tsjs/extractors.rs @@ -385,6 +385,12 @@ impl<'t> Walker<'t> { ); if let Some(row) = var_row { self.extract_variable_type_annotation(child, row); + // The declarator's walk stops here (visit_node skips a + // variable's children), so the initializer's string literals + // are reached from this side, owned by the declared symbol. + if let Some(v) = value { + self.markdown_refs_from_subtree(v, row); + } } // Exported const object-of-functions / store shapes. diff --git a/codegraph-kernel/src/tsjs/mod.rs b/codegraph-kernel/src/tsjs/mod.rs index afe6361d5..3fa2622c5 100644 --- a/codegraph-kernel/src/tsjs/mod.rs +++ b/codegraph-kernel/src/tsjs/mod.rs @@ -317,6 +317,8 @@ impl<'t> Walker<'t> { self.push_ref(self.top_row(), name, edge_kind_index("calls").unwrap(), node); } + markdown_refs_impl!(); + // --- createNode ----------------------------------------------------------- /// createNode (tree-sitter.ts): id, qualified name from the scope stack, @@ -614,6 +616,8 @@ impl<'t> Walker<'t> { // Function-as-value capture — independent of the dispatch ladder. self.maybe_capture_fn_refs(node); + let owner = self.top_row(); + self.markdown_refs_from_string(node, owner); if is_function_type(kind) { // (the isInsideClassLike + methodTypes overlap is Python/Ruby-only) @@ -691,6 +695,8 @@ impl<'t> Walker<'t> { stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "call_expression" { self.extract_call(node); diff --git a/site/src/content/docs/reference/languages.md b/site/src/content/docs/reference/languages.md index 0c5587773..f6b59ab94 100644 --- a/site/src/content/docs/reference/languages.md +++ b/site/src/content/docs/reference/languages.md @@ -31,3 +31,6 @@ Language support is automatic from the file extension — there's nothing to con | Lua | `.lua` | Full support (functions, methods, locals, `require` imports, call edges) | | R | `.R`, `.r` | Full support (functions, S4/R5/R6 classes with methods, `library`/`require` imports, `source()` file references, call edges) | | Luau | `.luau` | Full support (Lua, plus typed signatures, `type` aliases, Roblox `require`) | +| Markdown | `.md`, `.mdx`, `.markdown` | Documentation structure (headings, sections, local links, selected table rows/list items, shell command references) | + +Markdown files use a dedicated documentation extractor. `.mdx` files receive the same documentation indexing; embedded JSX and JavaScript are not parsed as MDX code. diff --git a/src/extraction/generated-detection.ts b/src/extraction/generated-detection.ts index b92ab860b..03116cfb5 100644 --- a/src/extraction/generated-detection.ts +++ b/src/extraction/generated-detection.ts @@ -249,5 +249,9 @@ export function hasGeneratedHeader(content: string): boolean { * indexer persists to `files.generated`. */ export function detectGeneratedFile(filePath: string, content: string): boolean { + // Markdown carries no banner: its `#` is a heading, not a comment leader, + // so a README that DESCRIBES generated code ("Code generated … DO NOT EDIT" + // under a heading) would flag itself. The path convention still applies. + if (/\.(?:md|markdown)$/i.test(filePath)) return isGeneratedFile(filePath); return isGeneratedFile(filePath) || hasGeneratedHeader(content); } diff --git a/src/extraction/grammars.ts b/src/extraction/grammars.ts index c7710f200..37ad0e0a4 100644 --- a/src/extraction/grammars.ts +++ b/src/extraction/grammars.ts @@ -11,7 +11,7 @@ import * as fsp from 'fs/promises'; import { Parser, Language as WasmLanguage } from 'web-tree-sitter'; import { Language } from '../types'; -export type GrammarLanguage = Exclude; +export type GrammarLanguage = Exclude; /** * WASM filename map — maps each language to its .wasm grammar file @@ -100,6 +100,9 @@ export const EXTENSION_MAP: Record = { '.yaml': 'yaml', // Twig templates (file-level tracking only, no symbol extraction) '.twig': 'twig', + '.md': 'markdown', + '.mdx': 'markdown', + '.markdown': 'markdown', '.rb': 'ruby', '.rake': 'ruby', '.swift': 'swift', @@ -588,6 +591,7 @@ export function isLanguageSupported(language: Language): boolean { if (language === 'twig') return true; // file-level tracking only if (language === 'xml') return true; // MyBatis mapper extractor if (language === 'properties') return true; // Spring config keys + if (language === 'markdown') return true; // custom documentation extractor if (language === 'unknown') return false; return language in WASM_GRAMMAR_FILES; } @@ -596,7 +600,7 @@ export function isLanguageSupported(language: Language): boolean { * Check if a grammar has been loaded and is ready for parsing. */ export function isGrammarLoaded(language: Language): boolean { - if (language === 'svelte' || language === 'vue' || language === 'astro' || language === 'liquid' || language === 'razor') return true; + if (language === 'svelte' || language === 'vue' || language === 'astro' || language === 'liquid' || language === 'razor' || language === 'markdown') return true; if (language === 'yaml' || language === 'twig') return true; // no WASM grammar needed if (language === 'xml' || language === 'properties') return true; // no WASM grammar needed return languageCache.has(language); @@ -619,7 +623,7 @@ export function isFileLevelOnlyLanguage(language: Language): boolean { * Get all supported languages (those with grammar definitions). */ export function getSupportedLanguages(): Language[] { - return [...(Object.keys(WASM_GRAMMAR_FILES) as GrammarLanguage[]), 'svelte', 'vue', 'astro', 'liquid']; + return [...(Object.keys(WASM_GRAMMAR_FILES) as GrammarLanguage[]), 'svelte', 'vue', 'astro', 'liquid', 'markdown']; } /** @@ -699,6 +703,7 @@ export function getLanguageDisplayName(language: Language): string { twig: 'Twig', xml: 'XML', properties: 'Java properties', + markdown: 'Markdown', cfml: 'CFML', cfscript: 'CFScript', cfquery: 'CFQuery (SQL)', diff --git a/src/extraction/kernel/decode.ts b/src/extraction/kernel/decode.ts index 611173373..5f067937a 100644 --- a/src/extraction/kernel/decode.ts +++ b/src/extraction/kernel/decode.ts @@ -31,6 +31,7 @@ import { PROVENANCES, REF, REF_FLAG_FILE_PATH, + REF_FLAG_LANGUAGE, REF_ROW_SIZE, VISIBILITIES, } from './layout'; @@ -155,7 +156,9 @@ export function decodeExtractBuffers( // exactly. The ONE exception is flagged (REF_FLAG_FILE_PATH): the // ruby/php visitNode hooks set `filePath: ctx.filePath` on their // mixin/trait `implements` refs — re-attach the decode call's own - // filePath, which is that exact value. + // filePath, which is that exact value. Markdown path refs go through + // `addReference`, which denormalizes BOTH fields, so they carry the + // language flag as well. const ref: UnresolvedReference = { fromNodeId: fromIdx === NONE ? str(arena, row, REF.fromIdStr)! : idByRow[fromIdx]!, referenceName: str(arena, row, REF.referenceName)!, @@ -166,7 +169,9 @@ export function decodeExtractBuffers( line: row.readUInt32LE(REF.line), column: row.readUInt32LE(REF.column), }; - if ((row.readUInt8(REF.flags) & REF_FLAG_FILE_PATH) !== 0) ref.filePath = filePath; + const refFlags = row.readUInt8(REF.flags); + if ((refFlags & REF_FLAG_FILE_PATH) !== 0) ref.filePath = filePath; + if ((refFlags & REF_FLAG_LANGUAGE) !== 0) ref.language = language; const candidates = strList(arena, row, REF.candidates); if (candidates !== undefined) ref.candidates = candidates; unresolvedReferences[i] = ref; diff --git a/src/extraction/kernel/layout.ts b/src/extraction/kernel/layout.ts index 1c490e6e2..65718d100 100644 --- a/src/extraction/kernel/layout.ts +++ b/src/extraction/kernel/layout.ts @@ -96,6 +96,7 @@ export const FUNCTION_REF_CODE = 200; * parameter, which is byte-identical. */ export const REF_FLAG_FILE_PATH = 1; +export const REF_FLAG_LANGUAGE = 2; /** Node bool-flag bit pairs: bit(2n) = present, bit(2n+1) = value. */ export const FLAG = { diff --git a/src/extraction/markdown-extractor.ts b/src/extraction/markdown-extractor.ts new file mode 100644 index 000000000..4695ea4be --- /dev/null +++ b/src/extraction/markdown-extractor.ts @@ -0,0 +1,1003 @@ +import * as path from 'path'; +import { Node, Edge, ExtractionResult, ExtractionError, UnresolvedReference, EdgeKind } from '../types'; +import { generateNodeId } from './tree-sitter-helpers'; + +interface HeadingInfo { + id: string; + level: number; + title: string; + slug: string; + line: number; + endLine: number; + node: Node; +} + +interface FenceState { + marker: string; + language: string; + startLine: number; + lines: Array<{ text: string; line: number }>; +} + +/** + * Lightweight Markdown extractor. + * + * Markdown is indexed as documentation structure rather than code syntax: + * headings become searchable module nodes, links become import nodes, and + * shell-like fenced blocks create command nodes plus file-path references. + */ +export class MarkdownExtractor { + private filePath: string; + private lines: string[]; + private nodes: Node[] = []; + private edges: Edge[] = []; + private unresolvedReferences: UnresolvedReference[] = []; + private errors: ExtractionError[] = []; + private headings: HeadingInfo[] = []; + private referenceKeys = new Set(); + + constructor(filePath: string, source: string) { + this.filePath = normalizeRelativePath(filePath); + this.lines = source.split('\n'); + } + + extract(): ExtractionResult { + const startTime = Date.now(); + + try { + const fileNode = this.createFileNode(); + this.extractHeadings(fileNode); + this.extractStructuredBlocks(fileNode); + this.extractLinksAndCommands(fileNode); + this.finalizeFileDigest(fileNode); + } catch (error) { + this.errors.push({ + message: `Markdown extraction error: ${error instanceof Error ? error.message : String(error)}`, + filePath: this.filePath, + severity: 'error', + code: 'parse_error', + }); + } + + return { + nodes: this.nodes, + edges: this.edges, + unresolvedReferences: this.unresolvedReferences, + errors: this.errors, + durationMs: Date.now() - startTime, + }; + } + + private createFileNode(): Node { + const id = generateNodeId(this.filePath, 'file', this.filePath, 1); + const fileNode: Node = { + id, + kind: 'file', + name: path.posix.basename(this.filePath), + qualifiedName: this.filePath, + filePath: this.filePath, + language: 'markdown', + startLine: 1, + endLine: Math.max(1, this.lines.length), + startColumn: 0, + endColumn: this.lines[this.lines.length - 1]?.length || 0, + // docstring is filled in by finalizeFileDigest() once headings and + // references are known, so it becomes a triage-friendly digest. + docstring: undefined, + updatedAt: Date.now(), + }; + + this.nodes.push(fileNode); + return fileNode; + } + + private extractHeadings(fileNode: Node): void { + const rawHeadings: Array<{ level: number; title: string; line: number; column: number }> = []; + const frontmatterEnd = this.frontmatterEndIndex(); + let fenceMarker: string | null = null; + + for (let i = 0; i < this.lines.length; i++) { + if (i <= frontmatterEnd) continue; + const line = this.lines[i]!; + + // Track fenced code blocks so `# foo` comments and `===`/`---` lines + // inside a code sample are never mistaken for document headings. + const fence = /^(\s*)(`{3,}|~{3,})/.exec(line); + if (fence) { + const marker = fence[2]![0]!.repeat(fence[2]!.length); + if (fenceMarker === null) fenceMarker = marker; + else if (line.trimStart().startsWith(fenceMarker)) fenceMarker = null; + continue; + } + if (fenceMarker !== null) continue; + + // ATX heading: `## Title` + const atx = /^(#{1,6})\s+(.+?)\s*#*\s*$/.exec(line); + if (atx) { + rawHeadings.push({ + level: atx[1]!.length, + title: stripInlineMarkdown(atx[2]!.trim()), + line: i + 1, + column: line.indexOf('#'), + }); + continue; + } + + // Setext heading: a paragraph line underlined by `===` (level 1) or + // `---` (level 2). CommonMark treats a `---` directly under a paragraph + // as a heading, not a thematic break — and the frontmatter skip above + // keeps a closing `---` from turning its last key into a heading. + const underline = /^(=+|-+)\s*$/.exec(line.trim()); + if (underline && i - 1 > frontmatterEnd && isSetextTextLine(this.lines[i - 1] ?? '')) { + rawHeadings.push({ + level: underline[1]![0] === '=' ? 1 : 2, + title: stripInlineMarkdown((this.lines[i - 1] ?? '').trim()), + line: i, // the text line carries the heading + column: 0, + }); + } + } + + const slugCounts = new Map(); + for (let i = 0; i < rawHeadings.length; i++) { + const heading = rawHeadings[i]!; + const baseSlug = slugifyHeading(heading.title); + const seen = slugCounts.get(baseSlug) ?? 0; + slugCounts.set(baseSlug, seen + 1); + const slug = seen === 0 ? baseSlug : `${baseSlug}-${seen}`; + + let endLine = this.lines.length; + for (let j = i + 1; j < rawHeadings.length; j++) { + if (rawHeadings[j]!.level <= heading.level) { + endLine = rawHeadings[j]!.line - 1; + break; + } + } + + const nodeId = generateNodeId(this.filePath, 'module', `${slug}:${heading.line}`, heading.line); + const node: Node = { + id: nodeId, + kind: 'module', + name: heading.title, + qualifiedName: `${this.filePath}#${slug}`, + filePath: this.filePath, + language: 'markdown', + signature: `${'#'.repeat(heading.level)} ${heading.title}`, + docstring: this.buildDocstring(heading.line + 1, endLine), + startLine: heading.line, + endLine, + startColumn: heading.column, + endColumn: this.lines[heading.line - 1]?.length || 0, + updatedAt: Date.now(), + }; + + this.nodes.push(node); + this.headings.push({ + id: nodeId, + level: heading.level, + title: heading.title, + slug, + line: heading.line, + endLine, + node, + }); + } + + const stack: HeadingInfo[] = []; + for (const heading of this.headings) { + while (stack.length > 0 && stack[stack.length - 1]!.level >= heading.level) { + stack.pop(); + } + const parent = stack[stack.length - 1]; + this.edges.push({ + source: parent?.id ?? fileNode.id, + target: heading.id, + kind: 'contains', + provenance: 'heuristic', + }); + stack.push(heading); + } + } + + private extractLinksAndCommands(fileNode: Node): void { + let fence: FenceState | null = null; + + for (let i = 0; i < this.lines.length; i++) { + const line = this.lines[i]!; + const lineNumber = i + 1; + const fenceMatch = /^(\s*)(`{3,}|~{3,})\s*([A-Za-z0-9_+.-]*)/.exec(line); + + if (fence) { + if (line.trimStart().startsWith(fence.marker)) { + this.extractCommandsFromFence(fence, fileNode); + fence = null; + } else { + fence.lines.push({ text: line, line: lineNumber }); + } + continue; + } + + if (fenceMatch) { + fence = { + marker: fenceMatch[2]![0]!.repeat(fenceMatch[2]!.length), + language: (fenceMatch[3] ?? '').toLowerCase(), + startLine: lineNumber, + lines: [], + }; + continue; + } + + const owner = this.findOwnerForLine(lineNumber) ?? fileNode; + this.extractMarkdownLinks(line, lineNumber, owner); + if (!isTableRowLine(line)) { + this.extractFileSymbolMentions(line, lineNumber, owner); + this.extractPathMentions(line, lineNumber, owner); + } + } + + if (fence) { + this.extractCommandsFromFence(fence, fileNode); + } + } + + private extractStructuredBlocks(fileNode: Node): void { + this.extractTables(fileNode); + this.extractListItems(fileNode); + } + + private extractTables(fileNode: Node): void { + let inFence: string | null = null; + + for (let i = 0; i < this.lines.length - 1; i++) { + const line = this.lines[i]!; + const fenceMatch = /^(\s*)(`{3,}|~{3,})/.exec(line); + if (fenceMatch) { + const marker = fenceMatch[2]!; + if (inFence === null) { + inFence = marker; + } else if ( + marker[0] === inFence[0] && + marker.length >= inFence.length && + line.slice(fenceMatch[0].length).trim() === '' + ) { + inFence = null; + } + continue; + } + if (inFence) continue; + + const separator = this.lines[i + 1]!; + if (!isTableHeaderLine(line) || !isTableSeparatorLine(separator)) continue; + + const headers = parseTableCells(line); + if (headers.length < 2) continue; + + let rowIndex = i + 2; + while (rowIndex < this.lines.length && isTableRowLine(this.lines[rowIndex]!)) { + const rowLine = this.lines[rowIndex]!; + const cells = parseTableCells(rowLine); + const lineNumber = rowIndex + 1; + if (shouldIndexTableRow(headers, cells)) { + this.createTableRowNode(headers, cells, rowLine, lineNumber, fileNode); + } + rowIndex++; + } + + i = rowIndex - 1; + } + } + + private extractListItems(fileNode: Node): void { + let inFence: string | null = null; + + for (let i = 0; i < this.lines.length; i++) { + const line = this.lines[i]!; + const fenceMatch = /^(\s*)(`{3,}|~{3,})/.exec(line); + if (fenceMatch) { + const marker = fenceMatch[2]!; + if (inFence === null) { + inFence = marker; + } else if ( + marker[0] === inFence[0] && + marker.length >= inFence.length && + line.slice(fenceMatch[0].length).trim() === '' + ) { + inFence = null; + } + continue; + } + if (inFence) continue; + + const match = /^\s*[-*+]\s+(?:\[[ xX]\]\s+)?(.+?)\s*$/.exec(line); + if (!match) continue; + + const text = stripInlineMarkdown(match[1]!.trim()); + if (!shouldIndexListItem(text)) continue; + + const lineNumber = i + 1; + const owner = this.findOwnerForLine(lineNumber) ?? fileNode; + const name = stableIdentifier(text) ?? truncateForName(text, 80); + const nodeId = generateNodeId(this.filePath, 'constant', `list:${lineNumber}:${name}`, lineNumber); + const node: Node = { + id: nodeId, + kind: 'constant', + name, + qualifiedName: `${this.filePath}::list-item:${slugifyHeading(name)}:${lineNumber}`, + filePath: this.filePath, + language: 'markdown', + signature: `list item: ${truncateForSignature(text, 180)}`, + docstring: text, + startLine: lineNumber, + endLine: lineNumber, + startColumn: Math.max(0, line.indexOf(match[1]!)), + endColumn: line.length, + updatedAt: Date.now(), + }; + + this.nodes.push(node); + this.edges.push({ source: owner.id, target: nodeId, kind: 'contains', provenance: 'heuristic' }); + this.extractStructuredReferences(text, lineNumber, node); + } + } + + private createTableRowNode( + headers: string[], + cells: string[], + rowLine: string, + lineNumber: number, + fileNode: Node + ): void { + const owner = this.findOwnerForLine(lineNumber) ?? fileNode; + const rowText = cells.join(' | '); + const name = stableIdentifier(rowText) ?? truncateForName(firstNonEmptyCell(cells) || `row ${lineNumber}`, 80); + const signature = summarizeTableRow(headers, cells); + const nodeId = generateNodeId(this.filePath, 'constant', `table:${lineNumber}:${name}`, lineNumber); + const node: Node = { + id: nodeId, + kind: 'constant', + name, + qualifiedName: `${this.filePath}::table-row:${slugifyHeading(name)}:${lineNumber}`, + filePath: this.filePath, + language: 'markdown', + signature, + docstring: signature, + startLine: lineNumber, + endLine: lineNumber, + startColumn: Math.max(0, rowLine.indexOf(firstNonEmptyCell(cells) || '|')), + endColumn: rowLine.length, + updatedAt: Date.now(), + }; + + this.nodes.push(node); + this.edges.push({ source: owner.id, target: nodeId, kind: 'contains', provenance: 'heuristic' }); + + this.extractStructuredReferences(rowText, lineNumber, node); + this.extractCommandsFromStructuredCells(cells, rowLine, lineNumber, node); + } + + private extractCommandsFromStructuredCells(cells: string[], rowLine: string, lineNumber: number, owner: Node): void { + for (const cell of cells) { + const candidates = extractCodeSpans(cell); + if (candidates.length === 0) candidates.push(stripInlineMarkdown(cell)); + + for (const candidate of candidates) { + const command = extractCommandFromText(candidate); + if (!command) continue; + const column = Math.max(0, rowLine.indexOf(candidate)); + this.createCommandNode(command, lineNumber, column, owner); + } + } + } + + private createCommandNode(command: string, line: number, column: number, owner: Node): void { + const nodeId = generateNodeId(this.filePath, 'function', `command:${line}:${column}:${command}`, line); + const node: Node = { + id: nodeId, + kind: 'function', + name: commandName(command), + qualifiedName: `${this.filePath}::command:${line}:${column}:${command}`, + filePath: this.filePath, + language: 'markdown', + signature: command, + startLine: line, + endLine: line, + startColumn: column, + endColumn: column + command.length, + updatedAt: Date.now(), + }; + + this.nodes.push(node); + this.edges.push({ source: owner.id, target: nodeId, kind: 'contains', provenance: 'heuristic' }); + + const scriptPath = extractScriptPath(command); + if (scriptPath) { + const normalizedTarget = this.normalizeReference(scriptPath); + if (normalizedTarget) { + this.addReference(nodeId, normalizedTarget, 'calls', line, column); + } + } + } + + private extractMarkdownLinks(line: string, lineNumber: number, owner: Node): void { + const linkRegex = /!?\[([^\]]*)\]\(([^)\s]+)(?:\s+["'][^"']*["'])?\)/g; + let match: RegExpExecArray | null; + + while ((match = linkRegex.exec(line)) !== null) { + const target = match[2]!; + if (!isLocalReference(target)) continue; + + const normalizedTarget = this.normalizeReference(target); + if (!normalizedTarget) continue; + + const column = match.index; + const label = stripInlineMarkdown(match[1] || target) || target; + const displayName = path.posix.basename(stripAnchor(normalizedTarget)) || label; + const nodeId = generateNodeId(this.filePath, 'import', `${normalizedTarget}:${lineNumber}:${column}`, lineNumber); + const node: Node = { + id: nodeId, + kind: 'import', + name: displayName, + qualifiedName: `${this.filePath}::link:${normalizedTarget}:${lineNumber}:${column}`, + filePath: this.filePath, + language: 'markdown', + signature: match[0], + docstring: label, + startLine: lineNumber, + endLine: lineNumber, + startColumn: column, + endColumn: column + match[0].length, + updatedAt: Date.now(), + }; + + this.nodes.push(node); + this.edges.push({ source: owner.id, target: nodeId, kind: 'contains', provenance: 'heuristic' }); + this.addReference(owner.id, normalizedTarget, 'imports', lineNumber, column); + } + } + + private extractPathMentions(line: string, lineNumber: number, owner: Node): void { + const pathRegex = /(?= best.level) best = heading; + } + } + return best?.node ?? null; + } + + private addReference( + fromNodeId: string, + referenceName: string, + referenceKind: EdgeKind, + line: number, + column: number + ): void { + const key = `${fromNodeId}:${referenceKind}:${referenceName}:${line}:${column}`; + if (this.referenceKeys.has(key)) return; + this.referenceKeys.add(key); + this.unresolvedReferences.push({ + fromNodeId, + referenceName, + referenceKind, + line, + column, + filePath: this.filePath, + language: 'markdown', + }); + } + + private normalizeReference(target: string): string | null { + const [pathAndSymbol, anchorPart] = splitAnchor(target.trim()); + const [pathPart, symbolPart] = splitFileSymbol(pathAndSymbol); + const cleanPath = decodePath(pathPart.split(/[?#]/)[0] ?? ''); + const anchor = anchorPart ? `#${slugifyHeading(anchorPart)}` : ''; + + if (!cleanPath && anchor) { + return `${this.filePath}${anchor}`; + } + if (!cleanPath) return null; + + const withoutLeadingSlash = cleanPath.startsWith('/') ? cleanPath.slice(1) : cleanPath; + const baseDir = path.posix.dirname(this.filePath); + const normalized = cleanPath.startsWith('/') || baseDir === '.' + ? path.posix.normalize(withoutLeadingSlash) + : path.posix.normalize(path.posix.join(baseDir, withoutLeadingSlash)); + + if (normalized.startsWith('../') || normalized === '..') return null; + return `${normalized}${symbolPart ? `::${symbolPart}` : ''}${anchor}`; + } + + private buildDocstring(startLine: number, endLine: number): string | undefined { + const text = this.lines + .slice(Math.max(0, startLine - 1), Math.max(0, endLine)) + .map((line) => stripInlineMarkdown(line.replace(/^#{1,6}\s+/, '').trim())) + .filter((line) => line && !/^(```|~~~)/.test(line)) + .join('\n') + .trim(); + + if (!text) return undefined; + return text.length > 600 ? `${text.slice(0, 600)}...` : text; + } + + /** + * Index of the closing `---` of a leading YAML frontmatter block, or -1 when + * the file has none. Lines at or before this index are metadata, not body — + * so they never produce headings and never seed the intro/digest. + */ + private frontmatterEndIndex(): number { + if ((this.lines[0] ?? '').trim() !== '---') return -1; + for (let j = 1; j < this.lines.length; j++) { + if (this.lines[j]!.trim() === '---') return j; + } + return -1; + } + + /** + * Replace the file node's docstring with a deterministic digest — a one-line + * "what is this doc about" intro plus the key files/symbols it references. + * It is derived purely from the already-extracted structure (no LLM), kept + * short enough to surface in node details, and — because docstrings are in + * the FTS index — makes a doc discoverable by the symbols it documents even + * when the query matches no heading or filename. See the reviewer thread on + * PR #361. + */ + private finalizeFileDigest(fileNode: Node): void { + const intro = this.buildIntro(); + const refs = this.collectKeyReferences(); + const parts: string[] = []; + if (intro) parts.push(intro); + if (refs.length > 0) parts.push(`refs: ${refs.join(', ')}`); + const digest = parts.join(' · ').trim(); + fileNode.docstring = digest || this.buildDocstring(1, Math.min(this.lines.length, 40)); + } + + /** + * First real prose line of the document — the closest deterministic stand-in + * for "what this file is about". Skips frontmatter, headings, fenced code, + * tables, badge/image-only lines, and raw HTML. + */ + private buildIntro(): string | null { + let fenceMarker: string | null = null; + for (let i = this.frontmatterEndIndex() + 1; i < this.lines.length; i++) { + const trimmed = this.lines[i]!.trim(); + const fence = /^(`{3,}|~{3,})/.exec(trimmed); + if (fence) { + const marker = fence[1]![0]!.repeat(fence[1]!.length); + fenceMarker = fenceMarker === null ? marker : (trimmed.startsWith(fenceMarker) ? null : fenceMarker); + continue; + } + if (fenceMarker !== null) continue; + if (!trimmed) continue; + if (/^#{1,6}\s/.test(trimmed)) continue; // ATX heading + if (/^(=+|-+|\*{3,}|_{3,})\s*$/.test(trimmed)) continue; // setext underline / hr + if (trimmed.startsWith('|')) continue; // table + if (/^!\[/.test(trimmed)) continue; // image / badge line + if (/^