diff --git a/graphify/extractors/markdown.py b/graphify/extractors/markdown.py index 387074a965..4dc2b8feb0 100644 --- a/graphify/extractors/markdown.py +++ b/graphify/extractors/markdown.py @@ -6,7 +6,7 @@ import unicodedata from pathlib import Path -from graphify.detect import CODE_EXTENSIONS +from graphify.detect import CODE_EXTENSIONS, DOC_EXTENSIONS from graphify.extractors.base import _file_stem, _make_id from graphify.security import sanitize_metadata @@ -27,7 +27,8 @@ # (``src/pkg/mod.py::Widget::render``) names the defining file and a # ``::``-separated symbol chain, exactly as pytest node ids and cite-lint # citations do. A bare mention (``Widget``, ``render()``, ``pkg.Widget``) names -# a symbol with no file evidence, so it resolves only on a unique match. +# a symbol with no file evidence, so it resolves only on a unique match; the +# dotted form keeps its qualifiers (``pkg``) as evidence the resolver checks. _MD_PATH_MENTION_RE = re.compile( r'^([A-Za-z0-9_./\-]+\.[A-Za-z0-9]+)::([A-Za-z_]\w*(?:::[A-Za-z_]\w*)*)(?:\(\))?$' ) @@ -257,10 +258,12 @@ def _code_span_mention(span: str) -> "tuple[str | None, list[str]] | None": Returns ``(path, names)``: ``path`` is the cited file for the ``path::Name`` form and None for a bare or dotted mention; ``names`` is the - symbol chain, outermost first. A trailing ``()`` is dropped from the last - name so ``render()`` and ``render`` cite the same symbol. Spans that read - as a file (``setup.py``), a shell command, an expression or prose are not - mentions and yield None. + symbol chain, outermost first, so ``pkg.sub.Widget`` yields + ``["pkg", "sub", "Widget"]`` and the resolver can hold the match to that + evidence. A trailing ``()`` is dropped from the last name so ``render()`` + and ``render`` cite the same symbol. Spans that read as a file + (``setup.py``, ``README.md``), a shell command, an expression or prose are + not mentions and yield None. """ text = span.strip() if not text or " " in text: @@ -272,10 +275,14 @@ def _code_span_mention(span: str) -> "tuple[str | None, list[str]] | None": if not m: return None if "." in text: - # ``a.b.Name`` is a qualified symbol; ``setup.py`` is a file. A dotted - # span whose last segment is a code extension is the latter. - if "." + m.group(1) in CODE_EXTENSIONS: + # ``a.b.Name`` is a qualified symbol; ``setup.py`` and ``README.md`` + # are files. A dotted span whose last segment is a code or document + # extension is the latter. Other file-like spans (``pyproject.toml``) + # classify as a mention and are rejected at resolution, where the + # qualifier ``pyproject`` matches no callable's file or owner. + if "." + m.group(1) in CODE_EXTENSIONS or "." + m.group(1) in DOC_EXTENSIONS: return None + return None, text.rstrip("()").split(".") return None, [m.group(1)] @@ -311,7 +318,9 @@ def extract_markdown(path: Path) -> dict: edges: the symbol they cite lives in another file, so the match is made once every file is extracted and ids are final, by the ``markdown_mentions`` language resolver (see - ``graphify.markdown_resolution``). That pass emits + ``graphify.markdown_resolution``). The dotted form travels with its + qualifiers, which the resolver checks against the match's owners and + file path. That pass emits heading --references--> code symbol (page --references--> symbol for a mention above the first heading): EXTRACTED for the path-qualified form, INFERRED for a bare name that matches exactly one code symbol. The shared diff --git a/graphify/markdown_resolution.py b/graphify/markdown_resolution.py index bbae40dcbb..e0464e4c8d 100644 --- a/graphify/markdown_resolution.py +++ b/graphify/markdown_resolution.py @@ -25,8 +25,17 @@ Only nodes the extractors marked ``_callable`` qualify, for the reason the indirect-call pass gives (#1566): a by-name match must land on a real function, method or class, never on a same-named data symbol such as a - JSON key. No tie-breaking: an ambiguous name is a guess, and a guess is - not an edge. The match is INFERRED (0.95, a named cross-file reference). + JSON key. A dotted mention (``pkg.Widget``, ``Widget.render``) keeps its + qualifiers, and a candidate survives only when every qualifier is a label + on its ``contains`` / ``method`` owner chain or a segment (or stem) of its + source path: ``time.sleep`` never lands on a repo's own ``sleep``, and + ``Widget.render`` picks the ``render`` that ``Widget`` owns. No other + tie-breaking: an ambiguous name is a guess, and a guess is not an edge. + The match is INFERRED (0.95, a named cross-file reference). + +An explicit relative cite (``./`` or ``../``) resolves against the document's +directory only; it never falls through to the suffix rule, so a path that +escapes the corpus cannot land on an unrelated copy of the file elsewhere. The shared cross-file call pass in ``extract`` skips ``markdown`` raw calls, so a mention never surfaces as a ``calls`` edge. @@ -34,6 +43,7 @@ from __future__ import annotations import os +import re from typing import Any from graphify.extractors.base import _LANGUAGE_BUILTIN_GLOBALS @@ -42,6 +52,10 @@ _CONTAINMENT_RELATIONS = frozenset({"contains", "method"}) +#: Leading ``./`` and ``../`` segments of a cited path, stripped as whole +#: segments (``lstrip("./")`` would eat the dot of ``.github/...``). +_RELATIVE_PREFIX_RE = re.compile(r"^(\.\.?/)+") + def _symbol_label(node: dict[str, Any]) -> str: """A node's label as a document would write it: no ``()``, no leading dot.""" @@ -63,19 +77,56 @@ def _match_cited_file(cited: str, doc_file: str, source_files: set[str]) -> str Tries the path relative to the citing document first (``../src/mod.py`` from ``docs/guide.md``), then an exact match, then a unique - segment-aligned suffix (``src/mod.py`` naming ``/repo/src/mod.py``). + segment-aligned suffix (``src/mod.py`` naming ``/repo/src/mod.py``). A + cite that is explicitly relative (``./x`` or ``../x``) stops after the + first step: it names one location, and a suffix match elsewhere would be a + different file. """ cited_posix = _posix(cited) doc_dir = os.path.dirname(doc_file) - relative = _posix(os.path.normpath(os.path.join(doc_dir, cited))) - for candidate in (relative, cited_posix): - if candidate in source_files: - return candidate - suffix = "/" + cited_posix.lstrip("./") - matches = [sf for sf in source_files if sf.endswith(suffix)] + relative = _posix(os.path.normpath(os.path.join(doc_dir, cited_posix))) + if relative in source_files: + return relative + if cited_posix.startswith(("./", "../")): + return None + if cited_posix in source_files: + return cited_posix + stripped = _RELATIVE_PREFIX_RE.sub("", cited_posix) + matches = [sf for sf in source_files + if sf == stripped or sf.endswith("/" + stripped)] return matches[0] if len(matches) == 1 else None +def _evidence(node_id: str, nodes_by_id: dict[str, dict[str, Any]], + parents: dict[str, set[str]]) -> set[str]: + """Labels a dotted mention may qualify ``node_id`` with. + + The labels of every node on its ``contains`` / ``method`` owner chain + (``Widget`` for ``Widget.render``) plus each segment and stem of its + source path (``pkg`` and ``mod`` for ``pkg.mod.Widget``). + """ + evidence: set[str] = set() + frontier = {node_id} + seen: set[str] = set() + while frontier: + nid = frontier.pop() + if nid in seen: + continue + seen.add(nid) + owners = parents.get(nid, set()) + for owner in owners: + node = nodes_by_id.get(owner) + if node is not None: + evidence.add(_symbol_label(node)) + frontier |= owners + node = nodes_by_id.get(node_id, {}) + for segment in _posix(str(node.get("source_file", ""))).split("/"): + if segment: + evidence.add(segment) + evidence.add(os.path.splitext(segment)[0]) + return evidence + + def _markdown_raw_calls(per_file: list[dict]) -> list[dict]: calls: list[dict] = [] for result in per_file: @@ -118,7 +169,8 @@ def resolve_markdown_mentions( for e in all_edges: if e.get("relation") in _CONTAINMENT_RELATIONS: parents.setdefault(str(e.get("target")), set()).add(str(e.get("source"))) - node_ids = {n["id"] for n in all_nodes if n.get("id")} + nodes_by_id = {n["id"]: n for n in all_nodes if n.get("id")} + node_ids = set(nodes_by_id) existing = { (e.get("source"), e.get("target")) for e in all_edges if e.get("relation") == "references" @@ -156,7 +208,11 @@ def _resolve_chain(file_labels: dict[str, list[str]], names: list[str]) -> str | else: if callee in _LANGUAGE_BUILTIN_GLOBALS: continue - candidates = by_label.get(callee, []) + qualifiers = set(names[:-1]) + candidates = [ + c for c in by_label.get(callee, []) + if qualifiers <= _evidence(c, nodes_by_id, parents) + ] target = candidates[0] if len(candidates) == 1 else None confidence, score = "INFERRED", 0.95 if target is None or target == caller or (caller, target) in existing: diff --git a/graphify/watch.py b/graphify/watch.py index 9943b5caf0..f95eb7af9d 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -539,13 +539,21 @@ def _reconcile_markdown_links( authored link only when both files have unique representatives. If either side is ambiguous, retain the existing AST edge instead of guessing or deleting it. A link removed from its owning Markdown source is pruned. + + Only authored links are owned here. A code-span mention (a ``references`` + edge the markdown_mentions resolver emits) targets a code symbol rather + than a file representative, so it is left to the AST ownership rule above: + re-extracting the document regenerates it and re-extracting the code side + keeps or drops it with the target node. """ from graphify.build import _is_ast_tier from graphify.extract import _file_node_id, _safe_extract_with_xaml_root from graphify.extractors.base import _make_id from graphify.extractors.markdown import extract_markdown + from graphify.markdown_resolution import _is_file_node all_nodes = result.get("nodes", []) + preserved_nodes + nodes_by_id = {node["id"]: node for node in all_nodes if node.get("id")} nodes_by_source: dict[str, list[dict]] = {} for node in all_nodes: if source_file := node.get("source_file"): @@ -648,9 +656,19 @@ def _matches_unresolved_target( candidate = project_root / Path(owner).parent / Path(target_source).name return raw_target == _make_id(str(candidate)) + def _is_code_span_mention(edge: dict) -> bool: + target = nodes_by_id.get(edge.get("target")) + return ( + target is not None + and target.get("file_type") == "code" + and not _is_file_node(target) + ) + def _keep_edge(edge: dict) -> bool: if not (_is_ast_tier(edge) and edge.get("relation") == "references"): return True + if _is_code_span_mention(edge): + return True owner = source_paths.normalize(edge.get("source_file")) if owner not in parsed_sources: return True diff --git a/tests/test_markdown_code_spans.py b/tests/test_markdown_code_spans.py index d081f84a77..01a9b2fa33 100644 --- a/tests/test_markdown_code_spans.py +++ b/tests/test_markdown_code_spans.py @@ -8,11 +8,13 @@ """ from __future__ import annotations +import json import os from pathlib import Path from graphify.extract import extract from graphify.extractors.markdown import _code_span_mention, extract_markdown +from graphify.markdown_resolution import _match_cited_file _WIDGET_PY = '''\ class Widget: @@ -81,15 +83,42 @@ def _references(r): def test_code_span_mentions_are_classified(): assert _code_span_mention("Widget") == (None, ["Widget"]) assert _code_span_mention("render()") == (None, ["render"]) - assert _code_span_mention("pkg.sub.Widget") == (None, ["Widget"]) + assert _code_span_mention("pkg.sub.Widget") == (None, ["pkg", "sub", "Widget"]) + assert _code_span_mention("Widget.render()") == (None, ["Widget", "render"]) + # A file-like span with an unknown extension classifies as a mention and + # is rejected at resolution, where `pyproject` matches no callable. + assert _code_span_mention("pyproject.toml") == (None, ["pyproject", "toml"]) pinned = ("src/widget.py", ["Widget", "render"]) assert _code_span_mention("src/widget.py::Widget::render") == pinned assert _code_span_mention("src/widget.py::Widget::render()") == pinned # Files, commands, expressions and prose are not symbol mentions. - for span in ("setup.py", "git revert", "x = 1", "a-b", "--flag", "", "src/widget.py"): + for span in ("setup.py", "README.md", "notes.txt", "git revert", "x = 1", "a-b", + "--flag", "", "src/widget.py"): assert _code_span_mention(span) is None, span +def test_cited_file_matching(): + sources = {"src/mod.py", ".github/scripts/check.py", "lib/x.py", "vendor/lib/x.py", + "docs/src/mod.py"} + # Doc-relative first, then exact, then a unique segment-aligned suffix. + assert _match_cited_file("src/mod.py", "docs/guide.md", sources) == "docs/src/mod.py" + assert _match_cited_file("src/mod.py", "README.md", sources) == "src/mod.py" + assert _match_cited_file("mod.py", "README.md", sources) is None + assert _match_cited_file("scripts/check.py", "README.md", sources) == ( + ".github/scripts/check.py") + # `./` and `../` are stripped as segments: a hidden directory keeps its dot. + assert _match_cited_file(".github/scripts/check.py", "docs/guide.md", sources) == ( + ".github/scripts/check.py") + assert _match_cited_file("./src/mod.py", "README.md", sources) == "src/mod.py" + # The suffix form also names a root-level file. + assert _match_cited_file("lib/x.py", "docs/guide.md", {"lib/x.py"}) == "lib/x.py" + # An explicit relative cite resolves against the document only: escaping + # the corpus never falls through to another copy of the file. + assert _match_cited_file("../lib/x.py", "docs/guide.md", sources) == "lib/x.py" + assert _match_cited_file("../lib/x.py", "README.md", sources) is None + assert _match_cited_file("../../vendor/lib/x.py", "docs/guide.md", sources) is None + + def test_extract_markdown_reports_mentions_as_markdown_raw_calls(tmp_path): doc = tmp_path / "docs" / "guide.md" doc.parent.mkdir() @@ -152,9 +181,11 @@ def test_mentions_resolve_to_references_edges_end_to_end(tmp_path): assert refs[(guide["id"], widget["id"])]["confidence"] == "INFERRED" assert refs[(guide["id"], widget["id"])]["confidence_score"] == 0.95 assert (rendering["id"], widget["id"]) in refs - # Dotted `Widget.render()` resolves by its last segment: `render` is - # ambiguous (two classes define it), so no edge; same for `helper()`. - assert {t for (s, t) in refs if s == rendering["id"]} == {widget["id"]} + # `render` is defined twice, but the qualifier in `Widget.render()` names + # the owner, so the mention resolves; the bare `helper()` stays ambiguous. + assert {t for (s, t) in refs if s == rendering["id"]} == { + widget["id"], widget_render["id"]} + assert refs[(rendering["id"], widget_render["id"])]["confidence"] == "INFERRED" # Path-qualified mentions are EXTRACTED and scoped to the cited file. assert refs[(pinned["id"], widget_render["id"])]["confidence"] == "EXTRACTED" assert refs[(pinned["id"], widget_render["id"])]["confidence_score"] == 1.0 @@ -183,6 +214,67 @@ def test_ambiguous_bare_name_yields_no_edge(tmp_path): assert {t for (s, t) in _references(r) if s == notes["id"]} == set() +def test_dotted_mentions_need_qualifier_evidence(tmp_path): + r = _extract(tmp_path, { + "src/widget.py": _WIDGET_PY + "\n\ndef sleep():\n pass\n", + "src/gadget.py": _GADGET_PY + "\n\ndef toml():\n pass\n", + "docs/notes.md": ( + "# Notes\n\n" + "`time.sleep` and `pyproject.toml` are not this repo's sleep or toml.\n" + "`widget.Widget`, `src.gadget.Gadget` and `Widget.render` are.\n" + "`Gadget.helper` is not: `helper` is not owned by `Gadget`.\n" + ), + }) + + notes = _node(r, "Notes") + cited = {t for (s, t) in _references(r) if s == notes["id"]} + widget = _node(r, "Widget", "widget.py") + gadget = _node(r, "Gadget", "gadget.py") + widget_render = next( + n for n in r["nodes"] if n["label"] == ".render()" and "widget" in n["id"]) + assert cited == {widget["id"], gadget["id"], widget_render["id"]} + + +def test_mentions_survive_a_rebuild_over_an_existing_graph(tmp_path): + """The watch reconcile owns authored ``[link](file)`` edges, not mentions. + + A rebuild over an existing graph re-parses the Markdown corpus and prunes + any ``references`` edge it did not author; a code-span mention targets a + code symbol, never a file, so it must survive a no-change rebuild and the + incremental rebuilds of either side. + """ + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "widget.py").write_text(_WIDGET_PY) + doc = corpus / "doc.md" + doc.write_text("# Doc\n\n## Usage\n\nBuild a `Widget`.\n") + graph_path = corpus / "graphify-out" / "graph.json" + + def mention_edges(): + links = json.loads(graph_path.read_text(encoding="utf-8"))["links"] + return {(e["source"], e["target"]) for e in links + if e.get("relation") == "references" and e.get("confidence_score")} + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + expected = mention_edges() + assert len(expected) == 1 + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + assert mention_edges() == expected, "no-change rebuild" + + doc.write_text(doc.read_text() + "\nStill a `Widget`.\n") + assert _rebuild_code(corpus, changed_paths=[doc], no_cluster=True, + acquire_lock=False) is True + assert mention_edges() == expected, "document re-extracted" + + (corpus / "widget.py").write_text(_WIDGET_PY + "\n\ndef extra():\n pass\n") + assert _rebuild_code(corpus, changed_paths=[corpus / "widget.py"], no_cluster=True, + acquire_lock=False) is True + assert mention_edges() == expected, "code re-extracted" + + def test_mentions_survive_the_extraction_cache(tmp_path): files = {"src/widget.py": _WIDGET_PY, "docs/guide.md": _GUIDE_MD} first = _extract(tmp_path, files)