From e3edfd43305a542044a89e2622a1d1726d352445 Mon Sep 17 00:00:00 2001 From: Brian O'Reilly Date: Sun, 13 Sep 2026 12:45:48 -0400 Subject: [PATCH 1/3] fix(commonlisp): keep specializes edge when the specializer is in another file A defmethod dispatching on a class defined in another file lost its specializes edge. The specializer resolved to a file-scoped id with no backing node, so the dangling-edge filter pruned the edge. Resolve through the same sourceless stub the cross-file superclass path uses, so the corpus rewire collapses it onto the real defclass; a same-file specializer still binds locally. Most methods in a Common Lisp system live apart from the class they dispatch on, so this removed nearly every specializes edge in a real codebase. Add specializes to the supertype relations at the same time. Functions and types occupy separate namespaces in Common Lisp, so one symbol is routinely both, and without the guard a specializer stub can collapse onto a same-named function and assert a dispatch relationship that does not exist. --- graphify/extract.py | 7 +++- graphify/extractors/commonlisp.py | 3 +- tests/test_commonlisp_specializer_binding.py | 32 +++++++++++++++++ tests/test_languages.py | 36 ++++++++++++++++++++ 4 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 tests/test_commonlisp_specializer_binding.py diff --git a/graphify/extract.py b/graphify/extract.py index 90a6ab0267..151b97572b 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -2809,7 +2809,12 @@ def _rewire_unique_stub_nodes(nodes: list[dict], edges: list[dict]) -> None: stub_ids = {str(s.get("id")) for s in stubs if s.get("id")} stub_families: dict[str, set] = {} supertype_stub_ids: set[str] = set() # stubs used as a base type — never a function - _SUPERTYPE_RELATIONS = {"inherits", "implements", "extends"} + # `specializes` joins these because a Common Lisp method dispatches on a + # TYPE, and functions and types occupy separate namespaces there: `list`, + # `stream` and `condition` are all routinely both. Without it a specializer + # stub can bind to a same-named function, which asserts a dispatch + # relationship that does not exist. + _SUPERTYPE_RELATIONS = {"inherits", "implements", "extends", "specializes"} for edge in edges: rel = edge.get("relation") for endpoint in ("source", "target"): diff --git a/graphify/extractors/commonlisp.py b/graphify/extractors/commonlisp.py index 2ed9d9fbdd..45b3aaaf98 100644 --- a/graphify/extractors/commonlisp.py +++ b/graphify/extractors/commonlisp.py @@ -319,7 +319,8 @@ def _handle_defun_node(node) -> None: syms = [c for c in param.children if c.type == "sym_lit"] if len(syms) >= 2: specializer_name = _text(syms[1]) - spec_nid = _cl_id(stem, specializer_name) + spec_nid = ensure_class_ref( + specializer_name, param.start_point[0] + 1) add_edge(func_nid, spec_nid, "specializes", param.start_point[0] + 1) break diff --git a/tests/test_commonlisp_specializer_binding.py b/tests/test_commonlisp_specializer_binding.py new file mode 100644 index 0000000000..20dd7e8379 --- /dev/null +++ b/tests/test_commonlisp_specializer_binding.py @@ -0,0 +1,32 @@ +"""A Common Lisp method specializer must never bind to a same-named function. + +Functions and types occupy separate namespaces in Common Lisp, so one symbol is +routinely both: `list`, `stream`, `condition`, `pathname`. When a specializer +names a type the corpus does not define, the cross-file stub must stay +unresolved rather than collapse onto a function that happens to share the name, +which would assert a dispatch relationship that does not exist. +""" +from __future__ import annotations + +from graphify.extract import extract + + +def test_specializer_does_not_bind_to_same_named_function(tmp_path): + a = tmp_path / "a.lisp" + a.write_text("(defun square (x) (* x x))\n") # a function named square + b = tmp_path / "b.lisp" + b.write_text( + "(defgeneric area (obj))\n" + "(defmethod area ((obj square)) 1)\n" # dispatches on a TYPE named square + ) + graph = extract([a, b], cache_root=tmp_path, parallel=False) + by_id = {n["id"]: n for n in graph["nodes"]} + for e in graph["edges"]: + if e.get("relation") != "specializes": + continue + target = by_id.get(e.get("target"), {}) + label = str(target.get("label", "")) + assert not label.endswith("()"), ( + f"specializes bound to the function {label!r}; " + "no class of that name exists in the corpus" + ) diff --git a/tests/test_languages.py b/tests/test_languages.py index aa7d49486f..ee93dbfda8 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -3811,6 +3811,42 @@ def test_cl_crossfile_superclass_inherits_edge_survives(tmp_path): assert base["source_file"] == "", "cross-file superclass must be a sourceless stub" +@_needs_commonlisp +def test_cl_crossfile_specializer_specializes_edge_survives(tmp_path): + """A method specialising on a class from another file must keep its edge. + + The specializer was resolved with a file-scoped id, so a method dispatching + on a class defined elsewhere produced an edge to a node that did not exist + and the dangling-edge filter removed it. Since most methods in a CL system + live apart from the class they dispatch on, that erased nearly every + specializes edge in a real codebase. Resolve through the same sourceless + stub the cross-file superclass path uses. + """ + f = tmp_path / "shapes.lisp" + f.write_text( + "(defclass circle () ())\n" + "(defgeneric area (obj))\n" + "(defmethod area ((obj circle)) 1)\n" + "(defmethod area ((obj square)) 2)\n" + ) + r = extract_commonlisp(f) + assert "error" not in r + id_to_node = {n["id"]: n for n in r["nodes"]} + targets = { + id_to_node[e["target"]]["label"] + for e in r["edges"] + if e["relation"] == "specializes" + and e["source"] in id_to_node and e["target"] in id_to_node + } + # same-file specializer still binds locally + assert "circle" in targets + # cross-file specializer survives via a sourceless stub instead of being dropped + assert "square" in targets + square = next(n for n in r["nodes"] if n["label"] == "square") + assert square["source_file"] == "", "cross-file specializer must be a sourceless stub" + + + @_needs_commonlisp def test_cl_imports(): r = extract_commonlisp(FIXTURES / "sample.lisp") From b26396b032a1faae426f71c1ad949485c9cc589a Mon Sep 17 00:00:00 2001 From: Brian O'Reilly Date: Sun, 13 Sep 2026 12:45:57 -0400 Subject: [PATCH 2/3] fix(commonlisp): give define-condition the inherits edges of its parent types define-condition was routed to the generic definer path, which records the name and never reads the parent list, so a condition hierarchy reached the graph as unrelated nodes even with both ends in the same file. It shares defclass's shape, (NAME (PARENTS) (SLOTS) ...), so the defclass handler covers it and the class path keeps working unchanged. Conditions are how a Common Lisp program signals, so for a codebase that leans on them this left a large part of its structure invisible. --- graphify/extractors/commonlisp.py | 6 +++++- tests/test_languages.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/graphify/extractors/commonlisp.py b/graphify/extractors/commonlisp.py index 45b3aaaf98..84c2c0766a 100644 --- a/graphify/extractors/commonlisp.py +++ b/graphify/extractors/commonlisp.py @@ -468,7 +468,11 @@ def _process_form(top) -> bool: current_package = _text(child) break return True - if first_lower == "defclass": + if first_lower in ("defclass", "define-condition"): + # define-condition shares defclass's shape, (NAME (PARENTS) (SLOTS) ...), + # so the same handler reads the parent list the generic definer path + # never looks at. A condition hierarchy is inheritance and belongs in + # the graph as such. _handle_defclass(top) return True if first_lower in ("require", "ql:quickload"): diff --git a/tests/test_languages.py b/tests/test_languages.py index ee93dbfda8..7ab58a70bc 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -3846,6 +3846,37 @@ def test_cl_crossfile_specializer_specializes_edge_survives(tmp_path): assert square["source_file"] == "", "cross-file specializer must be a sourceless stub" +@_needs_commonlisp +def test_cl_define_condition_emits_inherits_edges(tmp_path): + """A condition hierarchy must produce inherits edges like a class one. + + define-condition was routed to the generic definer path, which records the + name and never reads the parent list, so a condition hierarchy appeared in + the graph as unrelated nodes even with both ends in the same file. Codebases + that signal through conditions had that structure missing entirely. + """ + f = tmp_path / "conditions.lisp" + f.write_text( + "(define-condition app-error (error) ())\n" + "(define-condition parse-error (app-error) ())\n" + "(defclass control () ())\n" + "(defclass control-sub (control) ())\n" + ) + r = extract_commonlisp(f) + assert "error" not in r + id_to_node = {n["id"]: n for n in r["nodes"]} + inherits = { + (id_to_node[e["source"]]["label"], id_to_node[e["target"]]["label"]) + for e in r["edges"] + if e["relation"] == "inherits" + and e["source"] in id_to_node and e["target"] in id_to_node + } + assert ("parse-error", "app-error") in inherits + assert ("app-error", "error") in inherits + # control: the defclass path was already working and must stay working + assert ("control-sub", "control") in inherits + + @_needs_commonlisp def test_cl_imports(): From 7222d75f9a89ced8e63cc13380b2b05631bc5c65 Mon Sep 17 00:00:00 2001 From: Brian O'Reilly Date: Sun, 13 Sep 2026 12:46:06 -0400 Subject: [PATCH 3/3] feat(commonlisp): resolve calls to functions defined in other files A Common Lisp system spreads its functions across many files of one package and calls them by bare name, with no receiver at the call site and no import naming the target. The per-file extractor sees only the file it is parsing, so it could resolve a minority of the real calls and silently dropped the rest: a call graph that looked complete and was mostly missing. The extractor now reports what it cannot see as raw_calls, and a resolver runs over the merged corpus and binds a bare name to its definition when exactly one definition carries that name. An ambiguous name goes through the shared tie-breakers and is dropped unless one candidate survives, so a guess is never emitted. Resolved edges are INFERRED, since a name match across the corpus is weaker evidence than a call resolved inside a single file. Modelled on the Pascal resolver, which closes the same shape of gap for a call whose target lies outside any one file's scope; the resolution rule differs because Pascal walks an inherits chain to mirror method lookup, while a bare name in a Lisp package denotes one function. Calls into the standard library need no special handling and get none: nothing in the corpus defines car or format, so they match nothing and add no node. --- graphify/commonlisp_resolution.py | 101 ++++++++++++++++++++++++++++ graphify/extract.py | 11 +++ graphify/extractors/commonlisp.py | 29 ++++++-- tests/test_commonlisp_resolution.py | 95 ++++++++++++++++++++++++++ 4 files changed, 232 insertions(+), 4 deletions(-) create mode 100644 graphify/commonlisp_resolution.py create mode 100644 tests/test_commonlisp_resolution.py diff --git a/graphify/commonlisp_resolution.py b/graphify/commonlisp_resolution.py new file mode 100644 index 0000000000..8bf87afd59 --- /dev/null +++ b/graphify/commonlisp_resolution.py @@ -0,0 +1,101 @@ +"""Cross-file resolution for Common Lisp calls by bare name. + +The per-file Common Lisp extractor (``extract_commonlisp``) resolves a call +only against the definitions in the file being parsed, because each file is +extracted independently. A Common Lisp system spreads its functions across +many files of one package and calls them by bare name with no receiver and no +import statement at the call site, so the per-file pass resolves a minority of +the real calls and the rest have no candidate to bind to. Those are reported +as ``raw_calls`` rather than guessed at. + +This resolver runs after all files are extracted (registered in +``graphify.resolver_registry``) with the full merged corpus available, so a +bare name can be matched against every definition the corpus knows. Modelled +on ``pascal_resolution`` -- same shape of problem, a call whose target is +structurally outside any one file's scope -- but the resolution rule differs +because the languages differ. Pascal walks an ``inherits`` chain, mirroring +Delphi's method lookup. Common Lisp has no receiver to type and no chain to +walk: a bare name in a package denotes one function, so the corpus-wide name +is the resolution, guarded by requiring a single candidate. + +Calls into the standard library and into systems outside the scan need no +special handling and get none: nothing in the corpus defines ``car`` or +``format``, so they match no candidate and produce no edge. That falls out of +matching against definitions rather than against a name list, which is also +why no such list is maintained here. +""" +from __future__ import annotations + +from .paths import disambiguate_ambiguous_candidates +from .symbol_resolution import ( + build_label_index, + existing_edge_pairs, + iter_raw_calls, +) + +_COMMONLISP_SUFFIXES = (".lisp", ".cl", ".lsp", ".asd") + + +def resolve_commonlisp_calls( + per_file: list[dict], + all_nodes: list[dict], + all_edges: list[dict], +) -> None: + """Resolve Common Lisp calls whose target is defined in another file. + + Purely additive: emits edges only for raw calls the per-file pass could not + resolve locally. A name matching more than one definition goes through the + shared tie-breakers and is dropped unless exactly one survives, so an + ambiguous name produces no edge rather than a guess. + """ + label_index = build_label_index(all_nodes) + known = existing_edge_pairs(all_edges) + nid_to_source_file = { + str(n.get("id")): str(n.get("source_file", "")) + for n in all_nodes + if n.get("id") + } + + for rc in iter_raw_calls(per_file): + # raw_calls is shared by every language, so claim only our own. + if rc.get("lang") != "commonlisp": + continue + if not str(rc.get("source_file", "")).endswith(_COMMONLISP_SUFFIXES): + continue + callee = str(rc.get("callee", "")).strip() + caller = str(rc.get("caller_nid", "")) + if not callee or not caller: + continue + candidates = label_index.get(callee.lower(), []) + if not candidates: + continue + if len(candidates) == 1: + target: str | None = candidates[0] + else: + target = disambiguate_ambiguous_candidates( + candidates, + {c: nid_to_source_file.get(c, "") for c in candidates}, + str(rc.get("source_file", "")), + ) + if target is None: + continue + if target == caller: + continue + triple = (caller, target, "calls") + if triple in known: + continue + known.add(triple) + all_edges.append({ + "source": caller, + "target": target, + "relation": "calls", + "context": "call", + # INFERRED, matching the shared bare-name resolver: a name match + # across the corpus is weaker evidence than a call the extractor + # resolved inside one file. + "confidence": "INFERRED", + "confidence_score": 0.85, + "source_file": rc.get("source_file", ""), + "source_location": rc.get("source_location"), + "weight": 1.0, + }) diff --git a/graphify/extract.py b/graphify/extract.py index 151b97572b..f6eb81514a 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -22,6 +22,7 @@ register as register_language_resolver, run_language_resolvers, ) +from .commonlisp_resolution import resolve_commonlisp_calls from .ruby_resolution import resolve_ruby_member_calls from .csharp_dispatch import resolve_csharp_interface_dispatch from .pascal_resolution import resolve_pascal_inherited_calls @@ -4744,6 +4745,16 @@ def _resolve_kotlin_qualified_calls( register_language_resolver( LanguageResolver("ruby_member_calls", frozenset({".rb", ".rake"}), resolve_ruby_member_calls) ) +# Common Lisp bare-name calls across files. Lives in graphify.commonlisp_resolution; +# a CL system calls functions defined in sibling files by bare name, so the +# per-file pass cannot see most of its own call graph. +register_language_resolver( + LanguageResolver( + "commonlisp_calls", + frozenset({".lisp", ".cl", ".lsp", ".asd"}), + resolve_commonlisp_calls, + ) +) register_language_resolver( LanguageResolver("typescript_member_calls", frozenset({".ts", ".tsx", ".mts", ".cts", ".js", ".jsx"}), _resolve_typescript_member_calls) ) diff --git a/graphify/extractors/commonlisp.py b/graphify/extractors/commonlisp.py index 84c2c0766a..a720582974 100644 --- a/graphify/extractors/commonlisp.py +++ b/graphify/extractors/commonlisp.py @@ -470,9 +470,11 @@ def _process_form(top) -> bool: return True if first_lower in ("defclass", "define-condition"): # define-condition shares defclass's shape, (NAME (PARENTS) (SLOTS) ...), - # so the same handler reads the parent list the generic definer path - # never looks at. A condition hierarchy is inheritance and belongs in - # the graph as such. + # so the same handler gives it the inherits edges its parent types + # deserve. Routed here rather than to the generic definer path, which + # records the name and never reads the parent list: a condition + # hierarchy is inheritance, and a codebase that signals through + # conditions had that structure missing from its graph entirely. _handle_defclass(top) return True if first_lower in ("require", "ql:quickload"): @@ -519,6 +521,14 @@ def _walk_forms(parent) -> None: label_to_nid[normalised.lower()] = n["id"] seen_call_pairs: set[tuple[str, str]] = set() + # Callees this file cannot see. A Common Lisp system spreads its functions + # across files and calls them by bare name, so a per-file pass resolves only + # a minority of real calls. Report the rest rather than guessing: the + # cross-file resolver matches them against definitions the whole corpus + # knows about, which is also what keeps calls into the standard library out + # of the graph, since nothing in the corpus defines them. + raw_calls: list[dict] = [] + seen_raw: set[tuple[str, str]] = set() def walk_calls(node, caller_nid: str) -> None: if node.type == "defun": @@ -533,6 +543,17 @@ def walk_calls(node, caller_nid: str) -> None: seen_call_pairs.add(pair) add_edge(caller_nid, tgt_nid, "calls", node.start_point[0] + 1, confidence="EXTRACTED", weight=1.0) + elif not tgt_nid: + raw_pair = (caller_nid, callee.lower()) + if raw_pair not in seen_raw: + seen_raw.add(raw_pair) + raw_calls.append({ + "caller_nid": caller_nid, + "callee": callee, + "lang": "commonlisp", + "source_file": str_path, + "source_location": f"{node.start_point[0] + 1}", + }) for child in node.children: walk_calls(child, caller_nid) @@ -542,4 +563,4 @@ def walk_calls(node, caller_nid: str) -> None: clean_edges = [e for e in edges if e["source"] in seen_ids and (e["target"] in seen_ids or e["relation"] in ("imports", "imports_from"))] - return {"nodes": nodes, "edges": clean_edges} + return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls} diff --git a/tests/test_commonlisp_resolution.py b/tests/test_commonlisp_resolution.py new file mode 100644 index 0000000000..02ef6d23f0 --- /dev/null +++ b/tests/test_commonlisp_resolution.py @@ -0,0 +1,95 @@ +"""Tests for cross-file Common Lisp call resolution. + +A Common Lisp system spreads its functions across many files of one package +and calls them by bare name, with no receiver at the call site and no import +statement naming the target. The per-file extractor sees only the file it is +parsing, so most real calls have no candidate to bind to. It reports those as +raw_calls and graphify.commonlisp_resolution binds them against the merged +corpus. See that module's docstring for why the resolution rule differs from +the Pascal one it is modelled on. +""" +from __future__ import annotations + +from graphify.extract import extract, extract_commonlisp + + +def _call_edge(graph: dict, src_label: str, tgt_label: str) -> dict | None: + by_id = {n["id"]: n for n in graph["nodes"]} + for e in graph.get("edges", graph.get("links", [])): + if e.get("relation") != "calls": + continue + s = by_id.get(e.get("source"), {}).get("label") + t = by_id.get(e.get("target"), {}).get("label") + if s == src_label and t == tgt_label: + return e + return None + + +def _write(tmp_path, name: str, text: str): + p = tmp_path / name + p.write_text(text) + return p + + +def test_single_file_extraction_reports_unresolved_call(tmp_path): + """The gap this resolver closes: extracting the caller's file alone cannot + see the callee's file, so no calls edge may be invented there, and the call + must be reported rather than dropped.""" + b = _write(tmp_path, "b.lisp", "(defun caller (x) (helper x))\n") + r = extract_commonlisp(b) + assert _call_edge(r, "caller()", "helper()") is None + rc = next((c for c in r["raw_calls"] if c["callee"] == "helper"), None) + assert rc is not None + assert rc["caller_nid"] + assert rc["lang"] == "commonlisp" + + +def test_calls_resolve_across_files_by_name(tmp_path): + a = _write(tmp_path, "a.lisp", "(defun helper (x) (* x 2))\n") + b = _write(tmp_path, "b.lisp", "(defun caller (x) (helper x))\n") + graph = extract([a, b], cache_root=tmp_path, parallel=False) + edge = _call_edge(graph, "caller()", "helper()") + assert edge is not None + # A bare-name match across the corpus is weaker than a call the extractor + # resolved inside one file, and is marked as such. + assert edge.get("confidence") == "INFERRED" + + +def test_same_file_call_stays_extracted(tmp_path): + """Control: the per-file path still resolves its own calls, at full + confidence, so the resolver cannot pass by taking over everything.""" + a = _write( + tmp_path, "a.lisp", + "(defun helper (x) (* x 2))\n(defun local-caller (x) (helper x))\n", + ) + graph = extract([a], cache_root=tmp_path, parallel=False) + edge = _call_edge(graph, "local-caller()", "helper()") + assert edge is not None + assert edge.get("confidence") == "EXTRACTED" + + +def test_ambiguous_name_produces_no_edge(tmp_path): + """Two definitions of one name in unrelated files is not a resolution. + The single-candidate guard must drop it rather than pick one.""" + a = _write(tmp_path, "a.lisp", "(defun helper (x) 1)\n") + b = _write(tmp_path, "b.lisp", "(defun helper (x) 2)\n") + c = _write(tmp_path, "c.lisp", "(defun caller (x) (helper x))\n") + graph = extract([a, b, c], cache_root=tmp_path, parallel=False) + assert _call_edge(graph, "caller()", "helper()") is None + + +def test_standard_library_calls_produce_no_edges(tmp_path): + """Calls into the standard library need no denylist: nothing in the corpus + defines them, so they match no candidate. This is what keeps the graph from + gaining a node for every CL function a program happens to call.""" + a = _write(tmp_path, "a.lisp", "(defun f (x) (car x) (format nil \"~a\" x) (mapcar #'identity x))\n") + graph = extract([a], cache_root=tmp_path, parallel=False) + labels = {str(n.get("label", "")).strip("()").lower() for n in graph["nodes"]} + for stdlib in ("car", "format", "mapcar", "identity"): + assert stdlib not in labels, f"{stdlib} must not become a node" + + +def test_commonlisp_resolver_registered(): + from graphify.resolver_registry import registered_resolvers + names = {r.name for r in registered_resolvers()} + assert "commonlisp_calls" in names