diff --git a/graphify/commonlisp_resolution.py b/graphify/commonlisp_resolution.py new file mode 100644 index 0000000000..8bf87afd59 --- /dev/null +++ b/graphify/commonlisp_resolution.py @@ -0,0 +1,101 @@ +"""Cross-file resolution for Common Lisp calls by bare name. + +The per-file Common Lisp extractor (``extract_commonlisp``) resolves a call +only against the definitions in the file being parsed, because each file is +extracted independently. A Common Lisp system spreads its functions across +many files of one package and calls them by bare name with no receiver and no +import statement at the call site, so the per-file pass resolves a minority of +the real calls and the rest have no candidate to bind to. Those are reported +as ``raw_calls`` rather than guessed at. + +This resolver runs after all files are extracted (registered in +``graphify.resolver_registry``) with the full merged corpus available, so a +bare name can be matched against every definition the corpus knows. Modelled +on ``pascal_resolution`` -- same shape of problem, a call whose target is +structurally outside any one file's scope -- but the resolution rule differs +because the languages differ. Pascal walks an ``inherits`` chain, mirroring +Delphi's method lookup. Common Lisp has no receiver to type and no chain to +walk: a bare name in a package denotes one function, so the corpus-wide name +is the resolution, guarded by requiring a single candidate. + +Calls into the standard library and into systems outside the scan need no +special handling and get none: nothing in the corpus defines ``car`` or +``format``, so they match no candidate and produce no edge. That falls out of +matching against definitions rather than against a name list, which is also +why no such list is maintained here. +""" +from __future__ import annotations + +from .paths import disambiguate_ambiguous_candidates +from .symbol_resolution import ( + build_label_index, + existing_edge_pairs, + iter_raw_calls, +) + +_COMMONLISP_SUFFIXES = (".lisp", ".cl", ".lsp", ".asd") + + +def resolve_commonlisp_calls( + per_file: list[dict], + all_nodes: list[dict], + all_edges: list[dict], +) -> None: + """Resolve Common Lisp calls whose target is defined in another file. + + Purely additive: emits edges only for raw calls the per-file pass could not + resolve locally. A name matching more than one definition goes through the + shared tie-breakers and is dropped unless exactly one survives, so an + ambiguous name produces no edge rather than a guess. + """ + label_index = build_label_index(all_nodes) + known = existing_edge_pairs(all_edges) + nid_to_source_file = { + str(n.get("id")): str(n.get("source_file", "")) + for n in all_nodes + if n.get("id") + } + + for rc in iter_raw_calls(per_file): + # raw_calls is shared by every language, so claim only our own. + if rc.get("lang") != "commonlisp": + continue + if not str(rc.get("source_file", "")).endswith(_COMMONLISP_SUFFIXES): + continue + callee = str(rc.get("callee", "")).strip() + caller = str(rc.get("caller_nid", "")) + if not callee or not caller: + continue + candidates = label_index.get(callee.lower(), []) + if not candidates: + continue + if len(candidates) == 1: + target: str | None = candidates[0] + else: + target = disambiguate_ambiguous_candidates( + candidates, + {c: nid_to_source_file.get(c, "") for c in candidates}, + str(rc.get("source_file", "")), + ) + if target is None: + continue + if target == caller: + continue + triple = (caller, target, "calls") + if triple in known: + continue + known.add(triple) + all_edges.append({ + "source": caller, + "target": target, + "relation": "calls", + "context": "call", + # INFERRED, matching the shared bare-name resolver: a name match + # across the corpus is weaker evidence than a call the extractor + # resolved inside one file. + "confidence": "INFERRED", + "confidence_score": 0.85, + "source_file": rc.get("source_file", ""), + "source_location": rc.get("source_location"), + "weight": 1.0, + }) diff --git a/graphify/extract.py b/graphify/extract.py index 90a6ab0267..f6eb81514a 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -22,6 +22,7 @@ register as register_language_resolver, run_language_resolvers, ) +from .commonlisp_resolution import resolve_commonlisp_calls from .ruby_resolution import resolve_ruby_member_calls from .csharp_dispatch import resolve_csharp_interface_dispatch from .pascal_resolution import resolve_pascal_inherited_calls @@ -2809,7 +2810,12 @@ def _rewire_unique_stub_nodes(nodes: list[dict], edges: list[dict]) -> None: stub_ids = {str(s.get("id")) for s in stubs if s.get("id")} stub_families: dict[str, set] = {} supertype_stub_ids: set[str] = set() # stubs used as a base type — never a function - _SUPERTYPE_RELATIONS = {"inherits", "implements", "extends"} + # `specializes` joins these because a Common Lisp method dispatches on a + # TYPE, and functions and types occupy separate namespaces there: `list`, + # `stream` and `condition` are all routinely both. Without it a specializer + # stub can bind to a same-named function, which asserts a dispatch + # relationship that does not exist. + _SUPERTYPE_RELATIONS = {"inherits", "implements", "extends", "specializes"} for edge in edges: rel = edge.get("relation") for endpoint in ("source", "target"): @@ -4739,6 +4745,16 @@ def _resolve_kotlin_qualified_calls( register_language_resolver( LanguageResolver("ruby_member_calls", frozenset({".rb", ".rake"}), resolve_ruby_member_calls) ) +# Common Lisp bare-name calls across files. Lives in graphify.commonlisp_resolution; +# a CL system calls functions defined in sibling files by bare name, so the +# per-file pass cannot see most of its own call graph. +register_language_resolver( + LanguageResolver( + "commonlisp_calls", + frozenset({".lisp", ".cl", ".lsp", ".asd"}), + resolve_commonlisp_calls, + ) +) register_language_resolver( LanguageResolver("typescript_member_calls", frozenset({".ts", ".tsx", ".mts", ".cts", ".js", ".jsx"}), _resolve_typescript_member_calls) ) diff --git a/graphify/extractors/commonlisp.py b/graphify/extractors/commonlisp.py index 2ed9d9fbdd..a720582974 100644 --- a/graphify/extractors/commonlisp.py +++ b/graphify/extractors/commonlisp.py @@ -319,7 +319,8 @@ def _handle_defun_node(node) -> None: syms = [c for c in param.children if c.type == "sym_lit"] if len(syms) >= 2: specializer_name = _text(syms[1]) - spec_nid = _cl_id(stem, specializer_name) + spec_nid = ensure_class_ref( + specializer_name, param.start_point[0] + 1) add_edge(func_nid, spec_nid, "specializes", param.start_point[0] + 1) break @@ -467,7 +468,13 @@ def _process_form(top) -> bool: current_package = _text(child) break return True - if first_lower == "defclass": + if first_lower in ("defclass", "define-condition"): + # define-condition shares defclass's shape, (NAME (PARENTS) (SLOTS) ...), + # so the same handler gives it the inherits edges its parent types + # deserve. Routed here rather than to the generic definer path, which + # records the name and never reads the parent list: a condition + # hierarchy is inheritance, and a codebase that signals through + # conditions had that structure missing from its graph entirely. _handle_defclass(top) return True if first_lower in ("require", "ql:quickload"): @@ -514,6 +521,14 @@ def _walk_forms(parent) -> None: label_to_nid[normalised.lower()] = n["id"] seen_call_pairs: set[tuple[str, str]] = set() + # Callees this file cannot see. A Common Lisp system spreads its functions + # across files and calls them by bare name, so a per-file pass resolves only + # a minority of real calls. Report the rest rather than guessing: the + # cross-file resolver matches them against definitions the whole corpus + # knows about, which is also what keeps calls into the standard library out + # of the graph, since nothing in the corpus defines them. + raw_calls: list[dict] = [] + seen_raw: set[tuple[str, str]] = set() def walk_calls(node, caller_nid: str) -> None: if node.type == "defun": @@ -528,6 +543,17 @@ def walk_calls(node, caller_nid: str) -> None: seen_call_pairs.add(pair) add_edge(caller_nid, tgt_nid, "calls", node.start_point[0] + 1, confidence="EXTRACTED", weight=1.0) + elif not tgt_nid: + raw_pair = (caller_nid, callee.lower()) + if raw_pair not in seen_raw: + seen_raw.add(raw_pair) + raw_calls.append({ + "caller_nid": caller_nid, + "callee": callee, + "lang": "commonlisp", + "source_file": str_path, + "source_location": f"{node.start_point[0] + 1}", + }) for child in node.children: walk_calls(child, caller_nid) @@ -537,4 +563,4 @@ def walk_calls(node, caller_nid: str) -> None: clean_edges = [e for e in edges if e["source"] in seen_ids and (e["target"] in seen_ids or e["relation"] in ("imports", "imports_from"))] - return {"nodes": nodes, "edges": clean_edges} + return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls} diff --git a/tests/test_commonlisp_resolution.py b/tests/test_commonlisp_resolution.py new file mode 100644 index 0000000000..02ef6d23f0 --- /dev/null +++ b/tests/test_commonlisp_resolution.py @@ -0,0 +1,95 @@ +"""Tests for cross-file Common Lisp call resolution. + +A Common Lisp system spreads its functions across many files of one package +and calls them by bare name, with no receiver at the call site and no import +statement naming the target. The per-file extractor sees only the file it is +parsing, so most real calls have no candidate to bind to. It reports those as +raw_calls and graphify.commonlisp_resolution binds them against the merged +corpus. See that module's docstring for why the resolution rule differs from +the Pascal one it is modelled on. +""" +from __future__ import annotations + +from graphify.extract import extract, extract_commonlisp + + +def _call_edge(graph: dict, src_label: str, tgt_label: str) -> dict | None: + by_id = {n["id"]: n for n in graph["nodes"]} + for e in graph.get("edges", graph.get("links", [])): + if e.get("relation") != "calls": + continue + s = by_id.get(e.get("source"), {}).get("label") + t = by_id.get(e.get("target"), {}).get("label") + if s == src_label and t == tgt_label: + return e + return None + + +def _write(tmp_path, name: str, text: str): + p = tmp_path / name + p.write_text(text) + return p + + +def test_single_file_extraction_reports_unresolved_call(tmp_path): + """The gap this resolver closes: extracting the caller's file alone cannot + see the callee's file, so no calls edge may be invented there, and the call + must be reported rather than dropped.""" + b = _write(tmp_path, "b.lisp", "(defun caller (x) (helper x))\n") + r = extract_commonlisp(b) + assert _call_edge(r, "caller()", "helper()") is None + rc = next((c for c in r["raw_calls"] if c["callee"] == "helper"), None) + assert rc is not None + assert rc["caller_nid"] + assert rc["lang"] == "commonlisp" + + +def test_calls_resolve_across_files_by_name(tmp_path): + a = _write(tmp_path, "a.lisp", "(defun helper (x) (* x 2))\n") + b = _write(tmp_path, "b.lisp", "(defun caller (x) (helper x))\n") + graph = extract([a, b], cache_root=tmp_path, parallel=False) + edge = _call_edge(graph, "caller()", "helper()") + assert edge is not None + # A bare-name match across the corpus is weaker than a call the extractor + # resolved inside one file, and is marked as such. + assert edge.get("confidence") == "INFERRED" + + +def test_same_file_call_stays_extracted(tmp_path): + """Control: the per-file path still resolves its own calls, at full + confidence, so the resolver cannot pass by taking over everything.""" + a = _write( + tmp_path, "a.lisp", + "(defun helper (x) (* x 2))\n(defun local-caller (x) (helper x))\n", + ) + graph = extract([a], cache_root=tmp_path, parallel=False) + edge = _call_edge(graph, "local-caller()", "helper()") + assert edge is not None + assert edge.get("confidence") == "EXTRACTED" + + +def test_ambiguous_name_produces_no_edge(tmp_path): + """Two definitions of one name in unrelated files is not a resolution. + The single-candidate guard must drop it rather than pick one.""" + a = _write(tmp_path, "a.lisp", "(defun helper (x) 1)\n") + b = _write(tmp_path, "b.lisp", "(defun helper (x) 2)\n") + c = _write(tmp_path, "c.lisp", "(defun caller (x) (helper x))\n") + graph = extract([a, b, c], cache_root=tmp_path, parallel=False) + assert _call_edge(graph, "caller()", "helper()") is None + + +def test_standard_library_calls_produce_no_edges(tmp_path): + """Calls into the standard library need no denylist: nothing in the corpus + defines them, so they match no candidate. This is what keeps the graph from + gaining a node for every CL function a program happens to call.""" + a = _write(tmp_path, "a.lisp", "(defun f (x) (car x) (format nil \"~a\" x) (mapcar #'identity x))\n") + graph = extract([a], cache_root=tmp_path, parallel=False) + labels = {str(n.get("label", "")).strip("()").lower() for n in graph["nodes"]} + for stdlib in ("car", "format", "mapcar", "identity"): + assert stdlib not in labels, f"{stdlib} must not become a node" + + +def test_commonlisp_resolver_registered(): + from graphify.resolver_registry import registered_resolvers + names = {r.name for r in registered_resolvers()} + assert "commonlisp_calls" in names diff --git a/tests/test_commonlisp_specializer_binding.py b/tests/test_commonlisp_specializer_binding.py new file mode 100644 index 0000000000..20dd7e8379 --- /dev/null +++ b/tests/test_commonlisp_specializer_binding.py @@ -0,0 +1,32 @@ +"""A Common Lisp method specializer must never bind to a same-named function. + +Functions and types occupy separate namespaces in Common Lisp, so one symbol is +routinely both: `list`, `stream`, `condition`, `pathname`. When a specializer +names a type the corpus does not define, the cross-file stub must stay +unresolved rather than collapse onto a function that happens to share the name, +which would assert a dispatch relationship that does not exist. +""" +from __future__ import annotations + +from graphify.extract import extract + + +def test_specializer_does_not_bind_to_same_named_function(tmp_path): + a = tmp_path / "a.lisp" + a.write_text("(defun square (x) (* x x))\n") # a function named square + b = tmp_path / "b.lisp" + b.write_text( + "(defgeneric area (obj))\n" + "(defmethod area ((obj square)) 1)\n" # dispatches on a TYPE named square + ) + graph = extract([a, b], cache_root=tmp_path, parallel=False) + by_id = {n["id"]: n for n in graph["nodes"]} + for e in graph["edges"]: + if e.get("relation") != "specializes": + continue + target = by_id.get(e.get("target"), {}) + label = str(target.get("label", "")) + assert not label.endswith("()"), ( + f"specializes bound to the function {label!r}; " + "no class of that name exists in the corpus" + ) diff --git a/tests/test_languages.py b/tests/test_languages.py index aa7d49486f..7ab58a70bc 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -3811,6 +3811,73 @@ def test_cl_crossfile_superclass_inherits_edge_survives(tmp_path): assert base["source_file"] == "", "cross-file superclass must be a sourceless stub" +@_needs_commonlisp +def test_cl_crossfile_specializer_specializes_edge_survives(tmp_path): + """A method specialising on a class from another file must keep its edge. + + The specializer was resolved with a file-scoped id, so a method dispatching + on a class defined elsewhere produced an edge to a node that did not exist + and the dangling-edge filter removed it. Since most methods in a CL system + live apart from the class they dispatch on, that erased nearly every + specializes edge in a real codebase. Resolve through the same sourceless + stub the cross-file superclass path uses. + """ + f = tmp_path / "shapes.lisp" + f.write_text( + "(defclass circle () ())\n" + "(defgeneric area (obj))\n" + "(defmethod area ((obj circle)) 1)\n" + "(defmethod area ((obj square)) 2)\n" + ) + r = extract_commonlisp(f) + assert "error" not in r + id_to_node = {n["id"]: n for n in r["nodes"]} + targets = { + id_to_node[e["target"]]["label"] + for e in r["edges"] + if e["relation"] == "specializes" + and e["source"] in id_to_node and e["target"] in id_to_node + } + # same-file specializer still binds locally + assert "circle" in targets + # cross-file specializer survives via a sourceless stub instead of being dropped + assert "square" in targets + square = next(n for n in r["nodes"] if n["label"] == "square") + assert square["source_file"] == "", "cross-file specializer must be a sourceless stub" + + +@_needs_commonlisp +def test_cl_define_condition_emits_inherits_edges(tmp_path): + """A condition hierarchy must produce inherits edges like a class one. + + define-condition was routed to the generic definer path, which records the + name and never reads the parent list, so a condition hierarchy appeared in + the graph as unrelated nodes even with both ends in the same file. Codebases + that signal through conditions had that structure missing entirely. + """ + f = tmp_path / "conditions.lisp" + f.write_text( + "(define-condition app-error (error) ())\n" + "(define-condition parse-error (app-error) ())\n" + "(defclass control () ())\n" + "(defclass control-sub (control) ())\n" + ) + r = extract_commonlisp(f) + assert "error" not in r + id_to_node = {n["id"]: n for n in r["nodes"]} + inherits = { + (id_to_node[e["source"]]["label"], id_to_node[e["target"]]["label"]) + for e in r["edges"] + if e["relation"] == "inherits" + and e["source"] in id_to_node and e["target"] in id_to_node + } + assert ("parse-error", "app-error") in inherits + assert ("app-error", "error") in inherits + # control: the defclass path was already working and must stay working + assert ("control-sub", "control") in inherits + + + @_needs_commonlisp def test_cl_imports(): r = extract_commonlisp(FIXTURES / "sample.lisp")