From a232c311280e190d7e0aef8e20e9e45764a6b507 Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Wed, 18 Nov 2026 06:53:22 +0100 Subject: [PATCH] feat(hydra-gates): dependency analysis and modularity view tools for every app Port the dependency analysis, the component-view rating, and the deptrac config and summary helpers from openregister into scripts/lib, next to the gate-122 scorer they import. The app root is an argument, the components file defaults to /sigrid.components.yaml, and an app without one gets the directory views only. The foundation merge of view D is set with --foundation and --merge (default Kernel and Core). deptrac stays a per-app vendor-bin install; the package ships a generic layers.yaml and deptrac.sh. Assisted-by: ClaudeCode:claude-opus-5-5 --- hydra-gates/README.md | 56 +++ .../scripts/lib/analyse_dependencies.py | 464 ++++++++++++++++++ hydra-gates/scripts/lib/deptrac.sh | 70 +++ hydra-gates/scripts/lib/deptrac/layers.yaml | 48 ++ hydra-gates/scripts/lib/deptrac_areas.py | 100 ++++ hydra-gates/scripts/lib/deptrac_summary.py | 61 +++ .../scripts/lib/rate_component_views.py | 251 ++++++++++ .../scripts/lib/test_analyse_dependencies.py | 144 ++++++ hydra-gates/scripts/lib/test_deptrac_tools.py | 129 +++++ .../scripts/lib/test_rate_component_views.py | 210 ++++++++ hydra-gates/tests/helper-suite-timings.tsv | 3 + 11 files changed, 1536 insertions(+) create mode 100755 hydra-gates/scripts/lib/analyse_dependencies.py create mode 100755 hydra-gates/scripts/lib/deptrac.sh create mode 100644 hydra-gates/scripts/lib/deptrac/layers.yaml create mode 100755 hydra-gates/scripts/lib/deptrac_areas.py create mode 100755 hydra-gates/scripts/lib/deptrac_summary.py create mode 100755 hydra-gates/scripts/lib/rate_component_views.py create mode 100644 hydra-gates/scripts/lib/test_analyse_dependencies.py create mode 100644 hydra-gates/scripts/lib/test_deptrac_tools.py create mode 100644 hydra-gates/scripts/lib/test_rate_component_views.py diff --git a/hydra-gates/README.md b/hydra-gates/README.md index 2c5a29f7..c4479408 100644 --- a/hydra-gates/README.md +++ b/hydra-gates/README.md @@ -801,6 +801,62 @@ method body. --- +## Dependency and modularity analysis (tools, not gates) + +Next to gate 122's scorer (`scripts/lib/check_sig_maintainability.py`) the +package ships four tools that measure how a Nextcloud app's PHP backend hangs +together. They import the scorer from the same directory and reuse its own +dependency graph, line count, `structure()` and `rating_estimate()`. No gate +runs them and they never fail a build. Every number they print is an estimate +on the scorer's 0.5 to 5.5 scale, not an official rating. + +A Nextcloud app sorts its classes by kind: controllers in `lib/Controller/`, +mappers in `lib/Db/`. So a directory under `lib/` is a layer, not a component. +These tools measure the backend both ways: per directory, and per functional +area, as declared in the app's `sigrid.components.yaml` (a `components:` block +of include patterns). + +| Tool | What it measures | +|---|---| +| `analyse_dependencies.py` | Hidden code, communication lines and violation degree for the directory view and the functional view; layering (references going down, level or up between entry points, services, persistence and shared types); cycles (Tarjan) and the largest cyclic set; per area the exposed code and the pairs that reference each other both ways | +| `rate_component_views.py` | The estimated modularity rating under five component definitions: A the backend as one component, B every directory under `lib/`, C the functional areas, D as C with a foundation merged in, E as D without the foundation's references to other areas (a simulation). Also the exposed code by kind of file and by area, and per rating the largest exposed share and violation degree that still earn it | +| `deptrac_areas.py` | A deptrac config with one layer per functional area, in which every area may use the shared area and nothing else | +| `deptrac_summary.py` | A deptrac JSON report counted per layer pair: occurrences and distinct class pairs | + +``` +L=vendor/conduction/hydra-gates/hydra-gates/scripts/lib # or hydra-gates/scripts/lib in this repo + +python3 $L/analyse_dependencies.py . --out /tmp/deps # add --rev REV to measure an older commit +python3 $L/check_sig_maintainability.py report . --json /tmp/base.json +python3 $L/rate_component_views.py . --report /tmp/base.json --json /tmp/views.json +# --foundation NAME and --merge NAME[,NAME] pick the merge of view D (default Kernel and Core) +python3 $L/deptrac_areas.py . --out deptrac-areas.yaml # --check exits 1 when the file is stale +bash $L/deptrac.sh . layers # the generic layering rules +bash $L/deptrac.sh . deptrac-areas.yaml # the functional areas +``` + +The app root is the first argument. The components file defaults to +`/sigrid.components.yaml`; `--components FILE` points elsewhere. An app +without one still gets the directory view, the layering, the cycles, and views +A and B. The functional view and views C, D and E are skipped, and the output +says so. + +deptrac is not a dependency of this package. An app that wants the second +reading installs it with +[bamarni/composer-bin-plugin](https://github.com/bamarni/composer-bin-plugin) +in `vendor-bin/deptrac/composer.json`: + +```json +{ "require-dev": { "deptrac/deptrac": "^4.7" } } +``` + +and runs `composer --working-dir=vendor-bin/deptrac install`. `deptrac.sh` +looks for `/vendor-bin/deptrac/vendor/bin/deptrac`; `DEPTRAC=/path` +overrides it. `scripts/lib/deptrac/layers.yaml` encodes the same layer rule as +`analyse_dependencies.py`, and a test keeps the two in step. + +--- + ## Testing the package ``` diff --git a/hydra-gates/scripts/lib/analyse_dependencies.py b/hydra-gates/scripts/lib/analyse_dependencies.py new file mode 100755 index 00000000..3940a8b8 --- /dev/null +++ b/hydra-gates/scripts/lib/analyse_dependencies.py @@ -0,0 +1,464 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Conduction +# SPDX-License-Identifier: EUPL-1.2 +"""Dependency analysis of a Nextcloud app's PHP backend, in two component views. + + python3 analyse_dependencies.py [--rev REV] [--components FILE] [--out DIR] + +The module graph is the one the gate-122 scorer (check_sig_maintainability.py, same directory) +builds: one node per PHP file under /lib/ (lib/Migration excepted, which the analysis scope +leaves out by default), and an edge from a file to every class of the app it names in a `use` +import, a fully qualified name or a bare name in its own namespace. Comments are stripped first. +Lines of code are counted the same way as there. + +On that graph the script reports: + +- two component views, each with hidden code (the share of code no other component references), + communication lines and violation degree, computed with the scorer's own functions: + the directory view (every directory under lib/ is a component) and the functional view + (the `components:` block of the components file, default /sigrid.components.yaml); +- layering: every file gets a role from its directory (entry point, service, persistence or + shared type) and every edge between two roles is counted as downward, same-level or upward; +- cycles: the strongly connected sets of files, the largest one by directory and area, and the + upward edges inside it; +- per functional area: internal and outgoing references, exposed code, and the area pairs that + reference each other in both directions. + +An app without a components file gets the directory view, the layering and the cycles; the +functional view and the per-area tables are skipped, and the report says so. + +--rev measures a commit instead of the working tree: lib/ of that commit is exported with +`git archive` into a temporary directory. The functional definition is always read from +--components in the working tree, so an older commit is measured against today's definition. A +file the definition does not match (it moved or was deleted since) is assigned the area that most +files in its directory have, and the report counts those files. + +--out writes results.json, report.md and areas.dot to DIR. Without --out only report.md is printed. +All numbers are estimates on the scorer's scale, not an official measurement. +""" +import argparse +import collections +import contextlib +import json +import os +import subprocess +import sys +import tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +import check_sig_maintainability as tool # noqa: E402 + +COMPONENTS_FILE = "sigrid.components.yaml" + +# Layer of a directory name, from outer (3) to inner (0). The first directory on the path below +# lib/ that is named here decides the role, so lib/Files/Controller/X.php is an entry point and +# lib/Search/Db/View.php is persistence. deptrac/layers.yaml encodes the same rule. +ROLES = { + "Controller": ("entry point", 3), "Command": ("entry point", 3), "Listener": ("entry point", 3), + "BackgroundJob": ("entry point", 3), "Repair": ("entry point", 3), "Middleware": ("entry point", 3), + "Settings": ("entry point", 3), "Sections": ("entry point", 3), "Dashboard": ("entry point", 3), + "Capabilities": ("entry point", 3), + "Service": ("service", 2), + "Db": ("persistence", 1), + "Event": ("shared type", 0), "Exception": ("shared type", 0), "Kernel": ("shared type", 0), + "Contract": ("shared type", 0), "Dto": ("shared type", 0), "Support": ("shared type", 0), + "Formats": ("shared type", 0), +} +# lib/AppInfo wires every component together and is not a layer. +BOOTSTRAP = "AppInfo" + + +def default_components_file(app): + return os.path.join(app, COMPONENTS_FILE) + + +def read_components(path): + """The `components:` definition in `path`, parsed by the scorer's own scope-file reader, or + None when the file is missing or declares no `components:` block. The reader only looks for + sigrid.yaml in a directory, so the file is copied there first.""" + if not path or not os.path.isfile(path): + return None + with tempfile.TemporaryDirectory() as tmp: + with open(path, encoding="utf-8", errors="replace") as src, \ + open(os.path.join(tmp, "sigrid.yaml"), "w", encoding="utf-8") as dst: + dst.write(src.read()) + definition = tool.scope_file_components(tmp) + if not definition or "components" not in definition: + return None + return definition + + +@contextlib.contextmanager +def components_of(definition): + """Make the scorer's structure() use `definition` (a `components:` definition), or its + directory rule when `definition` is None, for the duration of the block.""" + saved = tool.COMPONENTS_FROM_SCOPE, tool.scope_file_components + tool.COMPONENTS_FROM_SCOPE = definition is not None + tool.scope_file_components = lambda _app: definition + try: + yield + finally: + tool.COMPONENTS_FROM_SCOPE, tool.scope_file_components = saved + + +def export_rev(app, rev, dest): + """Write lib/ of commit `rev` of the repository at `app` into `dest`; return the full hash.""" + full = subprocess.run(["git", "-C", app, "rev-parse", rev], check=True, capture_output=True, text=True).stdout.strip() + archive = subprocess.run(["git", "-C", app, "archive", full, "lib"], check=True, capture_output=True).stdout + subprocess.run(["tar", "-x", "-C", dest], input=archive, check=True) + return full + + +def load_graph(app): + """({module: set(modules)}, {module: loc}) for lib/ of `app`, lib/Migration left out.""" + rels = [] + for root, _dirs, files in os.walk(os.path.join(app, "lib")): + for name in files: + rel = os.path.relpath(os.path.join(root, name), app) + if name.endswith(".php") and not rel.startswith("lib/Migration/"): + rels.append(rel) + deps = tool.php_dependencies(app, sorted(rels)) + loc = {m: len(tool.code_lines(tool.read_text(os.path.join(app, m)), "php")) for m in deps} + return deps, loc + + +def role_of(module): + """(role, rank) of a module, or None for the bootstrap and for unlayered directories.""" + parts = module.split("/")[1:-1] + if parts and parts[0] == BOOTSTRAP: + return None + for part in parts: + if part in ROLES: + return ROLES[part] + return None + + +def directory_component(module): + parts = module.split("/") + return parts[1] if len(parts) > 2 else "(root)" + + +def functional_components(modules, components_file): + """({module: area}, [modules placed by directory majority]) from a `components:` block, or + (None, []) when the file is missing or has no such block.""" + definition = read_components(components_file) + if definition is None: + return None, [] + _source, match = tool.scope_component_factory(definition) + area = {m: match(m) for m in modules} + votes = collections.defaultdict(collections.Counter) + for m, a in area.items(): + if a != tool.UNASSIGNED: + parts = m.split("/") + for k in range(2, len(parts)): + votes["/".join(parts[:k])][a] += 1 + placed = [] + for m, a in area.items(): + if a != tool.UNASSIGNED: + continue + parts = m.split("/") + for k in range(len(parts) - 1, 1, -1): + if votes["/".join(parts[:k])]: + area[m] = votes["/".join(parts[:k])].most_common(1)[0][0] + placed.append(m) + break + return area, sorted(placed) + + +def incoming_of(deps): + incoming = collections.defaultdict(set) + for src, targets in deps.items(): + for t in targets: + incoming[t].add(src) + return incoming + + +def component_view(deps, loc, comp): + """Hidden code, communication lines and violation degree, as the scorer computes them.""" + incoming = incoming_of(deps) + total = sum(loc.values()) or 1 + hidden = sum(loc[m] for m in loc if all(comp[s] == comp[m] for s in incoming[m])) + lines = collections.Counter() + for t, srcs in incoming.items(): + for s in srcs: + if comp[s] != comp[t]: + lines[(comp[s], comp[t])] += 1 + degree, kinds, _examples = tool.violation_degree(dict(lines)) + sizes = collections.Counter() + for m in loc: + sizes[comp[m]] += loc[m] + share = round(hidden / total, 4) + return { + "components": len(sizes), + "largest_component": sizes.most_common(1)[0][0], + "largest_share": round(sizes.most_common(1)[0][1] / total, 4), + "hidden_share": share, + "hidden_stars_estimate": tool.stars_of("component_independence", share), + "communication_lines": len(lines), + "violation_degree": degree, + "violation_stars_estimate": tool.stars_of("component_entanglement", degree), + "violations": kinds, + "loc_per_component": dict(sizes.most_common()), + } + + +def layering(deps): + """Edges between two layered roles, split into downward, same-level and upward.""" + counts = collections.Counter() + upward = collections.defaultdict(list) + for src, targets in deps.items(): + rs = role_of(src) + for t in targets: + rt = role_of(t) + if rs is None or rt is None: + counts["unlayered"] += 1 + continue + if rs[1] > rt[1]: + counts["downward"] += 1 + elif rs[1] == rt[1]: + counts["same level"] += 1 + else: + counts["upward"] += 1 + upward[f"{rs[0]} -> {rt[0]}"].append(f"{src} -> {t}") + return counts, {k: sorted(v) for k, v in sorted(upward.items(), key=lambda kv: -len(kv[1]))} + + +def strongly_connected(deps): + """Strongly connected sets of modules (Tarjan, iterative), largest first.""" + index, low, on_stack, stack, out = {}, {}, set(), [], [] + counter = 0 + for start in sorted(deps): + if start in index: + continue + work = [(start, iter(sorted(deps[start])))] + index[start] = low[start] = counter + counter += 1 + stack.append(start) + on_stack.add(start) + while work: + node, it = work[-1] + advanced = False + for nxt in it: + if nxt not in deps: + continue + if nxt not in index: + index[nxt] = low[nxt] = counter + counter += 1 + stack.append(nxt) + on_stack.add(nxt) + work.append((nxt, iter(sorted(deps[nxt])))) + advanced = True + break + if nxt in on_stack: + low[node] = min(low[node], index[nxt]) + if advanced: + continue + work.pop() + if work: + low[work[-1][0]] = min(low[work[-1][0]], low[node]) + if low[node] == index[node]: + group = set() + while True: + m = stack.pop() + on_stack.discard(m) + group.add(m) + if m == node: + break + out.append(group) + return sorted(out, key=lambda g: (-len(g), sorted(g)[0])) + + +def cycles(deps, loc, area): + groups = [g for g in strongly_connected(deps) if len(g) > 1] + if not groups: + return {"cyclic_sets": 0} + big = groups[0] + up = [] + for src in sorted(big): + rs = role_of(src) + for t in sorted(deps[src] & big): + rt = role_of(t) + if rs and rt and rs[1] < rt[1]: + up.append(f"{src} -> {t}") + hubs = sorted(big, key=lambda m: (-len(deps[m] & big), m))[:10] + return { + "cyclic_sets": len(groups), + "files_in_cyclic_sets": sum(len(g) for g in groups), + "set_sizes": [len(g) for g in groups[:10]], + "largest": { + "files": len(big), + "loc": sum(loc[m] for m in big), + "loc_share": round(sum(loc[m] for m in big) / (sum(loc.values()) or 1), 4), + "by_directory": dict(collections.Counter("/".join(m.split("/")[1:3]) if m.count("/") > 2 else m.split("/")[1] for m in big).most_common()), + "by_area": dict(collections.Counter(area[m] for m in big).most_common()) if area else None, + "most_outgoing_inside": [f"{m} ({len(deps[m] & big)} references inside the set)" for m in hubs], + "upward_edges_inside": up, + "members": sorted(big), + }, + } + + +def areas(deps, loc, area): + incoming = incoming_of(deps) + stats = {a: {"loc": 0, "exposed_loc": 0, "internal_refs": 0, "outgoing_refs": 0, "incoming_refs": 0} for a in set(area.values())} + pairs = collections.Counter() + targets = collections.defaultdict(collections.Counter) + for m in loc: + stats[area[m]]["loc"] += loc[m] + if any(area[s] != area[m] for s in incoming[m]): + stats[area[m]]["exposed_loc"] += loc[m] + for src, ts in deps.items(): + for t in ts: + a, b = area[src], area[t] + if a == b: + stats[a]["internal_refs"] += 1 + else: + stats[a]["outgoing_refs"] += 1 + stats[b]["incoming_refs"] += 1 + pairs[(a, b)] += 1 + targets[(a, b)][t] += 1 + both = [] + for (a, b), w in pairs.most_common(): + if a < b and (b, a) in pairs: + w2 = pairs[(b, a)] + light = (a, b) if w <= w2 else (b, a) + both.append({"pair": [a, b], "refs": [w, w2], "lighter": f"{light[0]} -> {light[1]}", + "lighter_targets": [t for t, _n in targets[light].most_common(5)]}) + both.sort(key=lambda p: -min(p["refs"])) + return stats, both, pairs + + +def pct(x): + return f"{100 * x:.1f}%" + + +def report_md(r): + out = [f"# Dependency analysis of lib/ at {r['commit']}", "", + f"{r['modules']} PHP files, {r['loc']} lines of code, {r['references']} references between files " + f"(lib/Migration left out). Produced by hydra-gates `analyse_dependencies.py`; every number is " + "an estimate on the gate-122 scorer's scale.", "", + "## Component views", "", + "| View | Components | Largest | Hidden code | Communication lines | Violation degree |", + "|---|---|---|---|---|---|"] + for name, v in r["views"].items(): + out.append(f"| {name} | {v['components']} | {v['largest_component']} ({pct(v['largest_share'])}) | " + f"{pct(v['hidden_share'])} | {v['communication_lines']} | {v['violation_degree']} |") + if r["functional_definition"] is None: + out += ["", f"No `components:` block in {r['components_file']}: the functional view and the " + "per-area tables are skipped."] + elif r["placed_by_directory"]: + out += ["", f"{len(r['placed_by_directory'])} files are not matched by the functional definition " + "and were placed by directory majority (see results.json)."] + lay = r["layering"]["counts"] + layered = lay.get("downward", 0) + lay.get("same level", 0) + lay.get("upward", 0) + out += ["", "## Layering", "", + f"Of {layered} references between layered files, {lay.get('downward', 0)} go downward " + f"({pct(lay.get('downward', 0) / (layered or 1))}), {lay.get('same level', 0)} stay on the same level and " + f"{lay.get('upward', 0)} go upward ({pct(lay.get('upward', 0) / (layered or 1))}). " + f"{lay.get('unlayered', 0)} references touch the bootstrap or an unlayered directory.", "", + "| Upward reference | Count | Example |", "|---|---|---|"] + for kind, edges in r["layering"]["upward"].items(): + out.append(f"| {kind} | {len(edges)} | `{edges[0]}` |") + c = r["cycles"] + if c.get("cyclic_sets"): + big = c["largest"] + out += ["", "## Cycles", "", + f"{c['cyclic_sets']} sets of files reference each other in a cycle, {c['files_in_cyclic_sets']} files in all. " + f"The largest holds {big['files']} files and {big['loc']} lines ({pct(big['loc_share'])} of lib/).", "", + "By directory: " + ", ".join(f"{k} {v}" for k, v in big["by_directory"].items()), "", + *(["By area: " + ", ".join(f"{k} {v}" for k, v in big["by_area"].items()), ""] if big["by_area"] else []), + "Files with the most references inside the set:", ""] + out += [f"- `{h}`" for h in big["most_outgoing_inside"]] + out += ["", f"{len(big['upward_edges_inside'])} of the references inside the set go upward a layer; results.json lists them."] + if r["areas"] is None: + return "\n".join(out) + "\n" + out += ["", "## Functional areas", "", + "| Area | Lines | Exposed | Internal refs | Outgoing refs | Incoming refs |", "|---|---|---|---|---|---|"] + for a, s in sorted(r["areas"].items(), key=lambda kv: -kv[1]["loc"]): + out.append(f"| {a} | {s['loc']} | {pct(s['exposed_loc'] / (s['loc'] or 1))} | {s['internal_refs']} | " + f"{s['outgoing_refs']} | {s['incoming_refs']} |") + out += ["", "Area pairs that reference each other in both directions, heaviest lighter direction first:", "", + "| Pair | References | Lighter direction | Most referenced in the lighter direction |", "|---|---|---|---|"] + for p in r["two_way_pairs"][:15]: + out.append(f"| {p['pair'][0]} / {p['pair'][1]} | {p['refs'][0]} / {p['refs'][1]} | {p['lighter']} | " + + ", ".join(f"`{t[4:]}`" for t in p["lighter_targets"][:3]) + " |") + return "\n".join(out) + "\n" + + +def areas_dot(pairs, stats): + out = ["digraph areas {", " rankdir=LR;", " node [shape=box];"] + for a, s in sorted(stats.items()): + out.append(f' "{a}" [label="{a}\\n{s["loc"]} lines"];') + for (a, b), w in sorted(pairs.items()): + style = ', color="red"' if (b, a) in pairs else "" + out.append(f' "{a}" -> "{b}" [label="{w}", penwidth={1 + min(w, 2000) / 400:.1f}{style}];') + return "\n".join(out + ["}"]) + "\n" + + +def analyse(app, components_file, commit): + """(result, area pairs) for lib/ of `app`; `components_file` may be missing.""" + deps, loc = load_graph(app) + area, placed = functional_components(sorted(loc), components_file) + counts, upward = layering(deps) + views = {"directories under lib/": component_view(deps, loc, {m: directory_component(m) for m in loc})} + stats, both, pairs = None, [], collections.Counter() + if area is not None: + views["functional areas"] = component_view(deps, loc, area) + stats, both, pairs = areas(deps, loc, area) + return { + "commit": commit, + "components_file": components_file, + "functional_definition": components_file if area is not None else None, + "modules": len(loc), + "loc": sum(loc.values()), + "references": sum(len(t) for t in deps.values()), + "views": views, + "placed_by_directory": placed, + "layering": {"counts": dict(counts), "upward": upward}, + "cycles": cycles(deps, loc, area), + "areas": stats, + "two_way_pairs": both, + }, pairs + + +def main(argv): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("app", help="root of the app (the directory that holds lib/)") + ap.add_argument("--rev", help="measure this commit instead of the working tree") + ap.add_argument("--components", help=f"file with the functional `components:` block (default /{COMPONENTS_FILE})") + ap.add_argument("--out", help="directory for results.json, report.md and areas.dot") + args = ap.parse_args(argv) + repo = os.path.abspath(args.app) + if not os.path.isdir(os.path.join(repo, "lib")): + print(f"no lib/ in {repo}", file=sys.stderr) + return 2 + components = os.path.abspath(args.components) if args.components else default_components_file(repo) + if read_components(components) is None: + print(f"no `components:` block in {components}: functional view skipped", file=sys.stderr) + with tempfile.TemporaryDirectory() as tmp: + if args.rev: + commit, app = export_rev(repo, args.rev, tmp), tmp + else: + commit = subprocess.run(["git", "-C", repo, "rev-parse", "HEAD"], capture_output=True, text=True).stdout.strip() or "working tree" + app = repo + result, pairs = analyse(app, components, commit) + result["components_file"] = os.path.relpath(components, repo) + if result["functional_definition"]: + result["functional_definition"] = result["components_file"] + md = report_md(result) + if args.out: + os.makedirs(args.out, exist_ok=True) + with open(os.path.join(args.out, "results.json"), "w", encoding="utf-8") as fh: + json.dump(result, fh, indent=1, sort_keys=True) + with open(os.path.join(args.out, "report.md"), "w", encoding="utf-8") as fh: + fh.write(md) + if result["areas"] is not None: + with open(os.path.join(args.out, "areas.dot"), "w", encoding="utf-8") as fh: + fh.write(areas_dot(pairs, result["areas"])) + sys.stdout.write(md) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/hydra-gates/scripts/lib/deptrac.sh b/hydra-gates/scripts/lib/deptrac.sh new file mode 100755 index 00000000..ead630e0 --- /dev/null +++ b/hydra-gates/scripts/lib/deptrac.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Conduction +# SPDX-License-Identifier: EUPL-1.2 +# +# Run deptrac on lib/ of an app with one config: +# +# layers the generic layering rules of a Nextcloud app (deptrac/layers.yaml next to +# this script) +# FILE.yaml any other deptrac config, such as the functional areas that deptrac_areas.py +# writes from the app's components file +# +# Usage: deptrac.sh layers|FILE.yaml [--rev REV] [--out DIR] +# --rev REV analyse lib/ of that commit (exported with git archive) instead of the working tree +# --out DIR write deptrac-.json to DIR: the violations per layer pair, summarised by +# deptrac_summary.py +# +# Prints the violations per layer pair. Exits 0 whether or not deptrac finds violations; the +# numbers are the result, not a pass or fail. +# +# deptrac is not a dependency of hydra-gates. Each app installs it with bamarni/composer-bin-plugin +# in vendor-bin/deptrac/composer.json ({"require-dev": {"deptrac/deptrac": "^4.7"}}), then +# composer --working-dir=vendor-bin/deptrac install. DEPTRAC=/path/to/deptrac overrides the binary. + +set -euo pipefail + +usage="usage: deptrac.sh layers|FILE.yaml [--rev REV] [--out DIR]" +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +app="$(cd "${1:?${usage}}" && pwd)" +config_arg="${2:?${usage}}" +shift 2 +rev="" +out="" +while [ "$#" -gt 0 ]; do + case "$1" in + --rev) rev="$2"; shift 2 ;; + --out) out="$2"; shift 2 ;; + *) echo "unknown argument: $1" >&2; exit 2 ;; + esac +done + +if [ "${config_arg}" = "layers" ]; then + config="${here}/deptrac/layers.yaml" +else + config="$(cd "$(dirname "${config_arg}")" && pwd)/$(basename "${config_arg}")" +fi +name="$(basename "${config}" .yaml)" +deptrac="${DEPTRAC:-${app}/vendor-bin/deptrac/vendor/bin/deptrac}" + +[ -x "${deptrac}" ] || { echo "deptrac is not installed: composer --working-dir=${app}/vendor-bin/deptrac install" >&2; exit 2; } +[ -f "${config}" ] || { echo "no config ${config}" >&2; exit 2; } +[ -d "${app}/lib" ] || { echo "no lib/ in ${app}" >&2; exit 2; } + +work="$(mktemp -d)" +trap 'rm -rf "${work}"' EXIT +if [ -n "${rev}" ]; then + git -C "${app}" archive "${rev}" lib | tar -x -C "${work}" +else + ln -s "${app}/lib" "${work}/lib" +fi +cp "${config}" "${work}/deptrac.yaml" + +"${deptrac}" analyse --config-file="${work}/deptrac.yaml" --cache-file="${work}/.deptrac.cache" --no-progress \ + --formatter=json --output="${work}/report.json" >/dev/null || true +[ -s "${work}/report.json" ] || { echo "deptrac wrote no report" >&2; exit 1; } +summary=() +if [ -n "${out}" ]; then + mkdir -p "${out}" + summary=(--out "${out}/deptrac-${name}.json") +fi +python3 "${here}/deptrac_summary.py" "${work}/report.json" "${summary[@]}" diff --git a/hydra-gates/scripts/lib/deptrac/layers.yaml b/hydra-gates/scripts/lib/deptrac/layers.yaml new file mode 100644 index 00000000..9359911f --- /dev/null +++ b/hydra-gates/scripts/lib/deptrac/layers.yaml @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: 2026 Conduction +# SPDX-License-Identifier: EUPL-1.2 +# +# The layering of a Nextcloud app's lib/ as deptrac rules: an entry point may use services, +# persistence and shared types; a service may use persistence and shared types; persistence may +# use shared types; shared types use nothing in lib/. References inside one layer are always +# allowed. +# +# A file's layer is the first directory below lib/ with a layer name, the same rule as ROLES in +# analyse_dependencies.py: lib/Files/Controller/X.php is an entry point, lib/Controller/Support/X.php +# is an entry point too. The optional first segment of each pattern is a feature directory (Files, +# Search, Notifications, ...), never a layer directory. lib/AppInfo (the bootstrap) and +# directories without a layer name are in no layer. +# +# Generic for every app: run it through deptrac.sh, which points `paths` at the lib/ to analyse. +deptrac: + paths: + - ./lib + exclude_files: + - '#/lib/Migration/#' + layers: + - name: EntryPoint + collectors: + - type: directory + value: 'lib/((?!(Controller|Command|Listener|BackgroundJob|Repair|Middleware|Settings|Sections|Dashboard|Capabilities|Service|Db|Event|Exception|Kernel|Contract|Dto|Support|Formats|AppInfo)/)[^/]+/)?(Controller|Command|Listener|BackgroundJob|Repair|Middleware|Settings|Sections|Dashboard|Capabilities)/' + - name: Service + collectors: + - type: directory + value: 'lib/((?!(Controller|Command|Listener|BackgroundJob|Repair|Middleware|Settings|Sections|Dashboard|Capabilities|Service|Db|Event|Exception|Kernel|Contract|Dto|Support|Formats|AppInfo)/)[^/]+/)?Service/' + - name: Persistence + collectors: + - type: directory + value: 'lib/((?!(Controller|Command|Listener|BackgroundJob|Repair|Middleware|Settings|Sections|Dashboard|Capabilities|Service|Db|Event|Exception|Kernel|Contract|Dto|Support|Formats|AppInfo)/)[^/]+/)?Db/' + - name: SharedType + collectors: + - type: directory + value: 'lib/((?!(Controller|Command|Listener|BackgroundJob|Repair|Middleware|Settings|Sections|Dashboard|Capabilities|Service|Db|Event|Exception|Kernel|Contract|Dto|Support|Formats|AppInfo)/)[^/]+/)?(Event|Exception|Kernel|Contract|Dto|Support|Formats)/' + ruleset: + EntryPoint: + - Service + - Persistence + - SharedType + Service: + - Persistence + - SharedType + Persistence: + - SharedType + SharedType: ~ diff --git a/hydra-gates/scripts/lib/deptrac_areas.py b/hydra-gates/scripts/lib/deptrac_areas.py new file mode 100755 index 00000000..a9f5ef5e --- /dev/null +++ b/hydra-gates/scripts/lib/deptrac_areas.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Conduction +# SPDX-License-Identifier: EUPL-1.2 +"""Write a deptrac config from the functional areas in an app's components file. + + python3 deptrac_areas.py [--components FILE] [--shared NAME] [--out FILE] [--check] + +Every `components:` entry of the components file (default /sigrid.components.yaml) becomes a +deptrac layer with one directory collector per include pattern. An include pattern such as +`.*/lib/Db/(A|B)[.]php` is matched by the scorer against the full path; deptrac matches its +collector anywhere in the path, so the leading `.*/` is dropped and a `$` is added. Exclude +patterns are not carried over. The ruleset lets every area use the --shared area (default Kernel) +and nothing else, so each reference from one area into another is reported as a violation. When +no area carries the shared name, no area may use any other. + +Without --out the config goes to stdout. With --out it is written to FILE; --check then exits 1 +when FILE differs from what would be written. Run the config with deptrac.sh. +""" +import argparse +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +import analyse_dependencies as analysis # noqa: E402 + +SHARED = "Kernel" + +HEADER = """\ +# GENERATED by hydra-gates deptrac_areas.py from {source}. Do not edit by hand. +# +# The functional areas of lib/ as deptrac layers. {rule} +# Run through hydra-gates deptrac.sh. +""" + + +def quote(value): + return "'" + value.replace("'", "''") + "'" + + +def render(definition, source, shared=SHARED): + """The deptrac config for a `components:` definition; `source` names it in the header.""" + comps = definition["components"] + names = list(dict.fromkeys(c["name"] for c in comps)) + has_shared = shared in names + rule = (f"Every area may use {shared}; any other reference from one area into another is a violation." + if has_shared else "No area may use another; every reference between areas is a violation.") + out = [HEADER.format(source=source, rule=rule) + "deptrac:", " paths:", " - ./lib", + " exclude_files:", " - '#/lib/Migration/#'", " layers:"] + collectors = {name: [] for name in names} + for c in comps: + collectors[c["name"]] += [p.pattern for p in c["include"]] + for name in names: + out += [f" - name: {name}", " collectors:"] + for pattern in collectors[name]: + value = pattern[3:] if pattern.startswith(".*/") else pattern + out += [" - type: directory", f" value: {quote(value + '$')}"] + out.append(" ruleset:") + for name in names: + out.append(f" {name}: ~" if name == shared or not has_shared else f" {name}:\n - {shared}") + return "\n".join(out) + "\n" + + +def main(argv): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("app", help="root of the app") + ap.add_argument("--components", help=f"file with the `components:` block (default /{analysis.COMPONENTS_FILE})") + ap.add_argument("--shared", default=SHARED, help=f"the area every other area may use (default {SHARED})") + ap.add_argument("--out", help="write the config here instead of to stdout") + ap.add_argument("--check", action="store_true", help="with --out: exit 1 when the file is stale") + args = ap.parse_args(argv) + app = os.path.abspath(args.app) + source = os.path.abspath(args.components) if args.components else analysis.default_components_file(app) + definition = analysis.read_components(source) + if definition is None: + print(f"no `components:` block in {source}: there are no areas to write", file=sys.stderr) + return 2 + text = render(definition, os.path.relpath(source, app), args.shared) + if not args.out: + if args.check: + print("--check needs --out FILE", file=sys.stderr) + return 2 + sys.stdout.write(text) + return 0 + if args.check: + current = "" + if os.path.isfile(args.out): + with open(args.out, encoding="utf-8") as fh: + current = fh.read() + if current != text: + print(f"{args.out} is stale: rerun deptrac_areas.py without --check") + return 1 + return 0 + with open(args.out, "w", encoding="utf-8") as fh: + fh.write(text) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/hydra-gates/scripts/lib/deptrac_summary.py b/hydra-gates/scripts/lib/deptrac_summary.py new file mode 100755 index 00000000..7f3870b0 --- /dev/null +++ b/hydra-gates/scripts/lib/deptrac_summary.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Conduction +# SPDX-License-Identifier: EUPL-1.2 +"""Summarise a deptrac JSON report per pair of layers. + + python3 deptrac_summary.py REPORT.json [--out SUMMARY.json] + +deptrac reports one violation per occurrence: every line on which a class names a class of a +layer it may not use. This summary counts those occurrences and the distinct class pairs per +"from -> to" layer pair, and writes the file paths relative to lib/ so a report made in a +temporary directory compares with one made in the working tree. The table goes to stdout. +""" +import collections +import json +import re +import sys + +MESSAGE = re.compile(r"^(\S+) must not depend on (\S+) \((\w+) on (\w+)\)$") + + +def summarise(report): + occurrences = collections.Counter() + pairs = collections.defaultdict(set) + for _path, entry in report.get("files", {}).items(): + for message in entry.get("messages", []): + m = MESSAGE.match(message.get("message", "")) + if m: + # deptrac says "X must not depend on Y (A on B)": layer A depends on layer B. + key = f"{m.group(3)} -> {m.group(4)}" + occurrences[key] += 1 + pairs[key].add((m.group(1), m.group(2))) + totals = {k: v for k, v in report.get("Report", {}).items()} + rows = [{"layers": k, "occurrences": occurrences[k], "class_pairs": len(pairs[k]), + "examples": [f"{a} -> {b}" for a, b in sorted(pairs[k])[:3]]} + for k in sorted(occurrences, key=lambda k: -occurrences[k])] + return {"report": totals, "class_pairs": sum(len(p) for p in pairs.values()), "by_layers": rows} + + +def table(summary): + out = ["| From -> to | Occurrences | Class pairs |", "|---|---|---|"] + out += [f"| {r['layers']} | {r['occurrences']} | {r['class_pairs']} |" for r in summary["by_layers"]] + rep = summary["report"] + out += ["", f"Violations {rep.get('Violations', 0)}, allowed {rep.get('Allowed', 0)}, " + f"uncovered {rep.get('Uncovered', 0)}, distinct class pairs in violation {summary['class_pairs']}."] + return "\n".join(out) + "\n" + + +def main(argv): + if not argv: + print(__doc__) + return 2 + summary = summarise(json.load(open(argv[0], encoding="utf-8"))) + if "--out" in argv: + with open(argv[argv.index("--out") + 1], "w", encoding="utf-8") as fh: + json.dump(summary, fh, indent=1, sort_keys=True) + sys.stdout.write(table(summary)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/hydra-gates/scripts/lib/rate_component_views.py b/hydra-gates/scripts/lib/rate_component_views.py new file mode 100755 index 00000000..e8ef1d0f --- /dev/null +++ b/hydra-gates/scripts/lib/rate_component_views.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Conduction +# SPDX-License-Identifier: EUPL-1.2 +"""Estimated modularity ratings of an app's PHP backend under five component definitions. + + python3 check_sig_maintainability.py report --json /tmp/base.json + python3 rate_component_views.py --report /tmp/base.json [--components FILE] + [--foundation NAME] [--merge NAME[,NAME]] [--json OUT] + +The report gives every property that does not depend on the component definition (unit size, +duplication, the JavaScript structure and so on). For the PHP backend this script recomputes the +structure with the gate-122 scorer's own structure() for each definition, puts it in a copy of the +report and runs the scorer's rating_estimate() on it: + +- A the backend as one component: every file under lib/ in one component; +- B every directory under lib/ (the scorer's directory rule, without a `components:` block); +- C the functional areas of the components file (default /sigrid.components.yaml); +- D as C, with the --merge components (default Core) renamed into the --foundation component + (default Kernel): a shared foundation the other areas build on; +- E as D, without the references from the foundation to the other areas. This is a simulation of + a foundation that is only depended on, not a measurement of existing code. + +An app without a components file gets A and B only; C, D and E are skipped with a message. D and +E are also skipped when no component carries the foundation's name after the merge. + +For C the script also splits the exposed code by kind of file (exposed_by_kind) and by component +(exposed_by_component), and for D it lists the foundation's references to each other component +(foundation_backrefs). `targets` gives, per rating, the largest exposed share and violation degree +that still earn it, from the scorer's own rating scale. + +All ratings are estimates on the scorer's 0.5 to 5.5 scale; only the 4-star limits are published. +""" +import argparse +import collections +import copy +import json +import os +import re +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +import analyse_dependencies as analysis # noqa: E402 +import check_sig_maintainability as tool # noqa: E402 + +FOUNDATION = "Kernel" +MERGE = ("Core",) + + +def one_component(): + """A `components:` definition that puts every file under lib/ in one component.""" + return {"components": [{"name": "Backend", "include": [re.compile(r".*/lib/.*")], "exclude": []}]} + + +def merge_into_foundation(definition, foundation=FOUNDATION, merge=MERGE): + """A copy of `definition` with the `merge` components renamed to `foundation`. The first + matching component owns a file, so a shared name merges them without reordering.""" + out = {"components": [dict(c) for c in definition["components"]]} + for c in out["components"]: + if c["name"] in merge: + c["name"] = foundation + return out + + +def has_component(definition, name): + return any(c["name"] == name for c in definition["components"]) + + +def without_foundation_backrefs(deps, comp, foundation=FOUNDATION): + """deps without the edges from a foundation file to a file of another component.""" + return {src: {d for d in targets if not (comp(src) == foundation and comp(d) != foundation)} + for src, targets in deps.items()} + + +def kind_of(rel, text): + """The kind of a PHP file, by its declaration and name: interface, trait, exception, event, + entity, mapper or concrete class. An entity that does not extend Entity directly counts as a + concrete class.""" + name = os.path.basename(rel) + if re.search(r"^\s*interface\s", text, re.M): + return "interface" + if re.search(r"^\s*trait\s", text, re.M): + return "trait" + if name.endswith("Exception.php") or re.search(r"extends \\?\w*Exception\b", text): + return "exception" + if name.endswith("Event.php"): + return "event" + if "/Db/" in rel and re.search(r"extends \\?(OCP\\AppFramework\\Db\\)?Entity\b", text): + return "entity" + if name.endswith("Mapper.php"): + return "mapper" + return "concrete class" + + +def exposed_by_kind(app, deps, comp): + """{kind: {"files": n, "loc": lines}} of the files another component references.""" + incoming = collections.defaultdict(set) + for src, targets in deps.items(): + for t in targets: + incoming[t].add(src) + out = collections.defaultdict(lambda: {"files": 0, "loc": 0}) + for rel in deps: + if any(comp(s) != comp(rel) for s in incoming[rel]): + text = tool.read_text(os.path.join(app, rel)) + row = out[kind_of(rel, text)] + row["files"] += 1 + row["loc"] += len(tool.code_lines(text, "php")) + return dict(sorted(out.items(), key=lambda kv: -kv[1]["loc"])) + + +def exposed_by_component(app, deps, comp): + """{component: {"loc": lines, "exposed_loc": lines another component references}}.""" + incoming = collections.defaultdict(set) + for src, targets in deps.items(): + for t in targets: + incoming[t].add(src) + out = collections.defaultdict(lambda: {"loc": 0, "exposed_loc": 0}) + for rel in deps: + lines = len(tool.code_lines(tool.read_text(os.path.join(app, rel)), "php")) + row = out[comp(rel)] + row["loc"] += lines + if any(comp(s) != comp(rel) for s in incoming[rel]): + row["exposed_loc"] += lines + return dict(sorted(out.items(), key=lambda kv: -kv[1]["loc"])) + + +def max_value_for(kind, rating): + """The largest value of an exposed share or violation degree that still rates `rating`.""" + caps = tool.rating_caps(kind) + lo, hi = 0.0, 1.0 + for _ in range(50): + mid = (lo + hi) / 2 + if tool.interpolate(mid, caps) >= rating: + lo = mid + else: + hi = mid + return round(lo, 4) + + +def targets(ratings=(4.0, 4.5, 5.0, 5.5)): + """{rating: {"exposed_share": max, "violation_degree": max}}.""" + return {str(r): {"exposed_share": max_value_for("component_independence", r), + "violation_degree": max_value_for("component_entanglement", r)} for r in ratings} + + +def foundation_backrefs(deps, comp, foundation=FOUNDATION): + """{component: file references from the foundation to it}, heaviest first.""" + out = collections.Counter(comp(d) for src, targets in deps.items() if comp(src) == foundation + for d in targets if comp(d) != foundation) + return dict(out.most_common()) + + +def rate(report, structure): + rep = copy.deepcopy(report) + rep.setdefault("structure", {})["php"] = structure + rating = tool.rating_estimate(rep) + ci, ce = structure["component_independence"], structure["component_entanglement"] + out = { + "components": len(structure["components"]), + "hidden_share": ci["hidden_share"], + "violation_degree": ce["violation_degree"], + "violations": ce["violations"], + "independence": rating["properties"]["component_independence"]["rating"], + "entanglement": rating["properties"]["component_entanglement"]["rating"], + "modularity": rating["subcharacteristics"]["modularity"], + "overall": rating["overall"]["range"], + } + if structure.get("warning"): + out["warning"] = structure["warning"] + return out + + +def structure_under(app, files, deps, definition): + """(the scorer's structure(), its component function) with `definition` as the components; + None means the scorer's directory rule.""" + with analysis.components_of(definition): + st = tool.structure(app, files, deps) + if definition is None: + return st, tool.component_of_factory(sorted(deps))[1] + return st, tool.scope_component_factory(definition)[1] + + +def views(report, app, components_file=None, foundation=FOUNDATION, merge=MERGE): + """{view: rating} for A and B, and for C, D and E when the app has a components file. + Skipped views are listed under "skipped" with the reason.""" + files = {rel: lang for rel, lang in tool.production_files(app, tool.Scope(app)).items() if lang == "php"} + deps = tool.php_dependencies(app, files) + if not deps: + raise SystemExit(f"no PHP classes in {app}") + out, skipped = {}, {} + for key, definition in (("A", one_component()), ("B", None)): + out[key] = rate(report, structure_under(app, files, deps, definition)[0]) + path = components_file or analysis.default_components_file(app) + functional = analysis.read_components(path) + if functional is None: + reason = f"no `components:` block in {path}" + skipped.update(C=reason, D=reason, E=reason) + else: + st, comp = structure_under(app, files, deps, functional) + out["C"] = rate(report, st) + out["C"]["exposed_by_kind"] = exposed_by_kind(app, deps, comp) + out["C"]["exposed_by_component"] = exposed_by_component(app, deps, comp) + merged = merge_into_foundation(functional, foundation, merge) + if not has_component(merged, foundation): + reason = f"no component named {foundation} after merging {', '.join(merge) or 'nothing'}" + skipped.update(D=reason, E=reason) + else: + st, comp = structure_under(app, files, deps, merged) + out["D"] = rate(report, st) + out["D"]["foundation_backrefs"] = foundation_backrefs(deps, comp, foundation) + pruned = without_foundation_backrefs(deps, comp, foundation) + out["E"] = rate(report, structure_under(app, files, pruned, merged)[0]) + if skipped: + out["skipped"] = skipped + return out + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("app", help="root of the app the report was made of") + ap.add_argument("--report", required=True, help="JSON written by check_sig_maintainability.py report") + ap.add_argument("--components", help=f"file with the `components:` block (default /{analysis.COMPONENTS_FILE})") + ap.add_argument("--foundation", default=FOUNDATION, help=f"the shared foundation component of D and E (default {FOUNDATION})") + ap.add_argument("--merge", default=",".join(MERGE), + help=f"comma-separated components merged into the foundation for D and E (default {','.join(MERGE)})") + ap.add_argument("--json", help="write the results here") + args = ap.parse_args(argv) + with open(args.report, encoding="utf-8") as fh: + report = json.load(fh) + merge = tuple(n.strip() for n in args.merge.split(",") if n.strip()) + out = views(report, os.path.abspath(args.app), args.components, args.foundation, merge) + out["targets"] = targets() + for key, v in out.items(): + if key in ("targets", "skipped"): + continue + rng = v["overall"] + overall = f"{rng[0]} to {rng[1]}" if rng else "not rated" + line = (f"{key}: {v['components']} components, hidden {v['hidden_share']:.1%}, " + f"violation degree {v['violation_degree']:.4f}; independence {v['independence']}, " + f"entanglement {v['entanglement']}, modularity {v['modularity']}, overall {overall}") + print(line + (f" ({v['warning']})" if v.get("warning") else "")) + for key, reason in out.get("skipped", {}).items(): + print(f"{key}: skipped, {reason}") + if args.json: + with open(args.json, "w", encoding="utf-8") as fh: + json.dump(out, fh, indent=2) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/hydra-gates/scripts/lib/test_analyse_dependencies.py b/hydra-gates/scripts/lib/test_analyse_dependencies.py new file mode 100644 index 00000000..c77911a1 --- /dev/null +++ b/hydra-gates/scripts/lib/test_analyse_dependencies.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Conduction +# SPDX-License-Identifier: EUPL-1.2 +"""Tests for analyse_dependencies.py. + + python3 scripts/lib/test_analyse_dependencies.py + +The analysis runs on a small lib/ written into a temporary directory, so every count below can be +checked by hand: a controller calls a service, the service calls a mapper, the mapper calls back +into the service (one upward reference and one cycle of two files), and a listener in another +area calls the same service. +""" +import contextlib +import io +import os +import sys +import tempfile +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +import analyse_dependencies as analysis # noqa: E402 + +NS = "OCA\\OpenRegister" +FILES = { + "lib/Controller/FlowController.php": f"namespace {NS}\\Controller;\nuse {NS}\\Service\\FlowService;\nclass FlowController {{ public function __construct(FlowService $s) {{}} }}\n", + "lib/Service/FlowService.php": f"namespace {NS}\\Service;\nuse {NS}\\Db\\FlowMapper;\nclass FlowService {{ public function __construct(FlowMapper $m) {{}} }}\n", + "lib/Db/FlowMapper.php": f"namespace {NS}\\Db;\nuse {NS}\\Service\\FlowService;\nclass FlowMapper {{ public function x(FlowService $s) {{}} }}\n", + "lib/Listener/MailListener.php": f"namespace {NS}\\Listener;\nuse {NS}\\Service\\FlowService;\nclass MailListener {{ public function __construct(FlowService $s) {{}} }}\n", + "lib/Migration/Version1.php": f"namespace {NS}\\Migration;\nuse {NS}\\Db\\FlowMapper;\nclass Version1 {{ public function x(FlowMapper $m) {{}} }}\n", +} +COMPONENTS = """\ +components: + - name: Flows + include: + - ".*/lib/(Controller|Service|Db)/Flow[^/]*[.]php" + - name: Mail + include: + - ".*/lib/Listener/Mail[^/]*[.]php" +""" + + +class AnalysisTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.app = self.tmp.name + for rel, src in FILES.items(): + os.makedirs(os.path.dirname(os.path.join(self.app, rel)), exist_ok=True) + with open(os.path.join(self.app, rel), "w", encoding="utf-8") as fh: + fh.write(" service": ["lib/Db/FlowMapper.php -> lib/Service/FlowService.php"]}) + + def test_the_cycle_is_the_service_and_its_mapper(self): + cyc = self.result["cycles"] + self.assertEqual(cyc["cyclic_sets"], 1) + self.assertEqual(cyc["largest"]["members"], ["lib/Db/FlowMapper.php", "lib/Service/FlowService.php"]) + self.assertEqual(cyc["largest"]["upward_edges_inside"], ["lib/Db/FlowMapper.php -> lib/Service/FlowService.php"]) + + def test_directory_view_counts_layer_calls_between_components(self): + view = self.result["views"]["directories under lib/"] + self.assertEqual(view["components"], 4) + # Controller -> Service, Service -> Db, Db -> Service, Listener -> Service. Service and Db + # reference each other with equal weight, so both lines of that pair count as cyclic. + self.assertEqual(view["communication_lines"], 4) + self.assertEqual(view["violations"], {"cyclic": 2}) + + def test_functional_view_keeps_controller_service_and_mapper_together(self): + view = self.result["views"]["functional areas"] + self.assertEqual(view["components"], 2) + self.assertEqual(view["communication_lines"], 1) + self.assertEqual(view["violation_degree"], 0.0) + self.assertEqual(dict(self.pairs), {("Mail", "Flows"): 1}) + self.assertEqual(self.result["two_way_pairs"], []) + + def test_unmatched_file_is_placed_by_directory_majority(self): + area, placed = analysis.functional_components( + ["lib/Service/FlowService.php", "lib/Service/FlowOther.php", "lib/Service/Unknown.php"], self.components) + self.assertEqual(placed, ["lib/Service/Unknown.php"]) + self.assertEqual(area["lib/Service/Unknown.php"], "Flows") + + def test_strongly_connected_finds_a_three_cycle(self): + deps = {"a": {"b"}, "b": {"c"}, "c": {"a"}, "d": {"a"}} + groups = analysis.strongly_connected(deps) + self.assertEqual(groups[0], {"a", "b", "c"}) + self.assertEqual(len(groups), 2) + + + def test_without_a_components_file_only_the_directory_view_runs(self): + result, pairs = analysis.analyse(self.app, os.path.join(self.app, "missing.yaml"), "test") + self.assertEqual(list(result["views"]), ["directories under lib/"]) + self.assertIsNone(result["areas"]) + self.assertIsNone(result["functional_definition"]) + self.assertIsNone(result["cycles"]["largest"]["by_area"]) + self.assertEqual(dict(pairs), {}) + md = analysis.report_md(result) + self.assertIn("functional view and the per-area tables are skipped", md) + self.assertNotIn("## Functional areas", md) + + def test_the_default_components_file_sits_in_the_app_root(self): + self.assertEqual(analysis.default_components_file("/x/app"), "/x/app/sigrid.components.yaml") + + def test_components_of_restores_the_scorer(self): + before = analysis.tool.scope_file_components + with analysis.components_of(analysis.read_components(self.components)): + self.assertTrue(analysis.tool.COMPONENTS_FROM_SCOPE) + self.assertIs(analysis.tool.scope_file_components, before) + self.assertFalse(analysis.tool.COMPONENTS_FROM_SCOPE) + + def test_main_reads_the_app_argument_and_writes_the_outputs(self): + os.rename(self.components, os.path.join(self.app, "sigrid.components.yaml")) + out = os.path.join(self.app, "out") + with contextlib.redirect_stdout(io.StringIO()) as stdout: + self.assertEqual(analysis.main([self.app, "--out", out]), 0) + self.assertIn("| functional areas | 2 |", stdout.getvalue()) + self.assertEqual(sorted(os.listdir(out)), ["areas.dot", "report.md", "results.json"]) + + def test_main_refuses_a_directory_without_lib(self): + with contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(analysis.main([os.path.join(self.app, "lib", "Db")]), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/hydra-gates/scripts/lib/test_deptrac_tools.py b/hydra-gates/scripts/lib/test_deptrac_tools.py new file mode 100644 index 00000000..a53366dd --- /dev/null +++ b/hydra-gates/scripts/lib/test_deptrac_tools.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Conduction +# SPDX-License-Identifier: EUPL-1.2 +"""Tests for deptrac_areas.py, deptrac_summary.py and deptrac/layers.yaml. + + python3 scripts/lib/test_deptrac_tools.py + +AreasTest pins how a components file becomes a deptrac config: one layer per area, include +patterns without the leading wildcard and anchored at the end, and a ruleset in which every area +may use only the shared area; --check reports a stale file. SummaryTest pins how a deptrac JSON +report is counted: occurrences per layer pair, and distinct class pairs. LayersTest keeps the +generic layering config in step with the roles of analyse_dependencies.py. +""" +import contextlib +import io +import os +import re +import sys +import tempfile +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +import analyse_dependencies # noqa: E402 +import deptrac_areas # noqa: E402 +import deptrac_summary # noqa: E402 + +COMPONENTS = """components: + - name: Kernel + include: + - ".*/lib/Kernel/.*" + - name: Processes + include: + - ".*/lib/Service/Flow/.*" + - ".*/lib/Controller/Flow[^/]*[.]php" + - name: Search + include: + - ".*/lib/Search/.*" +""" + + +class AreasTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.app = self.tmp.name + with open(os.path.join(self.app, "sigrid.components.yaml"), "w", encoding="utf-8") as fh: + fh.write(COMPONENTS) + self.definition = analyse_dependencies.read_components(os.path.join(self.app, "sigrid.components.yaml")) + + def tearDown(self): + self.tmp.cleanup() + + def test_every_area_may_use_only_the_shared_area(self): + text = deptrac_areas.render(self.definition, "sigrid.components.yaml") + ruleset = text.split(" ruleset:\n", 1)[1] + self.assertIn(" Kernel: ~\n", ruleset) + self.assertEqual(ruleset.count(" - Kernel\n"), 2) + + def test_patterns_drop_the_leading_wildcard_and_anchor_the_end(self): + text = deptrac_areas.render(self.definition, "sigrid.components.yaml") + self.assertIn("value: 'lib/Kernel/.*$'", text) + self.assertIn("value: 'lib/Controller/Flow[^/]*[.]php$'", text) + + def test_the_shared_area_is_a_parameter(self): + text = deptrac_areas.render(self.definition, "x", shared="Search") + ruleset = text.split(" ruleset:\n", 1)[1] + self.assertIn(" Search: ~\n", ruleset) + self.assertEqual(ruleset.count(" - Search\n"), 2) + + def test_without_the_shared_area_no_area_may_use_another(self): + text = deptrac_areas.render(self.definition, "x", shared="Base") + ruleset = text.split(" ruleset:\n", 1)[1] + self.assertEqual(ruleset, " Kernel: ~\n Processes: ~\n Search: ~\n") + + def test_check_reports_a_stale_file(self): + out = os.path.join(self.app, "areas.yaml") + self.assertEqual(deptrac_areas.main([self.app, "--out", out]), 0) + self.assertEqual(deptrac_areas.main([self.app, "--out", out, "--check"]), 0) + with open(out, "a", encoding="utf-8") as fh: + fh.write("# edited\n") + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(deptrac_areas.main([self.app, "--out", out, "--check"]), 1) + + def test_an_app_without_components_is_refused(self): + os.remove(os.path.join(self.app, "sigrid.components.yaml")) + with contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(deptrac_areas.main([self.app]), 2) + + +class SummaryTest(unittest.TestCase): + REPORT = { + "Report": {"Violations": 3, "Allowed": 10}, + "files": { + "/tmp/x/lib/Db/A.php": {"messages": [ + {"message": "OCA\\X\\Db\\A must not depend on OCA\\X\\Service\\S (Persistence on Service)", "line": 3}, + {"message": "OCA\\X\\Db\\A must not depend on OCA\\X\\Service\\S (Persistence on Service)", "line": 9}, + ]}, + "/tmp/x/lib/Event/E.php": {"messages": [ + {"message": "OCA\\X\\Event\\E must not depend on OCA\\X\\Db\\A (SharedType on Persistence)", "line": 4}, + ]}, + }, + } + + def test_counts_occurrences_and_class_pairs(self): + summary = deptrac_summary.summarise(self.REPORT) + rows = {r["layers"]: (r["occurrences"], r["class_pairs"]) for r in summary["by_layers"]} + self.assertEqual(rows, {"Persistence -> Service": (2, 1), "SharedType -> Persistence": (1, 1)}) + self.assertEqual(summary["class_pairs"], 2) + self.assertNotIn("/tmp/", str(summary)) + + +class LayersTest(unittest.TestCase): + def test_layer_directories_are_the_roles_of_the_analysis(self): + with open(os.path.join(HERE, "deptrac", "layers.yaml"), encoding="utf-8") as fh: + text = fh.read() + values = re.findall(r"value: '([^']+)'", text) + self.assertEqual(len(values), 4) + by_layer = {} + for value in values: + own = re.search(r"\)\?\(?([A-Za-z|]+)\)?/$", value).group(1).split("|") + excluded = re.search(r"\?!\(([A-Za-z|]+)\)", value).group(1).split("|") + self.assertEqual(sorted(set(excluded) - {"AppInfo"}), sorted(analyse_dependencies.ROLES)) + for name in own: + by_layer[name] = analyse_dependencies.ROLES[name][1] + self.assertEqual(sorted(by_layer), sorted(analyse_dependencies.ROLES)) + + +if __name__ == "__main__": + unittest.main() diff --git a/hydra-gates/scripts/lib/test_rate_component_views.py b/hydra-gates/scripts/lib/test_rate_component_views.py new file mode 100644 index 00000000..5ee2236e --- /dev/null +++ b/hydra-gates/scripts/lib/test_rate_component_views.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Conduction +# SPDX-License-Identifier: EUPL-1.2 +"""Tests for rate_component_views.py. + + python3 scripts/lib/test_rate_component_views.py + +They pin the two transformations the script applies before it rates (merging components into the +foundation, and dropping the foundation's references to other components), and run the five views +on a small app: with a components file all five are rated, without one only A and B. +""" +import contextlib +import io +import json +import os +import sys +import tempfile +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +import analyse_dependencies as analysis # noqa: E402 +import rate_component_views as views # noqa: E402 + +COMPONENTS = """components: + - name: Kernel + include: + - ".*/lib/Contract/[^/]*" + - name: Core + include: + - ".*/lib/Db/MagicMapper[.]php" + - name: Processes + include: + - ".*/lib/Service/Flow/[^/]*" +""" + + +def parsed(text): + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "components.yaml") + with open(path, "w", encoding="utf-8") as fh: + fh.write(text) + return analysis.read_components(path) + + +class FoundationMergeTest(unittest.TestCase): + def test_core_is_renamed_to_the_foundation(self): + out = views.merge_into_foundation(parsed(COMPONENTS)) + self.assertEqual([c["name"] for c in out["components"]], ["Kernel", "Kernel", "Processes"]) + + def test_patterns_and_the_original_are_kept(self): + definition = parsed(COMPONENTS) + out = views.merge_into_foundation(definition) + self.assertEqual(out["components"][1]["include"][0].pattern, ".*/lib/Db/MagicMapper[.]php") + self.assertEqual(definition["components"][1]["name"], "Core") + + def test_foundation_and_merge_are_parameters(self): + out = views.merge_into_foundation(parsed(COMPONENTS), "Base", ("Kernel", "Core")) + self.assertEqual([c["name"] for c in out["components"]], ["Base", "Base", "Processes"]) + self.assertTrue(views.has_component(out, "Base")) + self.assertFalse(views.has_component(out, "Kernel")) + + +class BackrefsTest(unittest.TestCase): + def test_only_foundation_to_other_edges_are_dropped(self): + comp = {"lib/Db/MagicMapper.php": "Kernel", "lib/Contract/X.php": "Kernel", + "lib/Service/Flow/FlowService.php": "Processes"}.get + deps = { + "lib/Db/MagicMapper.php": {"lib/Contract/X.php", "lib/Service/Flow/FlowService.php"}, + "lib/Service/Flow/FlowService.php": {"lib/Db/MagicMapper.php"}, + "lib/Contract/X.php": set(), + } + out = views.without_foundation_backrefs(deps, comp) + self.assertEqual(out["lib/Db/MagicMapper.php"], {"lib/Contract/X.php"}) + self.assertEqual(out["lib/Service/Flow/FlowService.php"], {"lib/Db/MagicMapper.php"}) + + def test_foundation_backrefs_are_counted_per_component(self): + comp = {"a": "Kernel", "b": "Kernel", "c": "Processes", "d": "Compliance"}.get + deps = {"a": {"b", "c", "d"}, "b": {"c"}, "c": {"a"}, "d": set()} + self.assertEqual(views.foundation_backrefs(deps, comp), {"Processes": 2, "Compliance": 1}) + + def test_a_foundation_name_other_than_kernel_is_honoured(self): + comp = {"a": "Base", "b": "Processes"}.get + deps = {"a": {"b"}, "b": {"a"}} + self.assertEqual(views.without_foundation_backrefs(deps, comp, "Base"), {"a": set(), "b": {"a"}}) + self.assertEqual(views.foundation_backrefs(deps, comp, "Base"), {"Processes": 1}) + + def test_one_component_covers_lib(self): + [comp] = views.one_component()["components"] + self.assertTrue(comp["include"][0].fullmatch("/lib/Db/X.php")) + + +class KindTest(unittest.TestCase): + def test_declarations_decide_the_kind(self): + cases = { + ("lib/Contract/X.php", "