From 8be79b1bb72c145757b79a597bc1c7dc66174612 Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Thu, 5 Nov 2026 09:17:10 +0100 Subject: [PATCH 1/2] feat(hydra-gates): SIG v18.1 star bands for gates 122 and 123, and gates.json per run Gate 122 now rates against the 4-star limits of the SIG/TUV NORD CERT Guidance for Producers v18.1 (2026 model): unit size 48.0/24.2/9.1%, complexity 20.7/7.5/1.1%, interfacing 15.4/3.4/0.9%, coupling 9.7/5.3/1.8%, duplication 5.8%, entanglement 0.075, independence 94.5%. Volume is not rated. Every report row carries the star level its share earns: 5, 3 and 2 stars from the v18.1 limits scaled by the band ratios of Alves, Correia and Visser 2011 (duplication: Heitlager 2007), 1 star below 2. Independence and entanglement have 5 stars at half the 4-star exposure and no band under 4. The five-star profile uses the derived 5-star band instead of half of each 4-star limit. Gate 123 records the guidance in the floor file and never lowers a floor on re-recording: a floored check the new limits miss stays in the file and is named, so the change of status is visible. The runner writes gates.json next to the findings logs: every declared gate with its verdict, findings count, scope and matrix category. Assisted-by: ClaudeCode:claude-opus-5-5 --- hydra-gates/scripts/lib/check_sig_floor.py | 32 +++- .../scripts/lib/check_sig_maintainability.py | 140 +++++++++++------ .../scripts/lib/test_check_sig_floor.py | 30 +++- .../lib/test_check_sig_maintainability.py | 81 +++++++--- .../scripts/lib/test_write_gates_json.py | 132 ++++++++++++++++ hydra-gates/scripts/lib/write_gates_json.py | 142 ++++++++++++++++++ hydra-gates/scripts/run-hydra-gates.sh | 38 ++++- hydra-gates/tests/helper-suite-timings.tsv | 1 + 8 files changed, 514 insertions(+), 82 deletions(-) create mode 100644 hydra-gates/scripts/lib/test_write_gates_json.py create mode 100644 hydra-gates/scripts/lib/write_gates_json.py diff --git a/hydra-gates/scripts/lib/check_sig_floor.py b/hydra-gates/scripts/lib/check_sig_floor.py index f260e60b..c58d1d5e 100644 --- a/hydra-gates/scripts/lib/check_sig_floor.py +++ b/hydra-gates/scripts/lib/check_sig_floor.py @@ -46,8 +46,14 @@ Without a floor file the gate is NOT APPLICABLE (exit 4) before any tool is loaded or any base resolved. -The 5-star checks (half of every 4-star limit, gate 122's five-star profile) -are reported next to the 4-star ones and never block. +The 5-star checks (gate 122's five-star profile, derived from the v18.1 4-star +limits) are reported next to the 4-star ones and never block. + +`write` never lowers a floor. When the floor file already exists, every check +it lists stays in it, also one the current limits no longer meet (the 4-star +limits moved from v17.0 to v18.1, stricter on coupling and independence). Such +a check is printed as floored but missed, so the change of status is visible; +a check stays inherited until development meets it again. When the head meets checks the committed floor does not list, `--proposed` receives the raised floor file, for the workflow to offer as a pull request. @@ -250,6 +256,7 @@ def floor_document(app, labels): "Generated by check_sig_floor.py write; raised by the workflow, never lowered by hand.", "schema": SCHEMA, "profile": "four-star", + "limits": sig.GUIDANCE, "tools": {"lizard": sig.LIZARD_VERSION}, "unit_reader": sig.UNIT_READER_VERSION, "measured_at": head, @@ -327,8 +334,8 @@ def cell(r): return "n/a" if r is None else sig.pct(r["share"]) + ("" if r["ok"] else " ✗") out = [f"## SIG score floor: {app_name}", "", - "Every 4-star check in the floor must stay met. 5-star (half of each 4-star limit) " - "is reported only and never blocks. ✗ marks a miss.", "", + f"Every 4-star check in the floor must stay met ({sig.GUIDANCE}). 5-star (derived from the " + "4-star limits) is reported only and never blocks. ✗ marks a miss.", "", "| check | base | head | 4-star limit | floored | 5-star |", "|---|---|---|---|---|---|"] for label in sorted(set(head4) | set(floor) | set(base4 or {})): h, b, h5 = head4.get(label), (base4 or {}).get(label), head5.get(label) @@ -349,7 +356,7 @@ def cell(r): [f"- {lbl}: {why}" for lbl, why in inherited] five_missed = sorted(lbl for lbl, r in head5.items() if not r["ok"]) if five_missed: - out += ["", f"5-star findings (not blocking): {len(five_missed)} check(s) above half the 4-star limit: " + out += ["", f"5-star findings (not blocking): {len(five_missed)} check(s) miss the 5-star limit: " + ", ".join(five_missed)] if proposed: out += ["", "The head meets checks the committed floor does not list yet: " + ", ".join(proposed) @@ -370,13 +377,22 @@ def cmd_write(args, lizard): print(f"{GATE}: no measured source file in this repository, so there is no floor to write") return 4 rows = check_rows(report, "four-star") - met = [lbl for lbl, r in rows.items() if r["ok"]] + met = {lbl for lbl, r in rows.items() if r["ok"]} path = os.path.join(args.app, args.floor) + # A floored check that left the measured scope (a sigrid.yaml change) leaves the file too. + kept = set(parse_floor(sig.read_text(path), args.floor)) & set(rows) if os.path.isfile(path) else set() + missed = sorted(kept - met) + for lbl in missed: + r = rows[lbl] + cmp = "<" if r["higher_is_better"] else ">" + print(f" floored but missed at the current limits, kept in the floor: {lbl}: " + f"{sig.pct(r['share'])} {cmp} limit {sig.pct(r['limit'])}") os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "w") as fh: - json.dump(floor_document(args.app, met), fh, indent=1) + json.dump(floor_document(args.app, met | kept), fh, indent=1) fh.write("\n") - print(f"{GATE}: wrote {args.floor}: {len(met)} of {len(rows)} 4-star checks met and floored") + print(f"{GATE}: wrote {args.floor}: {len(met)} of {len(rows)} 4-star checks met, " + f"{len(met | kept)} floored ({len(missed)} floored but missed)") return 0 diff --git a/hydra-gates/scripts/lib/check_sig_maintainability.py b/hydra-gates/scripts/lib/check_sig_maintainability.py index e8ab0f5d..e56db257 100755 --- a/hydra-gates/scripts/lib/check_sig_maintainability.py +++ b/hydra-gates/scripts/lib/check_sig_maintainability.py @@ -7,7 +7,7 @@ language from risk profiles: the share of code that sits in units (methods and functions) above a size, complexity or parameter bound, plus duplication and a few architecture properties. The 4-star limits come from the SIG/TUV NORD CERT -"Guidance for Producers" v17.0. hydra's phpmd runs at its own defaults with +"Guidance for Producers" v18.1 (the 2026 model). hydra's phpmd runs at its own defaults with baselines, so a green phpmd is not a green SIG profile. TWO MODES @@ -32,14 +32,22 @@ PROFILES (report mode) ---------------------- -four-star the default: the 4-star limits of the Guidance for Producers. -five-star half of every 4-star limit (SIG does not publish 5-star limits, - so half the share of code allowed in each risk band is the - yardstick, rounded to one decimal where the half has two): - duplication 2.8%, unit size 23.6/11.6/4.2%, complexity - 10.1/3.7/0.55%, parameters 7.5/1.65/0.45%, module coupling - 5.0/2.8/0.95%, component independence 96.9% (half the exposed - share), entanglement 0.0385. +four-star the default: the 4-star limits of the Guidance for Producers + v18.1: unit size 48.0/24.2/9.1%, complexity 20.7/7.5/1.1%, + parameters 15.4/3.4/0.9%, module coupling 9.7/5.3/1.8%, + duplication 5.8%, component independence 94.5%, entanglement + 0.075. Volume is no longer rated. +five-star SIG publishes no 5-star limits. The 2011 benchmark table of Alves, + Correia and Visser gives the distance between star bands; its + ratios applied to the v18.1 limits give unit size 36.0/17.0/5.5%, + complexity 15.8/4.4/0.5%, parameters 12.5/2.5/0.6%, module + coupling 7.4/3.3/1.2%, duplication 3.5%. Independence 97.3% and + entanglement 0.0375 halve the 4-star exposure. + +Every row of a report also carries `stars`, the level its share earns on the +same derived scale (STAR_BANDS): 5 to 2 stars, and 1 star for anything worse +than 2. Component independence and entanglement have no band under 4 stars +and read "<4" there. Chosen with --profile, or HYDRA_GATE_SIG_PROFILE when the flag is absent. The profile changes only the limits a report is compared with; the ratchet's unit bounds and the gate's advisory status do not change. @@ -128,22 +136,61 @@ RATCHET_LABEL = {"nloc": "NLOC > 60", "ccn": "McCabe > 10", "params": "5 or more parameters"} DUP_MIN_LINES = 6 -# 4-star limits, per language, as (bound, max share of unit LOC strictly above bound). -LIMITS = { - "unit_size": [(15, .471), (30, .231), (60, .083)], - "unit_complexity": [(5, .202), (10, .073), (25, .011)], - "unit_interfacing": [(2, .150), (4, .033), (6, .009)], # 3+, 5+, 7+ parameters - "module_coupling": [(10, .100), (20, .056), (50, .019)], +# Star bands, per language: for each level, the max share of unit LOC strictly +# above each bound. 4 stars is the SIG/TUV NORD CERT Guidance for Producers +# v18.1 (31 March 2026, the 2026 model). SIG publishes no other level, so 5, 3 +# and 2 stars apply the band-to-4-star ratios of Alves, Correia and Visser 2011 +# (Table III) to the v18.1 limits, and duplication the Heitlager 2007 band +# ratios. Worse than 2 stars is 1 star. Volume is not rated since v18.1. +BOUNDS = { + "unit_size": (15, 30, 60), + "unit_complexity": (5, 10, 25), + "unit_interfacing": (2, 4, 6), # 3+, 5+, 7+ parameters + "module_coupling": (10, 20, 50), } -DUPLICATION_LIMIT = .056 -INDEPENDENCE_MIN = .937 -ENTANGLEMENT_LIMIT = .077 +STAR_BANDS = { + "unit_size": {5: (.360, .170, .055), 4: (.480, .242, .091), 3: (.630, .347, .154), 2: (.847, .490, .253)}, + "unit_complexity": {5: (.158, .044, .005), 4: (.207, .075, .011), 3: (.277, .106, .017), 2: (.346, .132, .027)}, + "unit_interfacing": {5: (.125, .025, .006), 4: (.154, .034, .009), 3: (.183, .048, .014), 2: (.260, .072, .021)}, + "module_coupling": {5: (.074, .033, .012), 4: (.097, .053, .018), 3: (.107, .059, .023), 2: (.130, .080, .038)}, + "duplication": {5: .035, 4: .058, 3: .116, 2: .230}, +} +# Component independence (a minimum) and entanglement have a published 4-star +# limit only. 5 stars halves the exposed share (1 - independence) and the +# entanglement limit; below 4 stars no level is estimated, shown as "<4". +INDEPENDENCE_BANDS = {5: .973, 4: .945} +ENTANGLEMENT_BANDS = {5: .0375, 4: .075} +BELOW_FOUR = "<4" + + +def band_limits(level): + """{property: [(bound, cap), ...]} of one star level.""" + return {prop: list(zip(BOUNDS[prop], STAR_BANDS[prop][level])) for prop in BOUNDS} + + +def stars_of(kind, share, index=None): + """The star level (5 to 1) one measured share earns. Independence and + entanglement have no level under 4, so they return 5, 4 or "<4".""" + if kind == "component_independence": + return next((lvl for lvl in (5, 4) if share >= INDEPENDENCE_BANDS[lvl]), BELOW_FOUR) + if kind == "component_entanglement": + return next((lvl for lvl in (5, 4) if share <= ENTANGLEMENT_BANDS[lvl]), BELOW_FOUR) + bands = STAR_BANDS[kind] + caps = {lvl: bands[lvl] if index is None else bands[lvl][index] for lvl in bands} + return next((lvl for lvl in (5, 4, 3, 2) if share <= caps[lvl]), 1) + + +LIMITS = band_limits(4) +DUPLICATION_LIMIT = STAR_BANDS["duplication"][4] +INDEPENDENCE_MIN = INDEPENDENCE_BANDS[4] +ENTANGLEMENT_LIMIT = ENTANGLEMENT_BANDS[4] +GUIDANCE = "SIG/TUV NORD CERT Guidance for Producers v18.1" PROFILES = { "four-star": { "stars": "4-star", - "source": "SIG/TUV NORD CERT Guidance for Producers v17.0, 4 stars", + "source": f"{GUIDANCE}, 4 stars", "limits": LIMITS, "duplication": DUPLICATION_LIMIT, "independence_min": INDEPENDENCE_MIN, @@ -151,17 +198,13 @@ }, "five-star": { "stars": "5-star", - "source": "half of each 4-star limit of the SIG/TUV NORD CERT Guidance for Producers v17.0 " + "source": f"5-star limits derived from the {GUIDANCE} 4-star limits with the band ratios of " + "Alves, Correia and Visser 2011; independence and entanglement at half the exposed share " "(SIG publishes no 5-star limits)", - "limits": { - "unit_size": [(15, .236), (30, .116), (60, .042)], - "unit_complexity": [(5, .101), (10, .037), (25, .0055)], - "unit_interfacing": [(2, .075), (4, .0165), (6, .0045)], - "module_coupling": [(10, .050), (20, .028), (50, .0095)], - }, - "duplication": .028, - "independence_min": .969, - "entanglement": .0385, + "limits": band_limits(5), + "duplication": STAR_BANDS["duplication"][5], + "independence_min": INDEPENDENCE_BANDS[5], + "entanglement": ENTANGLEMENT_BANDS[5], }, } DEFAULT_PROFILE = "four-star" @@ -518,12 +561,15 @@ def code_lines(text, lang): # report # -------------------------------------------------------------------------- -def profile(units, key, limits): +def profile(units, key, limits, kind): + """One row per bound: the share of unit LOC above it, the profile's limit, + and the star level the share earns.""" total = sum(u["nloc"] for u in units) or 1 rows = [] - for bound, cap in limits: + for i, (bound, cap) in enumerate(limits): share = sum(u["nloc"] for u in units if u[key] > bound) / total - rows.append({"above": bound, "share": round(share, 4), "limit": cap, "ok": share <= cap}) + rows.append({"above": bound, "share": round(share, 4), "limit": cap, "ok": share <= cap, + "stars": stars_of(kind, round(share, 4), i)}) return rows @@ -571,7 +617,7 @@ def duplication_profile(app, clones, files_by_lang, limit=DUPLICATION_LIMIT): def row(dup, total): share = dup / total if total else 0.0 return {"code_lines": total, "redundant_lines": dup, "share": round(share, 4), - "limit": limit, "ok": share <= limit} + "limit": limit, "ok": share <= limit, "stars": stars_of("duplication", round(share, 4))} out = {} for lang in sorted(set(files_by_lang.values())): files = [f for f, lg in files_by_lang.items() if lg == lang] @@ -806,9 +852,10 @@ def structure(app, files, deps, prof=PROFILES[DEFAULT_PROFILE]): incoming[t].add(src) total = sum(loc.values()) or 1 coupling = [] - for bound, cap in prof["limits"]["module_coupling"]: + for i, (bound, cap) in enumerate(prof["limits"]["module_coupling"]): share = sum(loc[m] for m in modules if len(incoming[m]) > bound) / total - coupling.append({"above": bound, "share": round(share, 4), "limit": cap, "ok": share <= cap}) + coupling.append({"above": bound, "share": round(share, 4), "limit": cap, "ok": share <= cap, + "stars": stars_of("module_coupling", round(share, 4), i)}) root, comp = component_of_factory(modules) hidden = sum(loc[m] for m in modules if all(comp(s) == comp(m) for s in incoming[m])) lines = collections.Counter() @@ -832,7 +879,8 @@ def structure(app, files, deps, prof=PROFILES[DEFAULT_PROFILE]): "component_root": root or ".", "components": dict(comp_loc.most_common()), "component_independence": {"hidden_share": round(independence, 4), "limit_min": prof["independence_min"], - "ok": independence >= prof["independence_min"]}, + "ok": independence >= prof["independence_min"], + "stars": stars_of("component_independence", round(independence, 4))}, "component_entanglement": { "communication_lines": len(lines), "connected_components": len(connected), @@ -842,8 +890,11 @@ def structure(app, files, deps, prof=PROFILES[DEFAULT_PROFILE]): "violation_examples": examples[:12], "limit": prof["entanglement"], "ok": None, + "stars": stars_of("component_entanglement", round(degree, 4)), "note": "SIG normalises density and violation degree by benchmark maxima it does not publish, " - "so this is the two inputs, not the rated value. Lower is better on both.", + "so this is the two inputs, not the rated value. Lower is better on both. " + "`stars` rates the violation degree against the entanglement bands as an estimate; " + "it is not a floored check.", }, } @@ -881,9 +932,9 @@ def build_report(app, lizard, jscpd, profile_name=DEFAULT_PROFILE, reader=None): "files": sum(1 for v in files.values() if v == lang), "units": len(rows), "unit_loc": sum(u["nloc"] for u in rows), - "unit_size": profile(rows, "nloc", prof["limits"]["unit_size"]), - "unit_complexity": profile(rows, "ccn", prof["limits"]["unit_complexity"]), - "unit_interfacing": profile(rows, "params", prof["limits"]["unit_interfacing"]), + "unit_size": profile(rows, "nloc", prof["limits"]["unit_size"], "unit_size"), + "unit_complexity": profile(rows, "ccn", prof["limits"]["unit_complexity"], "unit_complexity"), + "unit_interfacing": profile(rows, "params", prof["limits"]["unit_interfacing"], "unit_interfacing"), "duplication": dup[lang][0] if lang in dup else None, "duplication_jscpd": dup[lang][1] if lang in dup and jscpd else None, "largest_units": worst(rows, "nloc"), @@ -925,15 +976,16 @@ def pct(x): def markdown(report): out = [f"## SIG maintainability profile: {report['app']}", "", f"lizard {LIZARD_VERSION}, jscpd {JSCPD_VERSION}; limits: {report['limits']}. " - f"Each cell is the share of code above the bound, against the {report['stars']} limit.", "", + f"Each cell is the share of code above the bound, against the {report['stars']} limit, " + f"with the star level it earns.", "", "| language | units | size >15/30/60 | complexity >5/10/25 | params 3+/5+/7+ | duplication | jscpd |", "|---|---|---|---|---|---|---|"] def cells(rows): - return " / ".join(f"{pct(r['share'])}{'' if r['ok'] else ' ✗'}" for r in rows) + return " / ".join(f"{pct(r['share'])} {r['stars']}★{'' if r['ok'] else ' ✗'}" for r in rows) for lang, d in report["languages"].items(): dup = d.get("duplication") - dcell = f"{pct(dup['share'])}{'' if dup['ok'] else ' ✗'}" if dup else "n/a" + dcell = f"{pct(dup['share'])} {dup['stars']}★{'' if dup['ok'] else ' ✗'}" if dup else "n/a" jdup = d.get("duplication_jscpd") jcell = pct(jdup["share"]) if jdup else "n/a" out.append(f"| {lang} | {d['units']} | {cells(d['unit_size'])} | {cells(d['unit_complexity'])} " @@ -945,7 +997,7 @@ def cells(rows): ci = s["component_independence"] out += ["", f"{lang} structure ({s['modules']} modules, components under `{s['component_root']}`): " f"module coupling >10/20/50 {cells(s['module_coupling'])}; " - f"component independence {pct(ci['hidden_share'])}{'' if ci['ok'] else ' ✗'} (min {pct(ci['limit_min'])}); " + f"component independence {pct(ci['hidden_share'])} {ci['stars']}★{'' if ci['ok'] else ' ✗'} (min {pct(ci['limit_min'])}); " f"entanglement inputs: {e['communication_lines']} lines between {e['connected_components']} components, " f"density {e['communication_density']}, violation degree {e['violation_degree']}."] checks = checks_of(report) diff --git a/hydra-gates/scripts/lib/test_check_sig_floor.py b/hydra-gates/scripts/lib/test_check_sig_floor.py index 976d1da6..8fa33703 100644 --- a/hydra-gates/scripts/lib/test_check_sig_floor.py +++ b/hydra-gates/scripts/lib/test_check_sig_floor.py @@ -133,6 +133,26 @@ def test_the_floor_is_the_met_four_star_checks_under_gate_122_names(self): self.assertEqual(self.repo.floor_file()["checks"], met) self.assertIn("php unit_size >60", met) self.assertEqual(self.repo.floor_file()["profile"], "four-star") + self.assertIn("v18.1", self.repo.floor_file()["limits"]) + + def test_re_recording_never_lowers_the_floor_and_names_the_missed_check(self): + # A check floored under older limits that the current limits miss stays. + wide = [m.replace("(int $p0)", "(int $p0, int $q, int $r)") for m in SMALL[:3]] + self.repo.write("lib/Service/Small.php", php_class("Small", wide + SMALL[3:])) + self.repo.commit("three wide units") + before = self.repo.floor_file()["checks"] + self.assertIn("php unit_interfacing >2", before) + rc, out = run("write", self.repo.dir) + self.assertEqual(rc, 0, out) + self.assertEqual(self.repo.floor_file()["checks"], before) + self.assertIn("floored but missed at the current limits, kept in the floor: php unit_interfacing >2", out) + self.assertIn("(1 floored but missed)", out) + + def test_re_recording_drops_a_check_that_left_the_measured_scope(self): + self.repo.set_floor(self.repo.floor_file()["checks"] + ["python duplication"]) + rc, out = run("write", self.repo.dir) + self.assertEqual(rc, 0, out) + self.assertNotIn("python duplication", self.repo.floor_file()["checks"]) def test_check_rows_agree_with_gate_122_checks(self): self.repo.write("lib/Service/Big.php", php_class("Big", [php_method("huge", lines=70, tag="h")])) @@ -241,10 +261,10 @@ def test_sigrid_yaml_excludes_are_honoured(self): class FiveStar(FloorCase): def test_a_five_star_miss_is_reported_and_never_blocks(self): - # One of twelve equal units takes three parameters: 8.3% of the code, - # above the 5-star 7.5% and below the 4-star 15%. + # One of seven equal units takes three parameters: 14.3% of the code, + # above the 5-star 12.5% and below the 4-star 15.4%. wide = [SMALL[0].replace("(int $p0)", "(int $p0, int $q, int $r)")] - self.repo.write("lib/Service/Small.php", php_class("Small", wide + SMALL[1:])) + self.repo.write("lib/Service/Small.php", php_class("Small", wide + SMALL[1:7])) md = os.path.join(self.repo.dir, "..", os.path.basename(self.repo.dir) + ".md") rc, out = self.check("--md", md) try: @@ -346,7 +366,7 @@ def report(self, size, dup, coupling, hidden): "component_independence": {"hidden_share": hidden}}}} def test_a_share_equal_to_the_limit_meets_it(self): - rows = floor.check_rows(self.report(.471, .056, .100, .937), "four-star") + rows = floor.check_rows(self.report(.480, .058, .097, .945), "four-star") for label in ("php unit_size >15", "php duplication", "php module_coupling >10", "php component_independence"): self.assertTrue(rows[label]["ok"], label) @@ -358,7 +378,7 @@ def test_an_equal_share_is_not_worse_in_either_direction(self): self.assertTrue(floor.worse(row, dict(row, share=.6 if better_high else .4))) def test_just_past_the_limit_misses_it(self): - rows = floor.check_rows(self.report(.4711, .0561, .1001, .9369), "four-star") + rows = floor.check_rows(self.report(.4801, .0581, .0971, .9449), "four-star") for label in ("php unit_size >15", "php duplication", "php module_coupling >10", "php component_independence"): self.assertFalse(rows[label]["ok"], label) diff --git a/hydra-gates/scripts/lib/test_check_sig_maintainability.py b/hydra-gates/scripts/lib/test_check_sig_maintainability.py index bc8b85f5..9537946a 100755 --- a/hydra-gates/scripts/lib/test_check_sig_maintainability.py +++ b/hydra-gates/scripts/lib/test_check_sig_maintainability.py @@ -23,8 +23,11 @@ 4. REPORT: the shares are code-weighted and compared with the 4-star limits. 5. STRUCTURE: PHP and Python dependencies, the component root, and the violation degree on the examples in SIG's Guidance for Producers. - 6. PROFILES: five-star is half of every 4-star limit; a share between the - two passes four-star and misses five-star; HYDRA_GATE_SIG_PROFILE picks + 6. PROFILES: four-star is the v18.1 guidance; five-star is the derived + 5-star band, stricter than four-star on every check; a share between the + two passes four-star and misses five-star; every report row carries the + star level its share earns (5 to 1, "<4" for independence and + entanglement below 4 stars); HYDRA_GATE_SIG_PROFILE picks the profile, --profile overrides it, four-star is the default, and an unknown profile is exit 2. @@ -444,13 +447,13 @@ def test_no_measured_file_is_exit_4(self): class Profiles(unittest.TestCase): - """A fixture whose PHP has 44 of 260 unit lines (16.9%) in a unit over 30 - lines: inside the 4-star limit (23.1%), outside the 5-star one (11.6%).""" + """A fixture whose PHP has 44 of 242 unit lines (18.2%) in a unit over 30 + lines: inside the 4-star limit (24.2%), outside the 5-star one (17.0%).""" def setUp(self): self.repo = Repo() methods = [php_method("long", lines=40, tag="a")] - methods += [php_method(f"s{i}", lines=5, tag=f"b{i}") for i in range(24)] + methods += [php_method(f"s{i}", lines=5, tag=f"b{i}") for i in range(22)] self.repo.write("lib/Service/A.php", php_class("A", methods)) self.repo.commit("base") self.saved_env = os.environ.pop("HYDRA_GATE_SIG_PROFILE", None) @@ -473,35 +476,69 @@ def report(self, *extra): def above_30(self, report): return {r["above"]: r for r in report["languages"]["php"]["unit_size"]}[30] - def test_five_star_is_half_of_every_four_star_limit(self): - four, five = sig.PROFILES["four-star"], sig.PROFILES["five-star"] - for prop, rows in four["limits"].items(): - for (b4, c4), (b5, c5) in zip(rows, five["limits"][prop]): - self.assertEqual(b4, b5, prop) - self.assertAlmostEqual(c5, c4 / 2, delta=.00051, msg=f"{prop} >{b4}") - self.assertAlmostEqual(five["duplication"], four["duplication"] / 2, delta=.00051) - self.assertAlmostEqual(1 - five["independence_min"], (1 - four["independence_min"]) / 2, delta=.00051) - self.assertAlmostEqual(five["entanglement"], four["entanglement"] / 2, delta=.00005) + def test_four_star_is_the_v18_1_guidance(self): + four = sig.PROFILES["four-star"] + self.assertEqual(four["limits"], { + "unit_size": [(15, .480), (30, .242), (60, .091)], + "unit_complexity": [(5, .207), (10, .075), (25, .011)], + "unit_interfacing": [(2, .154), (4, .034), (6, .009)], + "module_coupling": [(10, .097), (20, .053), (50, .018)], + }) + self.assertEqual((four["duplication"], four["independence_min"], four["entanglement"]), (.058, .945, .075)) + self.assertIn("v18.1", four["source"]) + + def test_every_band_is_stricter_than_the_one_below_it(self): + for prop, bands in sig.STAR_BANDS.items(): + for hi, lo in ((5, 4), (4, 3), (3, 2)): + for a, b in zip(*(bands[x] if isinstance(bands[x], tuple) else (bands[x],) for x in (hi, lo))): + self.assertLess(a, b, f"{prop} {hi} vs {lo}") + five = sig.PROFILES["five-star"] + self.assertEqual(five["limits"]["unit_size"], [(15, .360), (30, .170), (60, .055)]) + self.assertEqual((five["duplication"], five["independence_min"], five["entanglement"]), (.035, .973, .0375)) + + def test_stars_of_each_band_edge(self): + for share, want in ((.360, 5), (.3601, 4), (.480, 4), (.4801, 3), (.630, 3), (.847, 2), (.8471, 1)): + self.assertEqual(sig.stars_of("unit_size", share, 0), want, share) + self.assertEqual(sig.stars_of("unit_complexity", .006, 2), 4) + self.assertEqual(sig.stars_of("unit_complexity", .012, 2), 3) + for share, want in ((.035, 5), (.058, 4), (.116, 3), (.230, 2), (.231, 1)): + self.assertEqual(sig.stars_of("duplication", share), want, share) + + def test_independence_and_entanglement_have_no_band_under_four(self): + self.assertEqual(sig.stars_of("component_independence", .973), 5) + self.assertEqual(sig.stars_of("component_independence", .945), 4) + self.assertEqual(sig.stars_of("component_independence", .10), "<4") + self.assertEqual(sig.stars_of("component_entanglement", .0375), 5) + self.assertEqual(sig.stars_of("component_entanglement", .075), 4) + self.assertEqual(sig.stars_of("component_entanglement", .9), "<4") + + def test_report_rows_carry_their_stars(self): + report, out = self.report() + php = report["languages"]["php"] + self.assertEqual(self.above_30(report)["stars"], 4) + self.assertEqual(php["duplication"]["stars"], 5) + self.assertIn(php["unit_size"][0]["stars"], (1, 2, 3, 4, 5)) + self.assertIn("4★", out) def test_default_is_four_star(self): report, out = self.report() self.assertEqual(report["profile"], "four-star") row = self.above_30(report) - self.assertAlmostEqual(row["share"], round(nloc(40) / (nloc(40) + 24 * nloc(5)), 4)) - self.assertEqual(row["limit"], .231) + self.assertAlmostEqual(row["share"], round(nloc(40) / (nloc(40) + 22 * nloc(5)), 4)) + self.assertEqual(row["limit"], .242) self.assertTrue(row["ok"]) - self.assertEqual(report["languages"]["php"]["duplication"]["limit"], .056) + self.assertEqual(report["languages"]["php"]["duplication"]["limit"], .058) self.assertIn("4-star checks met", out) - def test_five_star_flag_holds_the_same_code_to_half_the_limits(self): + def test_five_star_flag_holds_the_same_code_to_the_five_star_band(self): report, out = self.report("--profile", "five-star") self.assertEqual(report["profile"], "five-star") row = self.above_30(report) - self.assertEqual(row["limit"], .116) + self.assertEqual(row["limit"], .170) self.assertFalse(row["ok"]) - self.assertEqual(report["languages"]["php"]["duplication"]["limit"], .028) - self.assertEqual(report["structure"]["php"]["component_independence"]["limit_min"], .969) - self.assertEqual(report["structure"]["php"]["component_entanglement"]["limit"], .0385) + self.assertEqual(report["languages"]["php"]["duplication"]["limit"], .035) + self.assertEqual(report["structure"]["php"]["component_independence"]["limit_min"], .973) + self.assertEqual(report["structure"]["php"]["component_entanglement"]["limit"], .0375) self.assertIn("5-star checks met; missed: ", out) self.assertIn("php unit_size >30", out) self.assertIn("against the 5-star limit", out) diff --git a/hydra-gates/scripts/lib/test_write_gates_json.py b/hydra-gates/scripts/lib/test_write_gates_json.py new file mode 100644 index 00000000..21368eb1 --- /dev/null +++ b/hydra-gates/scripts/lib/test_write_gates_json.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Conduction +# SPDX-License-Identifier: EUPL-1.2 +"""Tests for write_gates_json.py, the runner's gates.json writer. + + 1. Every declared gate is listed; one that emitted nothing is "not-run". + 2. A PASS has 0 findings; a FAIL or WARNING takes the count its reason + states, ignoring numbers inside the log path; without one it counts the + gate's findings log; a skip has no count. + 3. Every declared gate has a category, and only the matrix categories occur. + 4. The runner itself writes gates.json at full scope. + +Run: python3 scripts/lib/test_write_gates_json.py +""" +import io +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +import write_gates_json as w # noqa: E402 + +RUNNER = os.path.join(os.path.dirname(HERE), "run-hydra-gates.sh") + + +def declared_of_runner(): + with open(RUNNER, encoding="utf-8") as fh: + text = fh.read() + return {int(n): name for n, name in re.findall(r'_(?:pass|fail|skip) ([0-9]+) "([^"]+)"', text)} + + +class Records(unittest.TestCase): + def setUp(self): + self.logs = tempfile.mkdtemp(prefix="gates-json-") + + def tearDown(self): + shutil.rmtree(self.logs, ignore_errors=True) + + def build(self, lines, declared): + return {g["gate"]: g for g in w.build(w.parse_records(io.StringIO("".join(lines))), declared, + "full", self.logs)["gates"]} + + def test_a_declared_gate_without_a_verdict_is_not_run(self): + gates = self.build(["1\tspdx-headers\tPASS\t\n"], {1: "spdx-headers", 2: "forbidden-patterns"}) + self.assertEqual(gates[1]["verdict"], "pass") + self.assertEqual(gates[1]["findings"], 0) + self.assertEqual(gates[2]["verdict"], "not-run") + self.assertIsNone(gates[2]["findings"]) + + def test_a_failure_takes_the_count_from_its_reason_not_from_the_log_path(self): + gates = self.build(["60\ticon-vocabulary\tFAIL\t3 icon(s) outside the canonical vocabulary (ADR-077); " + "see /tmp/hydra-gates.7/hydra-gate-icon-vocabulary.log\n"], {60: "icon-vocabulary"}) + self.assertEqual((gates[60]["findings"], gates[60]["findings_from"]), (3, "verdict")) + + def test_a_count_with_a_thousands_separator(self): + gates = self.build(["19\te2e-coverage\tWARNING\t1,187 scenario(s) missing @e2e\n"], {19: "e2e-coverage"}) + self.assertEqual(gates[19]["findings"], 1187) + + def test_a_failure_without_a_count_counts_its_log(self): + with open(os.path.join(self.logs, "hydra-gate-composer-audit.log"), "w") as fh: + fh.write("advisory one\n\nadvisory two\n") + gates = self.build(["4\tcomposer-audit\tFAIL\tCVEs or advisories — see /x/hydra-gate-composer-audit.log\n"], + {4: "composer-audit"}) + self.assertEqual((gates[4]["findings"], gates[4]["findings_from"]), (2, "log")) + + def test_a_failure_with_neither_has_no_count(self): + gates = self.build(["53\teffective-manifest-crossref\tFAIL\tajv not resolvable\n"], + {53: "effective-manifest-crossref"}) + self.assertIsNone(gates[53]["findings"]) + + def test_skips_and_not_applicable_have_no_count(self): + gates = self.build(["16\tspec-coverage\tNOT APPLICABLE\tno delta base, 0 methods\n", + "33\taxe-core\tSKIPPED\tstructural: 2 inputs missing\n"], + {16: "spec-coverage", 33: "axe-core"}) + self.assertEqual(gates[16]["verdict"], "not-applicable") + self.assertEqual(gates[33]["verdict"], "skipped") + self.assertIsNone(gates[16]["findings"]) + self.assertIsNone(gates[33]["findings"]) + + def test_the_last_verdict_of_a_gate_wins(self): + gates = self.build(["7\tno-admin-idor\tWARNING\t2 sub-findings\n", "7\tno-admin-idor\tFAIL\t5 methods\n"], + {7: "no-admin-idor"}) + self.assertEqual((gates[7]["verdict"], gates[7]["findings"]), ("fail", 5)) + + +class Categories(unittest.TestCase): + def test_every_declared_gate_has_a_matrix_category(self): + declared = declared_of_runner() + self.assertGreater(len(declared), 90) + missing = sorted(set(declared) - set(w.CATEGORY)) + self.assertEqual(missing, [], "declared gates without a category") + self.assertEqual(set(w.CATEGORY.values()), {"security", "accessibility", "architecture", "specs", + "licensing", "dependencies", "ui", "other"}) + + +class Runner(unittest.TestCase): + def test_the_runner_writes_gates_json(self): + app = tempfile.mkdtemp(prefix="gates-json-app-") + logs = tempfile.mkdtemp(prefix="gates-json-logs-") + try: + subprocess.run(["git", "init", "-q", app], check=True) + os.makedirs(os.path.join(app, "lib")) + with open(os.path.join(app, "lib", "A.php"), "w") as fh: + fh.write(" +# SPDX-License-Identifier: EUPL-1.2 +"""Write gates.json: one record per declared gate of one run-hydra-gates.sh run. + +The runner pipes its verdict records to stdin, one per line, tab separated: + + TAB TAB TAB + +where verdict is PASS, FAIL, WARNING, NOT APPLICABLE or SKIPPED. `--declared` +names the runner's inventory file (` ` per line), so a declared +gate that emitted no verdict is still listed, as "not-run". + +Each record holds gate, name, verdict, findings, findings_from, scope and +category. `findings` is + 0 for a PASS, + n for a FAIL or WARNING whose reason states a count (the first number + in the reason, before any log path), findings_from "verdict", + n otherwise the non-empty lines of the gate's findings log + (hydra-gate-.log in the log directory), findings_from "log", + null when neither is available, or the gate did not judge. + +Usage: + write_gates_json.py --out FILE --scope full|diff --declared FILE + [--log-dir DIR] [--app NAME] [--base REF] < records +""" +import argparse +import json +import os +import re +import sys + +# The quality-matrix category of every declared gate. Tests, coverage and the +# maintainability gates are "other": the matrix shows them as measures of their own. +CATEGORY = {} +for _cat, _gates in { + "security": (5, 6, 7, 8, 9, 11, 30, 48, 50, 82, 97, 121), + "accessibility": (12, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45), + "architecture": (10, 14, 17, 18, 20, 23, 24, 27, 49, 54, 56, 59, 61, 64, 66, 83, 98, 106, 108, 110, + 111, 115, 116, 119, 120), + "specs": (16, 46, 95, 113), + "licensing": (1, 28, 94), + "dependencies": (4, 84, 93), + "ui": (13, 15, 22, 51, 52, 53, 55, 60, 62, 63, 68, 69, 70, 96, 100, 102, 103, 104, 105, 107, 114), + "other": (2, 3, 19, 21, 25, 26, 29, 47, 57, 58, 65, 67, 101, 109, 112, 122, 123), +}.items(): + for _g in _gates: + CATEGORY[_g] = _cat + +VERDICTS = {"PASS": "pass", "FAIL": "fail", "WARNING": "warning", "NOT APPLICABLE": "not-applicable", + "SKIPPED": "skipped"} +COUNT = re.compile(r"(? 3 else "") + return out + + +def parse_declared(path): + out = {} + with open(path, encoding="utf-8") as fh: + for line in fh: + parts = line.split() + if len(parts) >= 2 and parts[0].isdigit(): + out[int(parts[0])] = parts[1] + return out + + +def build(records, declared, scope, log_dir, app=None, base=None): + gates = [] + for gate in sorted(set(records) | set(declared)): + if gate in records: + name, verdict, reason = records[gate] + gates.append(record(gate, name, verdict, reason, scope, log_dir)) + else: + gates.append(record(gate, declared[gate], "not-run", "", scope, log_dir)) + tally = {} + for g in gates: + tally[g["verdict"]] = tally.get(g["verdict"], 0) + 1 + return {"schema": 1, "app": app, "scope": scope, "base": base, "declared": len(declared), + "verdicts": tally, "gates": gates} + + +def main(argv): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--out", required=True) + ap.add_argument("--scope", required=True, choices=("full", "diff")) + ap.add_argument("--declared", required=True) + ap.add_argument("--log-dir") + ap.add_argument("--app") + ap.add_argument("--base") + args = ap.parse_args(argv) + doc = build(parse_records(sys.stdin), parse_declared(args.declared), args.scope, args.log_dir, + args.app, args.base or None) + with open(args.out, "w", encoding="utf-8") as fh: + json.dump(doc, fh, indent=1) + fh.write("\n") + print(f"[hydra-gates] gates.json: {len(doc['gates'])} gate(s) written to {args.out}") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/hydra-gates/scripts/run-hydra-gates.sh b/hydra-gates/scripts/run-hydra-gates.sh index 442e5b09..97de3fcb 100755 --- a/hydra-gates/scripts/run-hydra-gates.sh +++ b/hydra-gates/scripts/run-hydra-gates.sh @@ -1947,16 +1947,24 @@ _gate_stamp() { " _GATE_T_PREV="${_now}" } +# _record — one line of gates.json input, kept in +# memory and written once by the summary (scripts/lib/write_gates_json.py). +_GATE_RECORDS="" +_record() { + _GATE_RECORDS="${_GATE_RECORDS}$1 $2 $3 $(printf '%s' "${4:-}" | tr '\t\n' ' ') +" +} _fail() { set +e # backstop — see the errexit invariant at the top of this file local _reason _reason=$(printf '%s' "${3:-}" | tr '\n' ' ') + _record "$1" "$2" FAIL "${_reason}" echo "[gate-$1] $2: FAIL${_reason:+ — ${_reason}}" _FAILED=$((_FAILED + 1)) _EMITTED_GATES="${_EMITTED_GATES}$1 " _gate_stamp "$1" } -_pass() { set +e; echo "[gate-$1] $2: PASS"; _EMITTED_GATES="${_EMITTED_GATES}$1 "; _gate_stamp "$1"; } +_pass() { set +e; _record "$1" "$2" PASS ""; echo "[gate-$1] $2: PASS"; _EMITTED_GATES="${_EMITTED_GATES}$1 "; _gate_stamp "$1"; } # _warn — A GATE THAT REPORTED A REAL FINDING AND IS DELIBERATELY NOT BLOCKING. # @@ -1976,6 +1984,7 @@ _warn() { set +e local _reason _reason=$(printf '%s' "${3:-}" | tr '\n' ' ') + _record "$1" "$2" WARNING "${_reason}" echo "[gate-$1] $2: WARNING${_reason:+ — ${_reason}}" _WARNED=$((_WARNED + 1)) _EMITTED_GATES="${_EMITTED_GATES}$1 " @@ -2173,15 +2182,18 @@ _skip() { _gate_stamp "$1" case "${_cat}" in na) + _record "$1" "$2" "NOT APPLICABLE" "${_reason}" echo "[gate-$1] $2: NOT APPLICABLE — ${_reason}" _NA_GATES="${_NA_GATES}$1 " ;; structural|wiring) + _record "$1" "$2" SKIPPED "${_cat}: ${_reason}" echo "[gate-$1] $2: SKIPPED (${_cat}) — ${_reason}" _SKIPPED_GATES="${_SKIPPED_GATES}$1 " ;; *) echo "[gate-$1] $2: FAIL — internal error: _skip called with reason category '${_cat}', which is not one of na|structural|wiring. Refusing to guess: an unclassified skip would silently stop counting against coverage." + _record "$1" "$2" FAIL "internal error: _skip called with reason category '${_cat}'" _FAILED=$((_FAILED + 1)) _EMITTED_GATES="${_EMITTED_GATES}$1 " ;; @@ -13767,8 +13779,9 @@ fi # REPORT. The whole risk profile against the 4-star limits, written to # sig-profile.json and sig-profile.md in $HYDRA_GATE_SIG_REPORT_DIR (the # log directory by default). Report-only: it PASSES with its census line. -# HYDRA_GATE_SIG_PROFILE=five-star compares the report with half of each -# 4-star limit instead (the checker reads it; the ratchet ignores it). +# HYDRA_GATE_SIG_PROFILE=five-star compares the report with the derived +# 5-star limits instead (the checker reads it; the ratchet ignores it). +# Every row also carries the star level (5 to 1) its share earns. # # Ships as a WARNING under the fleet rule for new gates: the package resolves # at @main for every core app, so a blocking ratchet would land everywhere the @@ -14094,6 +14107,25 @@ _a11y_has_markup_dir || _declare_na "no src/, templates/ or appinfo/templates/ [ -f composer.json ] || _declare_na "no composer.json — this repo has no Composer surface, so there is no dependency resolution for a release-age cooldown to govern." \ 93 +# --------------------------------------------------------------------------- +# gates.json: every declared gate as one JSON record (gate, name, verdict, +# findings, scope, category), next to the findings logs, so a consumer reads +# the run without parsing verdict lines. $HYDRA_GATE_GATES_JSON moves it. +# A writer that fails is reported and never changes the run's verdict. +# --------------------------------------------------------------------------- +_gates_json="${HYDRA_GATE_GATES_JSON:-${HYDRA_GATE_LOG_DIR}/gates.json}" +_gates_declared="${HYDRA_GATE_LOG_DIR}/gates-declared.txt" +printf '%s\n' "${_declared}" > "${_gates_declared}" 2>/dev/null +if command -v python3 > /dev/null 2>&1 && [ -f "${SCRIPT_DIR}/lib/write_gates_json.py" ]; then + _gates_scope=full; [ "${SCOPE_TO_DIFF}" = "1" ] && _gates_scope=diff + printf '%s' "${_GATE_RECORDS}" | python3 "${SCRIPT_DIR}/lib/write_gates_json.py" --out "${_gates_json}" \ + --scope "${_gates_scope}" --declared "${_gates_declared}" --log-dir "${HYDRA_GATE_LOG_DIR}" \ + --app "$(basename "$(pwd)")" --base "${BASE_REF:-}" \ + || echo "[hydra-gates] gates.json: the writer failed; the verdict lines above are the record of this run." +else + echo "[hydra-gates] gates.json: not written (python3 or write_gates_json.py missing)." +fi + _emitted_n=$(printf '%s\n' ${_EMITTED_GATES} | grep -c . || true) _emitted_n="${_emitted_n:-0}" diff --git a/hydra-gates/tests/helper-suite-timings.tsv b/hydra-gates/tests/helper-suite-timings.tsv index 632eb078..04929242 100644 --- a/hydra-gates/tests/helper-suite-timings.tsv +++ b/hydra-gates/tests/helper-suite-timings.tsv @@ -143,3 +143,4 @@ test_php_template_scope.py 0 test_shipped_contracts_documented.py 0 test_source_scope.py 0 test_vendored_schema_freshness.js 1 +test_write_gates_json.py 25 From c67bf79fc69897bc971e1b66825767e983df6c82 Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Thu, 5 Nov 2026 11:44:17 +0100 Subject: [PATCH 2/2] feat(hydra-gates): gate 122 estimates the SIG maintainability rating and its levers The report now carries `rating`: a continuous 0.5 to 5.5 estimate per check (Alves, Correia and Visser 2011, eq. 1), per property and language, per property across languages (weighted by code), per sub-characteristic and overall. SIG publishes no power-mean exponent for the 2026 model, so the overall rating is reported under p = 1 and p = -1 as a range. Levers: per property and language, the overall gain from fixing the next 1,000 exposed lines worst band first, and the lines needed to raise the property by 0.5. A lever is sure when it ranks in the top 3 under both exponents. Report rows now also carry the lines above each bound. Component independence and entanglement get assumed 3 and 2 star caps for the estimate only; their discrete check still reads <4 below 4 stars. Assisted-by: ClaudeCode:claude-opus-5-5 --- .../scripts/lib/check_sig_maintainability.py | 283 +++++++++++++++++- .../lib/test_check_sig_maintainability.py | 143 +++++++++ 2 files changed, 421 insertions(+), 5 deletions(-) diff --git a/hydra-gates/scripts/lib/check_sig_maintainability.py b/hydra-gates/scripts/lib/check_sig_maintainability.py index e56db257..6e03d97b 100755 --- a/hydra-gates/scripts/lib/check_sig_maintainability.py +++ b/hydra-gates/scripts/lib/check_sig_maintainability.py @@ -49,6 +49,16 @@ than 2. Component independence and entanglement have no band under 4 stars and read "<4" there. Chosen with --profile, or HYDRA_GATE_SIG_PROFILE when the flag is absent. + +RATING ESTIMATE (report mode) +----------------------------- +`rating` in the JSON, and a paragraph in the markdown: an estimate of SIG's +continuous rating (0.5 to 5.5) per check, property, sub-characteristic and +overall, and the levers that raise the overall rating most per 1,000 lines +fixed. SIG publishes no power-mean exponent, so the overall rating is a range +over p = 1 and p = -1. The method and its assumptions are listed above +rating_estimate() and in research/sig-stars.md section 4. The profile does +not change the estimate. The profile changes only the limits a report is compared with; the ratchet's unit bounds and the gate's advisory status do not change. @@ -567,8 +577,9 @@ def profile(units, key, limits, kind): total = sum(u["nloc"] for u in units) or 1 rows = [] for i, (bound, cap) in enumerate(limits): - share = sum(u["nloc"] for u in units if u[key] > bound) / total - rows.append({"above": bound, "share": round(share, 4), "limit": cap, "ok": share <= cap, + loc = sum(u["nloc"] for u in units if u[key] > bound) + share = loc / total + rows.append({"above": bound, "share": round(share, 4), "loc": loc, "limit": cap, "ok": share <= cap, "stars": stars_of(kind, round(share, 4), i)}) return rows @@ -853,8 +864,9 @@ def structure(app, files, deps, prof=PROFILES[DEFAULT_PROFILE]): total = sum(loc.values()) or 1 coupling = [] for i, (bound, cap) in enumerate(prof["limits"]["module_coupling"]): - share = sum(loc[m] for m in modules if len(incoming[m]) > bound) / total - coupling.append({"above": bound, "share": round(share, 4), "limit": cap, "ok": share <= cap, + above = sum(loc[m] for m in modules if len(incoming[m]) > bound) + share = above / total + coupling.append({"above": bound, "share": round(share, 4), "loc": above, "limit": cap, "ok": share <= cap, "stars": stars_of("module_coupling", round(share, 4), i)}) root, comp = component_of_factory(modules) hidden = sum(loc[m] for m in modules if all(comp(s) == comp(m) for s in incoming[m])) @@ -878,7 +890,8 @@ def structure(app, files, deps, prof=PROFILES[DEFAULT_PROFILE]): for m in sorted(modules, key=lambda m: (-len(incoming[m]), m))[:10]], "component_root": root or ".", "components": dict(comp_loc.most_common()), - "component_independence": {"hidden_share": round(independence, 4), "limit_min": prof["independence_min"], + "component_independence": {"hidden_share": round(independence, 4), "hidden_loc": hidden, + "limit_min": prof["independence_min"], "ok": independence >= prof["independence_min"], "stars": stars_of("component_independence", round(independence, 4))}, "component_entanglement": { @@ -948,6 +961,7 @@ def build_report(app, lizard, jscpd, profile_name=DEFAULT_PROFILE, reader=None): report["structure"]["php"] = structure(app, php, php_dependencies(app, php), prof) if py: report["structure"]["python"] = structure(app, py, python_dependencies(app, py), prof) + report["rating"] = rating_estimate(report) return report @@ -969,10 +983,268 @@ def checks_of(report): return out +# ---- rating estimate --------------------------------------------------------- +# +# An estimate of SIG's continuous maintainability rating (0.5 to 5.5), built +# from the report. Sources and assumptions: research/sig-stars.md section 4. +# check Alves, Correia, Visser 2011, eq. 1: linear interpolation +# between the caps of the star level a share falls in; a +# property is the minimum over its checks. Worse than 2 stars +# falls linearly to 0.5 at 100% (assumed). +# language a property's per-language ratings, weighted by the code that +# property measures in each language (assumed). +# sub-char. plain mean of the mapped properties (criteria v12.0 mapping, +# volume and component balance dropped). +# overall power mean of the sub-characteristics. SIG publishes no +# exponent, so p = 1 and p = -1 are both reported. + +# Assumed caps on the exposed share (1 - hidden) and on the violation degree. +# Only 4 stars is published; every level doubles the exposure of the one above. +RATING_EXPOSED_CAPS = { + "component_independence": {5: .0275, 4: .055, 3: .11, 2: .22}, + "component_entanglement": {5: .0375, 4: .075, 3: .15, 2: .30}, +} +RATED_PROPERTIES = ("duplication", "unit_size", "unit_complexity", "unit_interfacing", + "module_coupling", "component_independence", "component_entanglement") +SUBCHARACTERISTICS = { + "analysability": ("duplication", "unit_size"), + "modifiability": ("duplication", "unit_complexity", "module_coupling"), + "testability": ("unit_complexity", "component_independence"), + "modularity": ("module_coupling", "component_independence", "component_entanglement"), + "reusability": ("unit_size", "unit_interfacing"), +} +EXPONENTS = (1, -1) +LEVER_LOC = 1000 +SURE_TOP = 3 + + +def rating_caps(kind, index=None): + """{5: cap, 4: cap, 3: cap, 2: cap} of one check, as a maximum exposed share.""" + if kind in RATING_EXPOSED_CAPS: + return RATING_EXPOSED_CAPS[kind] + bands = STAR_BANDS[kind] + return {lvl: bands[lvl] if index is None else bands[lvl][index] for lvl in (5, 4, 3, 2)} + + +def interpolate(value, caps): + """Continuous rating of one exposed share: s0 + 0.5 - (v - t0) / (t1 - t0), + t0 and t1 the caps around v (0 below 5 stars, 100% past 2 stars).""" + lower = 0.0 + for level in (5, 4, 3, 2): + if value <= caps[level]: + start, upper = level, caps[level] + break + lower = caps[level] + else: + start, upper = 1, 1.0 + if upper <= lower: + return start + 0.5 + return max(0.5, min(5.5, start + 0.5 - (value - lower) / (upper - lower))) + + +def power_mean(values, p): + values = [v for v in values if v is not None] + if not values: + return None + if p == 0: + product = 1.0 + for v in values: + product *= v + return product ** (1 / len(values)) + return (sum(v ** p for v in values) / len(values)) ** (1 / p) + + +def rating_inputs(report): + """{property: {language: {"loc": weight, "above": [exposed loc per check]}}} + from a gate-122 report. Entanglement carries "value" (the violation degree) + instead of exposed loc. Reports written before rows carried "loc" are + read from share x total.""" + def above(rows, total): + return [r["loc"] if "loc" in r else round(r["share"] * total) for r in rows] + out = {prop: {} for prop in RATED_PROPERTIES} + for lang, d in report.get("languages", {}).items(): + total = d.get("unit_loc") or 0 + for prop in ("unit_size", "unit_complexity", "unit_interfacing"): + out[prop][lang] = {"loc": total, "above": above(d[prop], total)} + dup = d.get("duplication") + if dup: + out["duplication"][lang] = {"loc": dup["code_lines"], "above": [dup["redundant_lines"]]} + for lang, s in (report.get("structure") or {}).items(): + if not s: + continue + total = s["module_loc"] + out["module_coupling"][lang] = {"loc": total, "above": above(s["module_coupling"], total)} + ci = s["component_independence"] + hidden = ci["hidden_loc"] if "hidden_loc" in ci else round(ci["hidden_share"] * total) + out["component_independence"][lang] = {"loc": total, "above": [total - hidden]} + out["component_entanglement"][lang] = {"loc": total, "value": s["component_entanglement"]["violation_degree"]} + return out + + +def check_ratings(prop, entry): + """The continuous rating of each check of one property in one language.""" + if "value" in entry: + return [interpolate(entry["value"], rating_caps(prop))] + single = len(entry["above"]) == 1 + return [interpolate(a / entry["loc"], rating_caps(prop, None if single else i)) + for i, a in enumerate(entry["above"])] + + +def property_rating(prop, entry): + if not entry or not entry.get("loc"): + return None + return min(check_ratings(prop, entry)) + + +def rate_inputs(inputs): + """Property, sub-characteristic and overall ratings of rating_inputs().""" + props = {} + for prop in RATED_PROPERTIES: + langs = {lang: (property_rating(prop, e), e["loc"]) for lang, e in inputs.get(prop, {}).items()} + weighted = [(r, w) for r, w in langs.values() if r is not None and w] + rating = sum(r * w for r, w in weighted) / sum(w for _r, w in weighted) if weighted else None + props[prop] = {"rating": rating, + "languages": {lg: {"rating": r, "loc": w} for lg, (r, w) in langs.items() if r is not None}} + subs = {} + for sub, members in SUBCHARACTERISTICS.items(): + subs[sub] = power_mean([props[m]["rating"] for m in members], 1) + overall = {p: power_mean(list(subs.values()), p) for p in EXPONENTS} + return {"properties": props, "subcharacteristics": subs, "overall": overall} + + +def fix_loc(prop, entry, n): + """`entry` with n exposed lines brought to low risk, worst band first: the + lines above the highest bound go before those above the next one down. + Fixed duplication is removed code, so it also leaves the total.""" + above = list(entry["above"]) + exclusive = [a - b for a, b in zip(above, above[1:] + [0])] + left = n + for i in reversed(range(len(exclusive))): + take = min(left, exclusive[i]) + exclusive[i] -= take + left -= take + cumulative = [sum(exclusive[i:]) for i in range(len(exclusive))] + total = entry["loc"] - (n - left if prop == "duplication" else 0) + return {"loc": total, "above": cumulative} + + +def with_entry(inputs, prop, lang, entry): + out = {p: dict(v) for p, v in inputs.items()} + out[prop][lang] = entry + return out + + +def levers(inputs, base=None): + """For every property and language: how much the overall rating rises when + the next LEVER_LOC exposed lines (or all of them, when fewer) move to low + risk, worst band first (a finite difference, under each exponent), and the + lines needed to raise the property by 0.5. Ranked by that gain under each + exponent and listed by the sum of both ranks; "sure" when a lever is in the + top SURE_TOP under every exponent.""" + base = base or rate_inputs(inputs) + if all(base["overall"][p] is None for p in EXPONENTS): + return [] + out = [] + for prop in RATED_PROPERTIES: + for lang, entry in sorted(inputs.get(prop, {}).items()): + if "value" in entry or not entry.get("loc") or entry["above"][0] <= 0: + continue + n = min(LEVER_LOC, entry["above"][0]) + after = rate_inputs(with_entry(inputs, prop, lang, fix_loc(prop, entry, n))) + out.append({"property": prop, "language": lang, "fixable_loc": entry["above"][0], "fixed_loc": n, + "gain_per_kloc": {p: after["overall"][p] - base["overall"][p] for p in EXPONENTS}, + "loc_for_half_star": loc_for_half_star(inputs, prop, lang, + base["properties"][prop]["rating"])}) + for p in EXPONENTS: + for rank, lever in enumerate(sorted(out, key=lambda lv: (-lv["gain_per_kloc"][p], lv["property"], + lv["language"])), 1): + lever.setdefault("rank", {})[p] = rank + for lever in out: + lever["sure"] = all(lever["rank"][p] <= SURE_TOP for p in EXPONENTS) + return sorted(out, key=lambda lv: (sum(lv["rank"].values()), lv["rank"][-1])) + + +def loc_for_half_star(inputs, prop, lang, before): + """Fewest lines fixed in one language that raise the property's + cross-language rating by 0.5, or None when fixing all of them does not.""" + entry = inputs[prop][lang] + target = min(before + 0.5, 5.5) + + def reached(n): + after = rate_inputs(with_entry(inputs, prop, lang, fix_loc(prop, entry, n))) + return after["properties"][prop]["rating"] >= target - 1e-9 + high = entry["above"][0] + if not reached(high): + return None + low = 0 + while high - low > 1: + mid = (low + high) // 2 + if reached(mid): + high = mid + else: + low = mid + return high + + +def rating_estimate(report): + """The rating section of a report: every level, the range, the levers.""" + inputs = rating_inputs(report) + rated = rate_inputs(inputs) + overall = [v for v in rated["overall"].values() if v is not None] + + def r2(x): + return None if x is None else round(x, 2) + return { + "note": "Estimate of SIG's maintainability rating, 0.5 to 5.5. SIG publishes 4-star limits only and " + "no power-mean exponent, so the overall rating is a range over p = 1 and p = -1.", + "overall": {"p1": r2(rated["overall"][1]), "p-1": r2(rated["overall"][-1]), + "range": [r2(min(overall)), r2(max(overall))] if overall else None}, + "subcharacteristics": {k: r2(v) for k, v in rated["subcharacteristics"].items()}, + "properties": {k: {"rating": r2(v["rating"]), + "languages": {lg: {"rating": r2(x["rating"]), "loc": x["loc"]} + for lg, x in v["languages"].items()}} + for k, v in rated["properties"].items()}, + "levers": [{"property": lv["property"], "language": lv["language"], + "fixable_loc": lv["fixable_loc"], "fixed_loc": lv["fixed_loc"], + "gain_per_kloc": {"p1": round(lv["gain_per_kloc"][1], 4), + "p-1": round(lv["gain_per_kloc"][-1], 4)}, + "rank": {"p1": lv["rank"][1], "p-1": lv["rank"][-1]}, + "sure": lv["sure"], "loc_for_half_star": lv["loc_for_half_star"]} + for lv in levers(inputs, rated)], + } + + def pct(x): return f"{x * 100:.1f}%" +def rating_markdown(rating, top=5): + """The rating estimate as markdown lines: range, sub-characteristics, properties, levers.""" + if not rating or not rating["overall"]["range"]: + return [] + + def r(x): + return "n/a" if x is None else f"{x:.2f}" + lo, hi = rating["overall"]["range"] + out = ["", f"Estimated SIG maintainability rating: {lo:.2f} to {hi:.2f} " + f"(power mean p = -1 {r(rating['overall']['p-1'])}, p = 1 {r(rating['overall']['p1'])}). " + "An estimate: SIG publishes only the 4-star limits and no exponent.", "", + "Sub-characteristics: " + ", ".join(f"{k} {r(v)}" for k, v in rating["subcharacteristics"].items()) + ".", + "Properties: " + ", ".join(f"{k.replace('_', ' ')} {r(v['rating'])}" + for k, v in rating["properties"].items()) + "."] + if rating["levers"]: + out += ["", f"Levers: the overall gain from fixing the next {LEVER_LOC:,} exposed lines (fewer when fewer " + "exist), worst band first.", "", + "| lever | fixable lines | gain, p = 1 / p = -1 | lines for +0.5 | sure |", "|---|---|---|---|---|"] + for lv in rating["levers"][:top]: + half = (f"{lv['loc_for_half_star']:,}" if lv["loc_for_half_star"] is not None + else f"not by {lv['language']} alone") + out.append(f"| {lv['property'].replace('_', ' ')} ({lv['language']}) | {lv['fixable_loc']:,} | " + f"{lv['gain_per_kloc']['p1']:+.4f} / {lv['gain_per_kloc']['p-1']:+.4f} | {half} | " + f"{'yes' if lv['sure'] else 'no'} |") + return out + + def markdown(report): out = [f"## SIG maintainability profile: {report['app']}", "", f"lizard {LIZARD_VERSION}, jscpd {JSCPD_VERSION}; limits: {report['limits']}. " @@ -1002,6 +1274,7 @@ def cells(rows): f"density {e['communication_density']}, violation degree {e['violation_degree']}."] checks = checks_of(report) out += ["", f"{sum(1 for _c, ok in checks if ok)} of {len(checks)} {report['stars']} checks met. ✗ marks a miss."] + out += rating_markdown(report.get("rating")) if report["skipped_files"]: out += ["", f"{len(report['skipped_files'])} file(s) lizard could not read, NOT measured: " + ", ".join(s["file"] for s in report["skipped_files"][:10])] diff --git a/hydra-gates/scripts/lib/test_check_sig_maintainability.py b/hydra-gates/scripts/lib/test_check_sig_maintainability.py index 9537946a..091c8fbd 100755 --- a/hydra-gates/scripts/lib/test_check_sig_maintainability.py +++ b/hydra-gates/scripts/lib/test_check_sig_maintainability.py @@ -30,6 +30,12 @@ entanglement below 4 stars); HYDRA_GATE_SIG_PROFILE picks the profile, --profile overrides it, four-star is the default, and an unknown profile is exit 2. + 7. RATING: the interpolation reproduces the worked example of Alves, + Correia and Visser 2011 (ArgoUML, 2.68); p = 1 is the plain mean of the + sub-characteristics and p = -1 is never higher; levers are ordered by + their gain on a fixture; a language with no code, a single language, + and a report without independence or entanglement are rated; a report + carries the estimate. Run: python3 scripts/lib/test_check_sig_maintainability.py """ @@ -518,6 +524,10 @@ def test_report_rows_carry_their_stars(self): self.assertEqual(self.above_30(report)["stars"], 4) self.assertEqual(php["duplication"]["stars"], 5) self.assertIn(php["unit_size"][0]["stars"], (1, 2, 3, 4, 5)) + self.assertEqual(php["unit_size"][1]["loc"], 44) + rating = report["rating"] + self.assertLessEqual(rating["overall"]["range"][0], rating["overall"]["range"][1]) + self.assertIn("Estimated SIG maintainability rating", out) self.assertIn("4★", out) def test_default_is_four_star(self): @@ -641,5 +651,138 @@ def test_php_and_python_dependencies(self): repo.close() +def rated_report(langs, structure=None): + """A report with the fields rating_inputs() reads. `langs` maps a language + to (unit_loc, size, complexity, interfacing, (code_lines, redundant)), + each property a triple of cumulative shares.""" + out = {"languages": {}, "structure": {}} + for lang, (loc, size, ccn, params, dup) in langs.items(): + out["languages"][lang] = { + "unit_loc": loc, + "unit_size": [{"share": x} for x in size], + "unit_complexity": [{"share": x} for x in ccn], + "unit_interfacing": [{"share": x} for x in params], + "duplication": {"code_lines": dup[0], "redundant_lines": dup[1]}, + } + for lang, (loc, coupling, hidden, degree) in (structure or {}).items(): + out["structure"][lang] = {"module_loc": loc, "module_coupling": [{"share": x} for x in coupling], + "component_independence": {"hidden_share": hidden}, + "component_entanglement": {"violation_degree": degree}} + return out + + +GOOD = (10000, (.30, .10, .02), (.10, .03, .0), (.10, .02, .0), (10000, 200)) + + +class Rating(unittest.TestCase): + def test_interpolation_reproduces_alves_2011_argouml(self): + # Table III (b), McCabe: cumulative caps for moderate / high / very high. + table = {5: (.179, .099, .033), 4: (.234, .169, .067), 3: (.313, .238, .106), 2: (.391, .298, .167)} + profile = (.258, .187, .099) + ratings = [sig.interpolate(v, {lvl: table[lvl][i] for lvl in table}) for i, v in enumerate(profile)] + self.assertAlmostEqual(min(ratings), 2.68, places=2) + self.assertEqual(ratings.index(min(ratings)), 2) + + def test_interpolation_edges(self): + caps = sig.rating_caps("duplication") + self.assertEqual(sig.interpolate(0, caps), 5.5) + self.assertAlmostEqual(sig.interpolate(caps[4], caps), 3.5) + self.assertAlmostEqual(sig.interpolate(caps[2], caps), 1.5) + self.assertAlmostEqual(sig.interpolate(1.0, caps), 0.5) + self.assertAlmostEqual(sig.interpolate((caps[2] + 1) / 2, caps), 1.0) + values = [sig.interpolate(x / 1000, caps) for x in range(1001)] + self.assertEqual(values, sorted(values, reverse=True)) + + def test_p1_is_the_plain_mean_and_harmonic_is_never_higher(self): + self.assertAlmostEqual(sig.power_mean([1.0, 2.0, 4.5], 1), 2.5) + self.assertAlmostEqual(sig.power_mean([1.0, 2.0, 4.0], -1), 3 / (1 + .5 + .25)) + report = rated_report({"php": GOOD, "vue": (2000, (.6, .4, .2), (.3, .2, .05), (.3, .1, .03), (3000, 900))}, + {"php": (8000, (.05, .02, .01), .80, .10)}) + rated = sig.rate_inputs(sig.rating_inputs(report)) + subs = list(rated["subcharacteristics"].values()) + self.assertAlmostEqual(rated["overall"][1], sum(subs) / len(subs)) + self.assertLessEqual(rated["overall"][-1], rated["overall"][1]) + modifiability = [rated["properties"][p]["rating"] for p in ("duplication", "unit_complexity", "module_coupling")] + self.assertAlmostEqual(rated["subcharacteristics"]["modifiability"], sum(modifiability) / 3) + + def test_languages_are_weighted_by_their_code(self): + report = rated_report({"php": GOOD, "vue": (2000, (.6, .4, .2), (.3, .2, .05), (.3, .1, .03), (3000, 900))}) + prop = sig.rate_inputs(sig.rating_inputs(report))["properties"]["unit_size"] + php, vue = prop["languages"]["php"]["rating"], prop["languages"]["vue"]["rating"] + self.assertAlmostEqual(prop["rating"], (php * 10000 + vue * 2000) / 12000) + + def test_a_language_without_code_carries_no_weight(self): + alone = sig.rate_inputs(sig.rating_inputs(rated_report({"php": GOOD}))) + empty = (0, (0, 0, 0), (0, 0, 0), (0, 0, 0), (0, 0)) + both = sig.rate_inputs(sig.rating_inputs(rated_report({"php": GOOD, "typescript": empty}))) + self.assertEqual(alone["overall"], both["overall"]) + self.assertNotIn("typescript", both["properties"]["unit_size"]["languages"]) + + def test_a_single_language_is_its_own_property_rating(self): + rated = sig.rate_inputs(sig.rating_inputs(rated_report({"php": GOOD}))) + for prop in ("unit_size", "unit_complexity", "unit_interfacing", "duplication"): + self.assertEqual(rated["properties"][prop]["rating"], rated["properties"][prop]["languages"]["php"]["rating"]) + + def test_no_structure_leaves_independence_and_entanglement_out(self): + rated = sig.rate_inputs(sig.rating_inputs(rated_report({"php": GOOD}))) + for prop in ("module_coupling", "component_independence", "component_entanglement"): + self.assertIsNone(rated["properties"][prop]["rating"]) + self.assertIsNone(rated["subcharacteristics"]["modularity"]) + self.assertEqual(rated["subcharacteristics"]["testability"], rated["properties"]["unit_complexity"]["rating"]) + self.assertIsNotNone(rated["overall"][-1]) + + def test_independence_and_entanglement_are_rated_below_four(self): + report = rated_report({"php": GOOD}, {"php": (10000, (0, 0, 0), .50, .20)}) + self.assertEqual(sig.stars_of("component_independence", .50), "<4") + props = sig.rate_inputs(sig.rating_inputs(report))["properties"] + self.assertLess(props["component_independence"]["rating"], 1.5) + self.assertGreater(props["component_independence"]["rating"], 0.5) + self.assertLess(props["component_entanglement"]["rating"], 2.5) + self.assertGreater(props["component_entanglement"]["rating"], 1.5) + + def test_levers_are_ordered_by_their_gain(self): + # php duplication is the worst property (30% redundant, 1 star), but on + # the flat 1-star slope 1,000 lines move it little; vue unit size and + # php unit complexity sit on steeper slopes and lead under both exponents. + bad_dup = (10000, (.30, .10, .02), (.10, .03, .0), (.10, .02, .0), (10000, 3000)) + report = rated_report({"php": bad_dup, "vue": (1500, (.6, .4, .2), (.1, .03, 0), (.1, .02, 0), (1500, 0))}, + {"php": (10000, (0, 0, 0), 1.0, .05)}) + inputs = sig.rating_inputs(report) + self.assertLess(sig.rate_inputs(inputs)["properties"]["duplication"]["rating"], 2) + levers = sig.levers(inputs) + order = [(lv["property"], lv["language"]) for lv in levers] + self.assertEqual(set(order[:2]), {("unit_size", "vue"), ("unit_complexity", "php")}) + self.assertEqual(order[4], ("duplication", "php")) + self.assertTrue(levers[0]["sure"] and levers[1]["sure"]) + self.assertFalse(levers[4]["sure"]) + for p in sig.EXPONENTS: + gains = [lv["gain_per_kloc"][p] for lv in sorted(levers, key=lambda lv: lv["rank"][p])] + self.assertEqual(gains, sorted(gains, reverse=True)) + self.assertTrue(all(g > 0 for g in gains)) + for lv in levers: + self.assertEqual(lv["sure"], max(lv["rank"].values()) <= sig.SURE_TOP) + vue = next(lv for lv in levers if order[levers.index(lv)] == ("unit_size", "vue")) + self.assertEqual(vue["fixed_loc"], 900) + self.assertNotIn("component_entanglement", {p for p, _l in order}) + + def test_loc_for_half_star_is_the_fewest_lines(self): + report = rated_report({"php": (10000, (.30, .10, .02), (.10, .03, .0), (.10, .02, .0), (10000, 1500))}) + inputs = sig.rating_inputs(report) + before = sig.rate_inputs(inputs)["properties"]["duplication"]["rating"] + n = sig.loc_for_half_star(inputs, "duplication", "php", before) + self.assertIsNotNone(n) + + def after(k): + fixed = sig.with_entry(inputs, "duplication", "php", sig.fix_loc("duplication", inputs["duplication"]["php"], k)) + return sig.rate_inputs(fixed)["properties"]["duplication"]["rating"] + self.assertGreaterEqual(after(n), before + 0.5 - 1e-9) + self.assertLess(after(n - 1), before + 0.5 - 1e-9) + + def test_fixing_takes_the_worst_band_first(self): + entry = {"loc": 1000, "above": [500, 300, 100]} + self.assertEqual(sig.fix_loc("unit_size", entry, 150), {"loc": 1000, "above": [350, 150, 0]}) + self.assertEqual(sig.fix_loc("duplication", {"loc": 1000, "above": [100]}, 40), {"loc": 960, "above": [60]}) + + if __name__ == "__main__": unittest.main(verbosity=1)