diff --git a/bench/report/facts.json b/bench/report/facts.json new file mode 100644 index 0000000..68fed4c --- /dev/null +++ b/bench/report/facts.json @@ -0,0 +1,24 @@ +{ + "_note": "Fixed facts the report states in words. Each carries the file or PR it was read from. Measured numbers never live here.", + "device": { + "value": "Raspberry Pi 5, 4 cores, 4 GB, Zynthian vangelis-2610.1-2, Debian 13 trixie arm64, kernel 6.18", + "source": "memory reference_zynthian_rpi5_2026-10-26.md" + }, + "live_ms": {"value": 5, "source": "FreeMixer/zynthian-ui PR 2, ZYNTHIAN_PLUGIN_LIVE_MS"}, + "ph_port": {"value": 5570, "source": "FreeMixer/zynthian-ui PR 1"}, + "modhost_port": {"value": 5555, "source": "FreeMixer/zynthian-sys PR 2"}, + "clap_files": {"value": 57, "source": "bench-harness brief, Pi 5 /usr/lib/clap"}, + "qualify_command": {"value": "plugin-qualify --all --isolate --rates 48000 --quanta 256", "source": "bench-harness brief"}, + "upstream_state": { + "value": "All changes are pull requests inside the FreeMixer forks. None is open on the zynthian organisation yet.", + "source": "briefs 2026-10-26-zynthian-ui-engine and zynthian-sys-openmixer: open nothing on the upstream organisation" + }, + "prs": [ + {"repo": "FreeMixer/zynthian-sys", "number": 1, "role": "OpenMixer apt repository and install recipe"}, + {"repo": "FreeMixer/zynthian-sys", "number": 2, "role": "plugin-hostd as the mod-host of MOD-UI, one switch"}, + {"repo": "FreeMixer/zynthian-sys", "number": 3, "role": "MOD GUIs generated from the plugin descriptions"}, + {"repo": "FreeMixer/zynthian-sys", "number": 4, "role": "OpenMixer in the default install and the update path"}, + {"repo": "FreeMixer/zynthian-ui", "number": 1, "role": "PH engine for LV2 and CLAP plugins"}, + {"repo": "FreeMixer/zynthian-ui", "number": 2, "role": "PH as default engine, snapshot converter, menus from qualification"} + ] +} diff --git a/bench/report/make_report.py b/bench/report/make_report.py new file mode 100644 index 0000000..855283c --- /dev/null +++ b/bench/report/make_report.py @@ -0,0 +1,596 @@ +#!/usr/bin/env python3 +"""Writes the Zynthian acceptance report (report.md and report.html) from measurement files. + +Every number in the report is read from a file named below; the prose only holds +facts listed in facts.json with their source. A section whose data does not exist +prints "not measured yet". With --strict the exit code is 1 when any section is +missing, so nothing unmeasured is published. + +Sources: + --bench PATH omx-bench/1 documents (files or directories); the newest of class rpi5 is used + --results DIR the device notes (grouped-vs-processes.md is read, never written) + --earlier DIR hand-run outputs (zy-96test*.out, zy-qpilot.out) from the Pi 5 + --qualification DIR plugin-qualify interchange documents (lv2-plugin-measurements) + --coverage FILE LV2 coverage summary.txt + --prs FILE pull request states (written by --fetch-prs) +""" +import argparse +import glob +import html +import json +import os +import re +import subprocess +import sys + +NOT_YET = "not measured yet" +HOSTS = [("H1", "omx-clap-host alone"), ("H2", "plugin-hostd"), ("H3", "mod-host"), ("H4", "jalv")] +HOST_FAMILY = ["idle", "clap1", "lv2-1", "chain3", "clap16"] + + +# ---------------------------------------------------------------- readers + +def load_bench(paths): + docs = [] + files = [] + for p in paths: + if os.path.isdir(p): + for root, _dirs, names in os.walk(p): + files += [os.path.join(root, n) for n in names if n.endswith(".json")] + elif os.path.isfile(p): + files.append(p) + for f in sorted(set(files)): + try: + with open(f) as fh: + d = json.load(fh) + except (OSError, ValueError): + continue + if isinstance(d, dict) and d.get("schema") == "omx-bench/1" and isinstance(d.get("scenarios"), list): + d["_file"] = f + docs.append(d) + return docs + + +def latest_of_class(docs, cls): + sel = [d for d in docs if d.get("machine", {}).get("class") == cls] + return max(sel, key=lambda d: d.get("id", "")) if sel else None + + +def _read(path): + try: + with open(path, errors="replace") as fh: + return fh.read() + except OSError: + return None + + +def parse_earlier(dirpath): + """Hand runs on the Pi 5: per rate the 3-plugin chain and Pianoteq, with the file they came from.""" + runs = {} + gov = None + for f in sorted(glob.glob(os.path.join(dirpath, "zy-96test*.out"))): + text = _read(f) + if not text: + continue + m = re.search(r"uri: (\S+)", text) + uri = m.group(1) if m else None + if re.search(r"governor: \w+ -> performance", text): + gov = "performance" + for block in re.split(r"(?m)^=== ", text)[1:]: + h = re.match(r"(\d+) Hz, (\d+) frames", block) + if not h: + continue + rate, frames = int(h.group(1)), int(h.group(2)) + r = runs.setdefault(rate, {"frames": frames}) + c = re.search(r"3-plugin chain \+ sine:\s+avg ([\d.]+)% max ([\d.]+)% \((\d+) samples\)", block) + if c: + r["chain3"] = {"avg": float(c.group(1)), "max": float(c.group(2)), "n": int(c.group(3)), "src": os.path.basename(f)} + add = re.search(r"add PIANOTEQ lv2 \(\d+\): resp (-?\d+)", block) + p = re.search(r"Pianoteq 5-note chords:\s+avg ([\d.]+)% max ([\d.]+)% \((\d+) samples\)", block) + if add and p and int(add.group(1)) >= 0: + r["pianoteq"] = {"avg": float(p.group(1)), "max": float(p.group(2)), "n": int(p.group(3)), "src": os.path.basename(f), "uri": uri} + return {"runs": runs, "governor": gov} if runs else None + + +def parse_qpilot(dirpath): + text = _read(os.path.join(dirpath, "zy-qpilot.out")) + if not text: + return None + sec = re.search(r"seconds=(\d+)", text) + counts = {} + summary = text.split("-- verdict summary", 1)[-1].split("-- sample", 1)[0] + for m in re.finditer(r"(?m)^\s+(\d+) (suitable|unsuitable|conditional|unknown)\s*$", summary): + counts[m.group(2)] = int(m.group(1)) + reasons = {} + sample = text.split("-- sample", 1)[-1].split("-- stderr", 1)[0] + for line in sample.splitlines()[1:]: + parts = line.split() + if len(parts) >= 3 and parts[1] in ("unsuitable", "conditional", "unknown"): + reasons[parts[2]] = reasons.get(parts[2], 0) + 1 + if not counts: + return None + return {"seconds": int(sec.group(1)) if sec else None, "counts": counts, "sample_reasons": reasons} + + +def parse_qdocs(dirpath): + """Verdict counts and deciding reasons from plugin-qualify interchange documents.""" + counts, reasons, n = {}, {}, 0 + for f in sorted(glob.glob(os.path.join(dirpath, "**", "*.json"), recursive=True)): + try: + with open(f) as fh: + d = json.load(fh) + except (OSError, ValueError): + continue + for v in d.get("verdicts", []) if isinstance(d, dict) else []: + n += 1 + r = v.get("rating", "unknown") + counts[r] = counts.get(r, 0) + 1 + if v.get("deciding") and r != "suitable": + reasons[v["deciding"]] = reasons.get(v["deciding"], 0) + 1 + return {"counts": counts, "reasons": reasons, "verdicts": n} if n else None + + +def parse_coverage(path): + text = _read(path) + if not text: + return None + out = {} + m = re.search(r"jalv [\d.]+ .*?\n\s+hosted (\d+) of (\d+)", text, re.S) + if m: + out["jalv"] = (int(m.group(1)), int(m.group(2))) + m = re.search(r"mod-host [\d.]+ .*?\n\s+hosted (\d+) of (\d+)", text, re.S) + if m: + out["mod-host"] = (int(m.group(1)), int(m.group(2))) + m = re.search(r"omx-clap-core: (\d+) of (\d+)", text) + if m: + out["omx-clap-core"] = (int(m.group(1)), int(m.group(2))) + return out if len(out) == 3 else None + + +def fetch_prs(facts, out_path, run=subprocess.run): + rows = [] + for p in facts["prs"]: + r = run(["gh", "pr", "view", str(p["number"]), "-R", p["repo"], "--json", + "number,title,state,baseRefName,headRefName,url"], capture_output=True, text=True) + if r.returncode != 0: + raise SystemExit("gh failed for %s#%s: %s" % (p["repo"], p["number"], r.stderr.strip())) + d = json.loads(r.stdout) + d["repo"] = p["repo"] + rows.append(d) + with open(out_path, "w") as fh: + json.dump(rows, fh, indent=1) + + +# ---------------------------------------------------------------- model + +class Doc: + def __init__(self): + self.blocks = [] + self.missing = [] + self.section = "" + + def h1(self, t): self.blocks.append(("h1", t)) + def h2(self, t): self.section = t; self.blocks.append(("h2", t)) + def h3(self, t): self.blocks.append(("h3", t)) + def p(self, t): self.blocks.append(("p", t)) + def quote(self, t): self.blocks.append(("quote", t)) + def table(self, head, rows): self.blocks.append(("table", head, rows)) + def bars(self, title, items, unit): self.blocks.append(("bars", title, items, unit)) + def ul(self, items): self.blocks.append(("ul", items)) + + def na(self, what): + self.missing.append("%s: %s" % (self.section, what)) + self.blocks.append(("na", "%s: %s." % (what, NOT_YET))) + + +def num(v, d=1): + if v is None: + return "n/a" + s = ("%." + str(d) + "f") % v + return s + + +# ---------------------------------------------------------------- sections + +def sec_change(doc, facts, prs): + doc.h2("1. What we change and why") + doc.p("Zynthian runs every LV2 plugin in its own jalv process and MOD-UI chains in a JACK internal " + "client called mod-host. We propose four changes, each a pull request in the FreeMixer forks " + "of zynthian-ui and zynthian-sys.") + doc.ul([ + "plugin-hostd becomes the host behind LV2 and CLAP plugins. It runs plugins as workers, and puts the members " + "of one Zynthian chain into one worker instead of one process each.", + "A new engine, PH, in zynthian-ui talks to one plugin-hostd daemon (port %s) and offers every LV2 and CLAP " + "plugin the daemon can host, with controller pages generated from the plugin descriptions." + % facts["ph_port"]["value"], + "The plugin menus use plugin-qualify records: a latency and real-time-safety tag on each entry, live " + "plugins first, plugins that failed qualification hidden with the reason one setting away.", + "OpenMixer is part of the default install and of the update path: the apt repository, the packages, " + "plugin-hostd as the mod-host of MOD-UI and the qualification sync. jack_load and jalv stay available " + "by one setting each.", + ]) + doc.p("The reason is the cost of one process per plugin (memory, context switches, a JACK client each), plugins " + "that crash or stall the host they share, and no way to tell a user before loading a plugin whether it " + "suits live playing. Sections 3 and 4 hold the measurements; section 5 holds the way back.") + doc.p(facts["upstream_state"]["value"]) + + +def sec_user(doc, facts): + doc.h2("2. What the user gets") + doc.ul([ + "More plugins that work: coverage of the installed LV2 set is in section 4, and CLAP plugins are " + "offered next to LV2 ones.", + "Bad plugins are hidden, with the reason: plugins pinned refused, or that crashed or hung during " + "qualification, leave the menu. ZYNTHIAN_PLUGIN_SHOW_HIDDEN=1 lists them last, greyed, with the reason.", + "Each entry carries a tag such as 'live, 1.2 ms, RT'. live means real-time safe with a measured latency " + "of at most %s ms at the worst measured rate; studio is the rest; unverified means no record yet. " + "The tag leaves out any figure the record does not hold." % facts["live_ms"]["value"], + "Chains in fewer processes: the members of one chain share one worker. Section 3 counts the processes " + "and the memory.", + "Controller pages come from the plugin descriptions, eight controls to a page, and MOD GUIs are generated " + "for plugins that ship none.", + "A crashed worker takes only its own plugins down; the daemon respawns it (recover time in section 3 when measured).", + ]) + + +def host_cell(s): + if s is None: + return "-" + if s["status"] != "ok": + return "%s (%s)" % (s["status"], s.get("note", "")) + return None + + +def sec_measure(doc, facts, rpi, earlier, results_dir, ci_docs): + doc.h2("3. Measurements") + doc.p("Machine: %s. DSP load is the JACK DSP load read with jack_cpu_load; memory is resident (RSS) and " + "proportional (PSS) from /proc. Values are never taken from a meter on the device." % facts["device"]["value"]) + if rpi is None: + doc.p("No omx-bench/1 run of class rpi5 exists yet.") + if ci_docs: + doc.p("Runs of other classes (%s) are not shown: they are not Pi 5 numbers." % + ", ".join(sorted(d["id"] for d in ci_docs))) + meta = None + else: + meta = rpi + m = rpi["machine"] + sw = ", ".join("%s %s" % (k, v) for k, v in rpi.get("software", {}).items() if v) + doc.p("Run %s on %s, governor %s. Software: %s." % (rpi["id"], m.get("model", "?"), m.get("governor", "?"), sw)) + scen = rpi["scenarios"] if rpi else [] + + def pick(sid, host, rate, frames): + for s in scen: + if s["id"] == sid and s["host"] == host and s["rate"] == rate and s["frames"] == frames: + return s + return None + + # 3a four hosts + doc.h3("The four hosts") + keys = sorted({(s["id"], s["rate"], s["frames"], s["title"]) for s in scen if s["id"] in HOST_FAMILY}, + key=lambda k: (HOST_FAMILY.index(k[0]), k[1], k[2])) + if not keys: + doc.na("CPU, RAM and process count for H1 omx-clap-host alone, H2 plugin-hostd, H3 mod-host, H4 jalv") + else: + head = ["scenario", "rate", "frames"] + [n for _h, n in HOSTS] + for title, fn in (("CPU: DSP load, average (maximum), percent", lambda s: "%s (%s)" % (num(s["dsp_load_avg"], 2), num(s["dsp_load_max"], 2))), + ("RAM: resident memory of the host processes, MB (PSS in brackets when measured)", + lambda s: "%s%s" % (num(s.get("rss_mb_total", s["rss_mb"])), (" (%s)" % num(s["pss_mb_total"])) if s.get("pss_mb_total") is not None else "")), + ("Processes", lambda s: str(s["processes"]) if s.get("processes") is not None else "n/a")): + rows = [] + for sid, rate, frames, t in keys: + row = [t, str(rate), str(frames)] + for h, _n in HOSTS: + s = pick(sid, h, rate, frames) + c = host_cell(s) + row.append(fn(s) if c is None else c) + rows.append(row) + doc.h3(title) + doc.table(head, rows) + # bars for the CPU of the busiest scenario family + items = [] + for sid, rate, frames, t in keys: + for h, n in HOSTS: + s = pick(sid, h, rate, frames) + if s and s["status"] == "ok" and s["dsp_load_avg"] is not None: + items.append(("%s, %d Hz/%d, %s" % (sid, rate, frames, n), s["dsp_load_avg"])) + doc.bars("DSP load, average", items, "%") + if any(s["id"] == "kill1" for s in scen): + doc.h3("Recovery after kill -9 of the process holding one plugin") + doc.table(["host", "rate", "frames", "recover ms", "status", "note"], + [[dict(HOSTS)[s["host"]], str(s["rate"]), str(s["frames"]), num(s["recover_ms"], 0), s["status"], s.get("note", "")] + for s in scen if s["id"] == "kill1"]) + else: + doc.na("recovery time after a killed worker") + # xruns and latency + doc.h3("Xruns and latency") + xr = [s for s in scen if s.get("xruns") is not None] + if xr: + doc.table(["scenario", "host", "rate", "frames", "xruns"], + [[s["id"], s["host"], str(s["rate"]), str(s["frames"]), str(s["xruns"])] for s in xr]) + else: + doc.na("xruns per host") + la = [s for s in scen if s.get("latency_ms") is not None] + if la: + doc.table(["scenario", "host", "rate", "frames", "latency ms"], + [[s["id"], s["host"], str(s["rate"]), str(s["frames"]), num(s["latency_ms"], 2)] for s in la]) + else: + doc.na("round-trip latency per host") + + # 3b grouped vs processes + doc.h3("One worker for a chain against one process per plugin") + cvp = [s for s in scen if s.get("mode") and s["id"].startswith("cvp")] + if not cvp: + doc.na("grouped against individual processes (3, 8 and 16 plugins)") + else: + rows = [] + for key in sorted({(s["id"], s["rate"], s["frames"]) for s in cvp}, key=lambda k: (int(re.sub(r"\D", "", k[0]) or 0), k[1], k[2])): + for s in sorted([x for x in cvp if (x["id"], x["rate"], x["frames"]) == key], key=lambda x: (x["host"], x["mode"])): + if s["status"] != "ok": + rows.append([s["title"], str(s["rate"]), s["host"] + " " + s["mode"], s["status"] + ": " + s.get("note", ""), "", "", "", "", ""]) + continue + rows.append([s["title"], str(s["rate"]), s["host"] + " " + s["mode"], str(s.get("processes", "n/a")), + num(s.get("rss_mb_total")), num(s.get("pss_mb_total")), num(s.get("cpu_pct"), 2), + num(s["dsp_load_avg"], 2), num(s["add_ms"], 0)]) + doc.table(["set", "rate", "mode", "processes", "RSS MB", "PSS MB", "CPU % of a core", "DSP load %", "add ms"], rows) + for key in sorted({(s["id"], s["rate"], s["frames"]) for s in cvp}): + ind = next((s for s in cvp if (s["id"], s["rate"], s["frames"]) == key and s["host"] == "H2" and s["mode"] == "individual" and s["status"] == "ok"), None) + ch = next((s for s in cvp if (s["id"], s["rate"], s["frames"]) == key and s["host"] == "H2" and s["mode"] == "chain" and s["status"] == "ok"), None) + if ind and ch and ind.get("processes") and ch.get("processes") and ind.get("pss_mb_total") and ch.get("pss_mb_total"): + doc.p("%s at %d Hz: the chain runs in %d process%s against %d, and uses %.2f times the proportional memory of the individual mode." + % (ind["title"], key[1], ch["processes"], "" if ch["processes"] == 1 else "es", ind["processes"], + ch["pss_mb_total"] / ind["pss_mb_total"])) + stress = [s for s in scen if s["id"].startswith("stress")] + doc.h3("Under stress") + if stress: + doc.table(["scenario", "host", "mode", "rate", "processes", "PSS MB", "CPU %", "DSP load %", "xruns"], + [[s["title"], s["host"], s.get("mode", ""), str(s["rate"]), str(s.get("processes", "n/a")), + num(s.get("pss_mb_total")), num(s.get("cpu_pct"), 2), num(s["dsp_load_avg"], 2), str(s.get("xruns", "n/a"))] for s in stress]) + else: + doc.na("grouped against individual processes under stress") + gp = _read(os.path.join(results_dir, "grouped-vs-processes.md")) + if gp: + doc.p("Device notes, read as written from grouped-vs-processes.md:") + doc.quote(gp.strip()) + + # 3c Pianoteq + doc.h3("Pianoteq 9 at 48 and 96 kHz") + rows, src = [], None + for rate in (48000, 96000): + s = next((x for x in scen if x["id"] == "pianoteq" and x["rate"] == rate and x["status"] == "ok"), None) + if s: + rows.append([str(rate), str(s["frames"]), dict(HOSTS)[s["host"]], num(s["dsp_load_avg"]), num(s["dsp_load_max"]), str(s["samples"]), rpi["id"]]) + elif earlier and earlier["runs"].get(rate, {}).get("pianoteq"): + e = earlier["runs"][rate] + q = e["pianoteq"] + rows.append([str(rate), str(e["frames"]), "plugin-hostd (hand run)", num(q["avg"]), num(q["max"]), str(q["n"]), q["src"]]) + if len(rows) == 2: + doc.p("Five-note chords from jack_midiseq into the plugin's MIDI input, DSP load averaged over the samples.") + doc.table(["rate", "frames", "host", "DSP load avg %", "max %", "samples", "source"], rows) + else: + doc.na("Pianoteq 9 at both rates") + # hand-run chain3 when the bench doc has none + if earlier and not any(s["id"] == "chain3" and s["host"] == "H2" for s in scen): + r = [[str(rate), str(e["frames"]), num(e["chain3"]["avg"]), num(e["chain3"]["max"]), str(e["chain3"]["n"]), e["chain3"]["src"]] + for rate, e in sorted(earlier["runs"].items()) if e.get("chain3")] + if r: + doc.p("Hand run of the three-plugin chain (a CLAP compressor, the Calf compressor, a CLAP delay) under plugin-hostd, " + "governor %s:" % (earlier.get("governor") or "unrecorded")) + doc.table(["rate", "frames", "DSP load avg %", "max %", "samples", "source"], r) + + +def sec_qual(doc, facts, rpi, qpilot, qdocs, coverage): + doc.h2("4. Qualification coverage") + counts = reasons = None + seconds = None + src = None + if qdocs: + counts, reasons, src = qdocs["counts"], qdocs["reasons"], "plugin-qualify documents (%d verdicts)" % qdocs["verdicts"] + elif qpilot: + counts, reasons, src = qpilot["counts"], None, "qualification run on the Pi 5, host profile openmixer-console (zy-qpilot.out)" + if qpilot: + seconds = qpilot["seconds"] + elif rpi and rpi.get("qualification", {}).get("seconds") is not None: + seconds = rpi["qualification"]["seconds"] + if not counts: + doc.na("verdict counts") + else: + total = sum(counts.values()) + doc.p("Source: %s. %d plugins judged%s." % (src, total, + ", %s s with %s over %d CLAP files" % (num(seconds, 0), facts["qualify_command"]["value"], facts["clap_files"]["value"]) if seconds else "")) + doc.table(["verdict", "plugins", "share"], [[k, str(v), "%.1f%%" % (100.0 * v / total)] for k, v in sorted(counts.items(), key=lambda kv: -kv[1])]) + hidden = counts.get("unsuitable", 0) + doc.p("%d of %d plugins are unsuitable (a measured failure) and are not qualified. The menus hide the plugins pinned " + "refused or that crashed or hung; the others are offered with their tag." % (hidden, total)) + doc.h3("Refused, with reasons") + if reasons: + doc.table(["deciding reason", "plugins"], [[k, str(v)] for k, v in sorted(reasons.items(), key=lambda kv: -kv[1])]) + elif qpilot and qpilot["sample_reasons"]: + doc.table(["deciding reason (first lines of the run's sample only)", "plugins in sample"], [[k, str(v)] for k, v in qpilot["sample_reasons"].items()]) + doc.na("the full reason breakdown of the unsuitable plugins (needs the interchange documents from the Pi 5)") + else: + doc.na("reasons for refused plugins") + doc.h3("LV2 hosting coverage of the installed set") + if coverage: + doc.table(["host", "hosted", "of"], [[k, str(v[0]), str(v[1])] for k, v in coverage.items()]) + doc.p("Measured on the desk (x86_64 Fedora 44), not on the Pi 5.") + else: + doc.na("LV2 coverage") + + +def sec_risk(doc, facts): + doc.h2("5. Risk and how to revert") + doc.ul([ + "mod-host backend: one variable, ZYNTHIAN_MODHOST_BACKEND, picks what answers MOD-UI on port %s: jack_load " + "(Zynthian's own mod-host, as today) or plugin-hostd. The unit keeps its name, mod-host.service, so MOD-UI " + "and the MOD-UI engine are untouched. sbin/set_modhost_backend.sh jack_load|plugin-hostd flips it, restarts " + "mod-ui and prints each step." % facts["modhost_port"]["value"], + "Engine: ZYNTHIAN_PLUGIN_ENGINE=jalv brings the jalv menus back. The jalv entries stay in the engine list, so a " + "snapshot that holds a JV processor loads as saved.", + "Snapshots: nothing converts at load time. tools/convert_snapshots_to_ph.py is run once by hand, writes " + "foo.ph.zss beside foo.zss and never overwrites.", + "Defaults: each install step runs once and is recorded in /zynthian/config/openmixer-defaults, so a setting " + "changed by hand stays. The install restarts no audio service and says what takes effect at the next start.", + "Unchanged: MOD-UI chains keep the MD engine, MIDI tools stay with jalv, Zynthian's own services and presets " + "are not touched, and without plugin-hostd installed the menus keep jalv.", + ]) + + +def sec_try(doc, facts): + doc.h2("6. How to try it") + doc.ul([ + "Add the repository and the packages: curl -fsSLO https://www.openmixer.org/deb/openmixer-release.deb, then " + "sudo apt install ./openmixer-release.deb and sudo apt install plugin-hostd omx-clap-host omx-clap-scan omx-panel.", + "Check out the zynthian-ui branch of the PH engine in /zynthian/zynthian-ui and restart zynthian. Enable one " + "LV2 effect in webconf so its (PH) entry shows up, then add a chain with an LV2 and a CLAP (PH) effect.", + "Existing Zynthian: check out the zynthian-sys branch in /zynthian/zynthian-sys and run " + "scripts/update_zynthian_sys.sh; read the 'OpenMixer defaults' report; a second run reports nothing to change. " + "Apply with sbin/set_modhost_backend.sh plugin-hostd or reboot.", + "New image: run setup_zynthian.sh, then scripts/omx_prequalify_image.sh on an arm64 machine before taking the " + "image, so the image ships qualified. The first boot otherwise runs the qualification sync two minutes after " + "boot at the lowest priority.", + "To go back: section 5.", + ]) + + +def sec_ask(doc, facts, prs): + doc.h2("7. What we ask of the Zynthian project") + doc.p("Review and take the pull requests below, in this order. The sys pull requests come first because the " + "engine reads what they install; the second ui pull request is stacked on the first.") + if not prs: + doc.na("pull request states") + return + role = {(p["repo"], p["number"]): p["role"] for p in facts["prs"]} + doc.table(["repository", "PR", "what", "state", "base"], + [[p["repo"], "#%d" % p["number"], role.get((p["repo"], p["number"]), p["title"]), p["state"].lower(), p["baseRefName"]] for p in prs]) + doc.p(facts["upstream_state"]["value"]) + + +def build(args): + with open(args.facts) as fh: + facts = json.load(fh) + docs = load_bench(args.bench) + rpi = latest_of_class(docs, "rpi5") + others = [d for d in docs if d is not rpi and d.get("machine", {}).get("class") != "rpi5"] + earlier = parse_earlier(args.earlier) if args.earlier else None + qpilot = parse_qpilot(args.earlier) if args.earlier else None + qdocs = parse_qdocs(args.qualification) if args.qualification and os.path.isdir(args.qualification) else None + coverage = parse_coverage(args.coverage) if args.coverage else None + prs = None + if args.prs and os.path.isfile(args.prs): + with open(args.prs) as fh: + prs = json.load(fh) + doc = Doc() + doc.h1("plugin-hostd, the PH engine, qualified menus and OpenMixer by default in Zynthian") + sec_change(doc, facts, prs) + sec_user(doc, facts) + sec_measure(doc, facts, rpi, earlier, args.results, others) + sec_qual(doc, facts, rpi, qpilot, qdocs, coverage) + sec_risk(doc, facts) + sec_try(doc, facts) + sec_ask(doc, facts, prs) + return doc + + +# ---------------------------------------------------------------- renderers + +def md_cell(c): + return str(c).replace("|", "\\|").replace("\n", " ") + + +def to_md(doc): + out = [] + for b in doc.blocks: + k = b[0] + if k == "h1": out.append("# " + b[1]) + elif k == "h2": out.append("## " + b[1]) + elif k == "h3": out.append("### " + b[1]) + elif k == "p": out.append(b[1]) + elif k == "na": out.append("*" + b[1] + "*") + elif k == "quote": out.append("\n".join("> " + l for l in b[1].splitlines())) + elif k == "ul": out.append("\n".join("- " + i for i in b[1])) + elif k == "table": + out.append("| " + " | ".join(md_cell(h) for h in b[1]) + " |\n|" + "---|" * len(b[1]) + "\n" + + "\n".join("| " + " | ".join(md_cell(c) for c in r) + " |" for r in b[2])) + elif k == "bars": + out.append("\n".join("- %s: %s%s" % (l, num(v, 2), b[3]) for l, v in b[2]) if b[2] else "") + return "\n\n".join(x for x in out if x) + "\n" + + +CSS = """:root{--bg:#fff;--fg:#1d2129;--mut:#5b6573;--line:#d9dee5;--bar:#2b6cb0;--bgq:#f3f5f8} +@media(prefers-color-scheme:dark){:root{--bg:#14171c;--fg:#e6e9ee;--mut:#9aa4b2;--line:#2c333d;--bar:#6aa7e6;--bgq:#1c2128}} +body{background:var(--bg);color:var(--fg);font:16px/1.55 system-ui,sans-serif;max-width:60rem;margin:0 auto;padding:1rem 16px 4rem} +h1{font-size:1.6rem}h2{margin-top:2.5rem;border-bottom:1px solid var(--line);padding-bottom:.2rem}h3{margin-top:1.6rem} +table{border-collapse:collapse;display:block;overflow-x:auto;font-size:.9rem}th,td{border:1px solid var(--line);padding:.25rem .5rem;text-align:left} +th{background:var(--bgq)}blockquote{background:var(--bgq);margin:1rem 0;padding:.5rem 1rem;white-space:pre-wrap;border-left:3px solid var(--line)} +.na{color:var(--mut);font-style:italic}svg text{fill:var(--fg);font-size:11px}.bar{fill:var(--bar)}""" + + +def svg_bars(items): + if not items: + return "" + mx = max(v for _l, v in items) or 1.0 + w, lab, h = 560, 280, 18 + rows = [] + for i, (l, v) in enumerate(items): + y = i * h + bw = max(1.0, (w - lab - 60) * v / mx) + rows.append('%s' + '%s' % (y + 11, html.escape(l), lab, y, bw, lab + bw + 4, y + 11, num(v, 2))) + return '%s' % (w, h * len(items), "".join(rows)) + + +def to_html(doc): + e = html.escape + out = ["" + "Zynthian acceptance report" % CSS] + for b in doc.blocks: + k = b[0] + if k in ("h1", "h2", "h3"): out.append("<%s>%s" % (k, e(b[1]), k)) + elif k == "p": out.append("

%s

" % e(b[1])) + elif k == "na": out.append('

%s

' % e(b[1])) + elif k == "quote": out.append("
%s
" % e(b[1])) + elif k == "ul": out.append("" % "".join("
  • %s
  • " % e(i) for i in b[1])) + elif k == "table": + out.append("%s%s
    " % ( + "".join("%s" % e(h) for h in b[1]), + "".join("%s" % "".join("%s" % e(str(c)) for c in r) for r in b[2]))) + elif k == "bars": + out.append("

    %s (%s)

    %s" % (e(b[1]), e(b[3]), svg_bars(b[2]))) + out.append("") + return "\n".join(out) + + +def main(argv=None): + here = os.path.dirname(os.path.abspath(__file__)) + home = os.path.expanduser("~/Devel/audio/zynthian-results") + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--bench", nargs="*", default=[os.path.join(home, "data")]) + ap.add_argument("--results", default=home) + ap.add_argument("--earlier", default=os.path.join(home, "data", "earlier")) + ap.add_argument("--qualification", default=os.path.join(home, "data", "qualification")) + ap.add_argument("--coverage", default=os.path.join(home, "data", "lv2-coverage-summary.txt")) + ap.add_argument("--prs", default=os.path.join(home, "data", "prs.json")) + ap.add_argument("--facts", default=os.path.join(here, "facts.json")) + ap.add_argument("--out", default=home) + ap.add_argument("--strict", action="store_true", help="exit 1 when any section lacks data") + ap.add_argument("--fetch-prs", action="store_true", help="refresh --prs with gh and exit") + a = ap.parse_args(argv) + if a.fetch_prs: + with open(a.facts) as fh: + fetch_prs(json.load(fh), a.prs) + print("wrote", a.prs) + return 0 + doc = build(a) + os.makedirs(a.out, exist_ok=True) + with open(os.path.join(a.out, "report.md"), "w") as fh: + fh.write(to_md(doc)) + with open(os.path.join(a.out, "report.html"), "w") as fh: + fh.write(to_html(doc)) + for m in doc.missing: + print("missing:", m, file=sys.stderr) + print("wrote %s/report.md and report.html; %d missing" % (a.out, len(doc.missing))) + return 1 if (a.strict and doc.missing) else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bench/report/test_make_report.py b/bench/report/test_make_report.py new file mode 100644 index 0000000..d53040d --- /dev/null +++ b/bench/report/test_make_report.py @@ -0,0 +1,129 @@ +import json +import os +import tempfile +import unittest + +import make_report as m + +FACTS = os.path.join(os.path.dirname(os.path.abspath(m.__file__)), "facts.json") + + +def scen(sid, host, rate=48000, frames=256, **kw): + d = dict(id=sid, title=sid, host=host, rate=rate, frames=frames, plugins=[], dsp_load_avg=1.5, dsp_load_max=2.0, + samples=10, rss_mb=10.0, add_ms=5.0, recover_ms=None, status="ok", note="") + d.update(kw) + return d + + +def bench(cls, scens, id="rpi5-20261027-1"): + return {"schema": "omx-bench/1", "id": id, "machine": {"class": cls, "model": "x", "governor": "performance"}, + "software": {"plugin-hostd": "0.12.0"}, "scenarios": scens} + + +class Case(unittest.TestCase): + def setUp(self): + self.t = tempfile.TemporaryDirectory() + self.d = self.t.name + self.empty = os.path.join(self.d, "empty") + os.makedirs(self.empty) + + def tearDown(self): + self.t.cleanup() + + def write(self, name, text): + p = os.path.join(self.d, name) + os.makedirs(os.path.dirname(p), exist_ok=True) + with open(p, "w") as fh: + fh.write(text if isinstance(text, str) else json.dumps(text)) + return p + + def report(self, name="report.md"): + with open(os.path.join(self.d, "o", name)) as fh: + return fh.read() + + def run_main(self, *extra, results=None): + return m.main(["--bench", os.path.join(self.d, "b"), "--results", results or self.empty, "--earlier", self.empty, + "--qualification", self.empty, "--coverage", os.path.join(self.d, "none"), + "--prs", os.path.join(self.d, "none"), "--facts", FACTS, "--out", os.path.join(self.d, "o")] + list(extra)) + + def test_strict_fails_with_no_data_and_says_not_measured(self): + self.assertEqual(self.run_main("--strict"), 1) + self.assertIn("not measured yet", self.report()) + + def test_non_strict_exits_zero(self): + self.assertEqual(self.run_main(), 0) + + def test_ci_run_is_never_shown_as_pi_numbers(self): + self.write("b/ci.json", bench("ci", [scen("chain3", "H2", dsp_load_avg=77.77)], "ci-1")) + self.run_main() + self.assertNotIn("77.77", self.report()) + + def test_host_table_and_ratio_values(self): + sc = [scen("chain3", h, dsp_load_avg=v) for h, v in (("H1", 1.11), ("H2", 2.22))] + sc.append(scen("chain3", "H3", status="unsupported", note="no mod-host", dsp_load_avg=None, + dsp_load_max=None, rss_mb=None, samples=0)) + sc += [scen("cvp3", "H2", mode="individual", processes=3, pss_mb_total=30.0, rss_mb_total=60.0, cpu_pct=1.0), + scen("cvp3", "H2", mode="chain", processes=1, pss_mb_total=15.0, rss_mb_total=30.0, cpu_pct=0.5)] + self.write("b/rpi.json", bench("rpi5", sc)) + self.run_main() + md = self.report() + self.assertIn("1.11 (2.00)", md) + self.assertIn("unsupported (no mod-host)", md) + self.assertIn("uses 0.50 times the proportional memory", md) + self.assertIn(" device says 42", self.report()) + with open(os.path.join(self.d, "r", "grouped-vs-processes.md")) as fh: + self.assertEqual(fh.read(), "device says 42") + + def test_fetch_prs_uses_gh_and_fails_loudly(self): + class R: + def __init__(s, rc, out): + s.returncode, s.stdout, s.stderr = rc, out, "boom" + facts = {"prs": [{"repo": "o/r", "number": 1, "role": "x"}]} + out = os.path.join(self.d, "prs.json") + m.fetch_prs(facts, out, run=lambda *a, **k: R(0, json.dumps({"number": 1, "title": "t", "state": "OPEN", "baseRefName": "b"}))) + with open(out) as fh: + self.assertEqual(json.load(fh)[0]["repo"], "o/r") + with self.assertRaises(SystemExit): + m.fetch_prs(facts, out, run=lambda *a, **k: R(1, "")) + + +if __name__ == "__main__": + unittest.main()