diff --git a/bench/report/facts.json b/bench/report/facts.json
new file mode 100644
index 0000000..68fed4c
--- /dev/null
+++ b/bench/report/facts.json
@@ -0,0 +1,24 @@
+{
+ "_note": "Fixed facts the report states in words. Each carries the file or PR it was read from. Measured numbers never live here.",
+ "device": {
+ "value": "Raspberry Pi 5, 4 cores, 4 GB, Zynthian vangelis-2610.1-2, Debian 13 trixie arm64, kernel 6.18",
+ "source": "memory reference_zynthian_rpi5_2026-10-26.md"
+ },
+ "live_ms": {"value": 5, "source": "FreeMixer/zynthian-ui PR 2, ZYNTHIAN_PLUGIN_LIVE_MS"},
+ "ph_port": {"value": 5570, "source": "FreeMixer/zynthian-ui PR 1"},
+ "modhost_port": {"value": 5555, "source": "FreeMixer/zynthian-sys PR 2"},
+ "clap_files": {"value": 57, "source": "bench-harness brief, Pi 5 /usr/lib/clap"},
+ "qualify_command": {"value": "plugin-qualify --all --isolate --rates 48000 --quanta 256", "source": "bench-harness brief"},
+ "upstream_state": {
+ "value": "All changes are pull requests inside the FreeMixer forks. None is open on the zynthian organisation yet.",
+ "source": "briefs 2026-10-26-zynthian-ui-engine and zynthian-sys-openmixer: open nothing on the upstream organisation"
+ },
+ "prs": [
+ {"repo": "FreeMixer/zynthian-sys", "number": 1, "role": "OpenMixer apt repository and install recipe"},
+ {"repo": "FreeMixer/zynthian-sys", "number": 2, "role": "plugin-hostd as the mod-host of MOD-UI, one switch"},
+ {"repo": "FreeMixer/zynthian-sys", "number": 3, "role": "MOD GUIs generated from the plugin descriptions"},
+ {"repo": "FreeMixer/zynthian-sys", "number": 4, "role": "OpenMixer in the default install and the update path"},
+ {"repo": "FreeMixer/zynthian-ui", "number": 1, "role": "PH engine for LV2 and CLAP plugins"},
+ {"repo": "FreeMixer/zynthian-ui", "number": 2, "role": "PH as default engine, snapshot converter, menus from qualification"}
+ ]
+}
diff --git a/bench/report/make_report.py b/bench/report/make_report.py
new file mode 100644
index 0000000..855283c
--- /dev/null
+++ b/bench/report/make_report.py
@@ -0,0 +1,596 @@
+#!/usr/bin/env python3
+"""Writes the Zynthian acceptance report (report.md and report.html) from measurement files.
+
+Every number in the report is read from a file named below; the prose only holds
+facts listed in facts.json with their source. A section whose data does not exist
+prints "not measured yet". With --strict the exit code is 1 when any section is
+missing, so nothing unmeasured is published.
+
+Sources:
+ --bench PATH omx-bench/1 documents (files or directories); the newest of class rpi5 is used
+ --results DIR the device notes (grouped-vs-processes.md is read, never written)
+ --earlier DIR hand-run outputs (zy-96test*.out, zy-qpilot.out) from the Pi 5
+ --qualification DIR plugin-qualify interchange documents (lv2-plugin-measurements)
+ --coverage FILE LV2 coverage summary.txt
+ --prs FILE pull request states (written by --fetch-prs)
+"""
+import argparse
+import glob
+import html
+import json
+import os
+import re
+import subprocess
+import sys
+
+NOT_YET = "not measured yet"
+HOSTS = [("H1", "omx-clap-host alone"), ("H2", "plugin-hostd"), ("H3", "mod-host"), ("H4", "jalv")]
+HOST_FAMILY = ["idle", "clap1", "lv2-1", "chain3", "clap16"]
+
+
+# ---------------------------------------------------------------- readers
+
+def load_bench(paths):
+ docs = []
+ files = []
+ for p in paths:
+ if os.path.isdir(p):
+ for root, _dirs, names in os.walk(p):
+ files += [os.path.join(root, n) for n in names if n.endswith(".json")]
+ elif os.path.isfile(p):
+ files.append(p)
+ for f in sorted(set(files)):
+ try:
+ with open(f) as fh:
+ d = json.load(fh)
+ except (OSError, ValueError):
+ continue
+ if isinstance(d, dict) and d.get("schema") == "omx-bench/1" and isinstance(d.get("scenarios"), list):
+ d["_file"] = f
+ docs.append(d)
+ return docs
+
+
+def latest_of_class(docs, cls):
+ sel = [d for d in docs if d.get("machine", {}).get("class") == cls]
+ return max(sel, key=lambda d: d.get("id", "")) if sel else None
+
+
+def _read(path):
+ try:
+ with open(path, errors="replace") as fh:
+ return fh.read()
+ except OSError:
+ return None
+
+
+def parse_earlier(dirpath):
+ """Hand runs on the Pi 5: per rate the 3-plugin chain and Pianoteq, with the file they came from."""
+ runs = {}
+ gov = None
+ for f in sorted(glob.glob(os.path.join(dirpath, "zy-96test*.out"))):
+ text = _read(f)
+ if not text:
+ continue
+ m = re.search(r"uri: (\S+)", text)
+ uri = m.group(1) if m else None
+ if re.search(r"governor: \w+ -> performance", text):
+ gov = "performance"
+ for block in re.split(r"(?m)^=== ", text)[1:]:
+ h = re.match(r"(\d+) Hz, (\d+) frames", block)
+ if not h:
+ continue
+ rate, frames = int(h.group(1)), int(h.group(2))
+ r = runs.setdefault(rate, {"frames": frames})
+ c = re.search(r"3-plugin chain \+ sine:\s+avg ([\d.]+)% max ([\d.]+)% \((\d+) samples\)", block)
+ if c:
+ r["chain3"] = {"avg": float(c.group(1)), "max": float(c.group(2)), "n": int(c.group(3)), "src": os.path.basename(f)}
+ add = re.search(r"add PIANOTEQ lv2 \(\d+\): resp (-?\d+)", block)
+ p = re.search(r"Pianoteq 5-note chords:\s+avg ([\d.]+)% max ([\d.]+)% \((\d+) samples\)", block)
+ if add and p and int(add.group(1)) >= 0:
+ r["pianoteq"] = {"avg": float(p.group(1)), "max": float(p.group(2)), "n": int(p.group(3)), "src": os.path.basename(f), "uri": uri}
+ return {"runs": runs, "governor": gov} if runs else None
+
+
+def parse_qpilot(dirpath):
+ text = _read(os.path.join(dirpath, "zy-qpilot.out"))
+ if not text:
+ return None
+ sec = re.search(r"seconds=(\d+)", text)
+ counts = {}
+ summary = text.split("-- verdict summary", 1)[-1].split("-- sample", 1)[0]
+ for m in re.finditer(r"(?m)^\s+(\d+) (suitable|unsuitable|conditional|unknown)\s*$", summary):
+ counts[m.group(2)] = int(m.group(1))
+ reasons = {}
+ sample = text.split("-- sample", 1)[-1].split("-- stderr", 1)[0]
+ for line in sample.splitlines()[1:]:
+ parts = line.split()
+ if len(parts) >= 3 and parts[1] in ("unsuitable", "conditional", "unknown"):
+ reasons[parts[2]] = reasons.get(parts[2], 0) + 1
+ if not counts:
+ return None
+ return {"seconds": int(sec.group(1)) if sec else None, "counts": counts, "sample_reasons": reasons}
+
+
+def parse_qdocs(dirpath):
+ """Verdict counts and deciding reasons from plugin-qualify interchange documents."""
+ counts, reasons, n = {}, {}, 0
+ for f in sorted(glob.glob(os.path.join(dirpath, "**", "*.json"), recursive=True)):
+ try:
+ with open(f) as fh:
+ d = json.load(fh)
+ except (OSError, ValueError):
+ continue
+ for v in d.get("verdicts", []) if isinstance(d, dict) else []:
+ n += 1
+ r = v.get("rating", "unknown")
+ counts[r] = counts.get(r, 0) + 1
+ if v.get("deciding") and r != "suitable":
+ reasons[v["deciding"]] = reasons.get(v["deciding"], 0) + 1
+ return {"counts": counts, "reasons": reasons, "verdicts": n} if n else None
+
+
+def parse_coverage(path):
+ text = _read(path)
+ if not text:
+ return None
+ out = {}
+ m = re.search(r"jalv [\d.]+ .*?\n\s+hosted (\d+) of (\d+)", text, re.S)
+ if m:
+ out["jalv"] = (int(m.group(1)), int(m.group(2)))
+ m = re.search(r"mod-host [\d.]+ .*?\n\s+hosted (\d+) of (\d+)", text, re.S)
+ if m:
+ out["mod-host"] = (int(m.group(1)), int(m.group(2)))
+ m = re.search(r"omx-clap-core: (\d+) of (\d+)", text)
+ if m:
+ out["omx-clap-core"] = (int(m.group(1)), int(m.group(2)))
+ return out if len(out) == 3 else None
+
+
+def fetch_prs(facts, out_path, run=subprocess.run):
+ rows = []
+ for p in facts["prs"]:
+ r = run(["gh", "pr", "view", str(p["number"]), "-R", p["repo"], "--json",
+ "number,title,state,baseRefName,headRefName,url"], capture_output=True, text=True)
+ if r.returncode != 0:
+ raise SystemExit("gh failed for %s#%s: %s" % (p["repo"], p["number"], r.stderr.strip()))
+ d = json.loads(r.stdout)
+ d["repo"] = p["repo"]
+ rows.append(d)
+ with open(out_path, "w") as fh:
+ json.dump(rows, fh, indent=1)
+
+
+# ---------------------------------------------------------------- model
+
+class Doc:
+ def __init__(self):
+ self.blocks = []
+ self.missing = []
+ self.section = ""
+
+ def h1(self, t): self.blocks.append(("h1", t))
+ def h2(self, t): self.section = t; self.blocks.append(("h2", t))
+ def h3(self, t): self.blocks.append(("h3", t))
+ def p(self, t): self.blocks.append(("p", t))
+ def quote(self, t): self.blocks.append(("quote", t))
+ def table(self, head, rows): self.blocks.append(("table", head, rows))
+ def bars(self, title, items, unit): self.blocks.append(("bars", title, items, unit))
+ def ul(self, items): self.blocks.append(("ul", items))
+
+ def na(self, what):
+ self.missing.append("%s: %s" % (self.section, what))
+ self.blocks.append(("na", "%s: %s." % (what, NOT_YET)))
+
+
+def num(v, d=1):
+ if v is None:
+ return "n/a"
+ s = ("%." + str(d) + "f") % v
+ return s
+
+
+# ---------------------------------------------------------------- sections
+
+def sec_change(doc, facts, prs):
+ doc.h2("1. What we change and why")
+ doc.p("Zynthian runs every LV2 plugin in its own jalv process and MOD-UI chains in a JACK internal "
+ "client called mod-host. We propose four changes, each a pull request in the FreeMixer forks "
+ "of zynthian-ui and zynthian-sys.")
+ doc.ul([
+ "plugin-hostd becomes the host behind LV2 and CLAP plugins. It runs plugins as workers, and puts the members "
+ "of one Zynthian chain into one worker instead of one process each.",
+ "A new engine, PH, in zynthian-ui talks to one plugin-hostd daemon (port %s) and offers every LV2 and CLAP "
+ "plugin the daemon can host, with controller pages generated from the plugin descriptions."
+ % facts["ph_port"]["value"],
+ "The plugin menus use plugin-qualify records: a latency and real-time-safety tag on each entry, live "
+ "plugins first, plugins that failed qualification hidden with the reason one setting away.",
+ "OpenMixer is part of the default install and of the update path: the apt repository, the packages, "
+ "plugin-hostd as the mod-host of MOD-UI and the qualification sync. jack_load and jalv stay available "
+ "by one setting each.",
+ ])
+ doc.p("The reason is the cost of one process per plugin (memory, context switches, a JACK client each), plugins "
+ "that crash or stall the host they share, and no way to tell a user before loading a plugin whether it "
+ "suits live playing. Sections 3 and 4 hold the measurements; section 5 holds the way back.")
+ doc.p(facts["upstream_state"]["value"])
+
+
+def sec_user(doc, facts):
+ doc.h2("2. What the user gets")
+ doc.ul([
+ "More plugins that work: coverage of the installed LV2 set is in section 4, and CLAP plugins are "
+ "offered next to LV2 ones.",
+ "Bad plugins are hidden, with the reason: plugins pinned refused, or that crashed or hung during "
+ "qualification, leave the menu. ZYNTHIAN_PLUGIN_SHOW_HIDDEN=1 lists them last, greyed, with the reason.",
+ "Each entry carries a tag such as 'live, 1.2 ms, RT'. live means real-time safe with a measured latency "
+ "of at most %s ms at the worst measured rate; studio is the rest; unverified means no record yet. "
+ "The tag leaves out any figure the record does not hold." % facts["live_ms"]["value"],
+ "Chains in fewer processes: the members of one chain share one worker. Section 3 counts the processes "
+ "and the memory.",
+ "Controller pages come from the plugin descriptions, eight controls to a page, and MOD GUIs are generated "
+ "for plugins that ship none.",
+ "A crashed worker takes only its own plugins down; the daemon respawns it (recover time in section 3 when measured).",
+ ])
+
+
+def host_cell(s):
+ if s is None:
+ return "-"
+ if s["status"] != "ok":
+ return "%s (%s)" % (s["status"], s.get("note", ""))
+ return None
+
+
+def sec_measure(doc, facts, rpi, earlier, results_dir, ci_docs):
+ doc.h2("3. Measurements")
+ doc.p("Machine: %s. DSP load is the JACK DSP load read with jack_cpu_load; memory is resident (RSS) and "
+ "proportional (PSS) from /proc. Values are never taken from a meter on the device." % facts["device"]["value"])
+ if rpi is None:
+ doc.p("No omx-bench/1 run of class rpi5 exists yet.")
+ if ci_docs:
+ doc.p("Runs of other classes (%s) are not shown: they are not Pi 5 numbers." %
+ ", ".join(sorted(d["id"] for d in ci_docs)))
+ meta = None
+ else:
+ meta = rpi
+ m = rpi["machine"]
+ sw = ", ".join("%s %s" % (k, v) for k, v in rpi.get("software", {}).items() if v)
+ doc.p("Run %s on %s, governor %s. Software: %s." % (rpi["id"], m.get("model", "?"), m.get("governor", "?"), sw))
+ scen = rpi["scenarios"] if rpi else []
+
+ def pick(sid, host, rate, frames):
+ for s in scen:
+ if s["id"] == sid and s["host"] == host and s["rate"] == rate and s["frames"] == frames:
+ return s
+ return None
+
+ # 3a four hosts
+ doc.h3("The four hosts")
+ keys = sorted({(s["id"], s["rate"], s["frames"], s["title"]) for s in scen if s["id"] in HOST_FAMILY},
+ key=lambda k: (HOST_FAMILY.index(k[0]), k[1], k[2]))
+ if not keys:
+ doc.na("CPU, RAM and process count for H1 omx-clap-host alone, H2 plugin-hostd, H3 mod-host, H4 jalv")
+ else:
+ head = ["scenario", "rate", "frames"] + [n for _h, n in HOSTS]
+ for title, fn in (("CPU: DSP load, average (maximum), percent", lambda s: "%s (%s)" % (num(s["dsp_load_avg"], 2), num(s["dsp_load_max"], 2))),
+ ("RAM: resident memory of the host processes, MB (PSS in brackets when measured)",
+ lambda s: "%s%s" % (num(s.get("rss_mb_total", s["rss_mb"])), (" (%s)" % num(s["pss_mb_total"])) if s.get("pss_mb_total") is not None else "")),
+ ("Processes", lambda s: str(s["processes"]) if s.get("processes") is not None else "n/a")):
+ rows = []
+ for sid, rate, frames, t in keys:
+ row = [t, str(rate), str(frames)]
+ for h, _n in HOSTS:
+ s = pick(sid, h, rate, frames)
+ c = host_cell(s)
+ row.append(fn(s) if c is None else c)
+ rows.append(row)
+ doc.h3(title)
+ doc.table(head, rows)
+ # bars for the CPU of the busiest scenario family
+ items = []
+ for sid, rate, frames, t in keys:
+ for h, n in HOSTS:
+ s = pick(sid, h, rate, frames)
+ if s and s["status"] == "ok" and s["dsp_load_avg"] is not None:
+ items.append(("%s, %d Hz/%d, %s" % (sid, rate, frames, n), s["dsp_load_avg"]))
+ doc.bars("DSP load, average", items, "%")
+ if any(s["id"] == "kill1" for s in scen):
+ doc.h3("Recovery after kill -9 of the process holding one plugin")
+ doc.table(["host", "rate", "frames", "recover ms", "status", "note"],
+ [[dict(HOSTS)[s["host"]], str(s["rate"]), str(s["frames"]), num(s["recover_ms"], 0), s["status"], s.get("note", "")]
+ for s in scen if s["id"] == "kill1"])
+ else:
+ doc.na("recovery time after a killed worker")
+ # xruns and latency
+ doc.h3("Xruns and latency")
+ xr = [s for s in scen if s.get("xruns") is not None]
+ if xr:
+ doc.table(["scenario", "host", "rate", "frames", "xruns"],
+ [[s["id"], s["host"], str(s["rate"]), str(s["frames"]), str(s["xruns"])] for s in xr])
+ else:
+ doc.na("xruns per host")
+ la = [s for s in scen if s.get("latency_ms") is not None]
+ if la:
+ doc.table(["scenario", "host", "rate", "frames", "latency ms"],
+ [[s["id"], s["host"], str(s["rate"]), str(s["frames"]), num(s["latency_ms"], 2)] for s in la])
+ else:
+ doc.na("round-trip latency per host")
+
+ # 3b grouped vs processes
+ doc.h3("One worker for a chain against one process per plugin")
+ cvp = [s for s in scen if s.get("mode") and s["id"].startswith("cvp")]
+ if not cvp:
+ doc.na("grouped against individual processes (3, 8 and 16 plugins)")
+ else:
+ rows = []
+ for key in sorted({(s["id"], s["rate"], s["frames"]) for s in cvp}, key=lambda k: (int(re.sub(r"\D", "", k[0]) or 0), k[1], k[2])):
+ for s in sorted([x for x in cvp if (x["id"], x["rate"], x["frames"]) == key], key=lambda x: (x["host"], x["mode"])):
+ if s["status"] != "ok":
+ rows.append([s["title"], str(s["rate"]), s["host"] + " " + s["mode"], s["status"] + ": " + s.get("note", ""), "", "", "", "", ""])
+ continue
+ rows.append([s["title"], str(s["rate"]), s["host"] + " " + s["mode"], str(s.get("processes", "n/a")),
+ num(s.get("rss_mb_total")), num(s.get("pss_mb_total")), num(s.get("cpu_pct"), 2),
+ num(s["dsp_load_avg"], 2), num(s["add_ms"], 0)])
+ doc.table(["set", "rate", "mode", "processes", "RSS MB", "PSS MB", "CPU % of a core", "DSP load %", "add ms"], rows)
+ for key in sorted({(s["id"], s["rate"], s["frames"]) for s in cvp}):
+ ind = next((s for s in cvp if (s["id"], s["rate"], s["frames"]) == key and s["host"] == "H2" and s["mode"] == "individual" and s["status"] == "ok"), None)
+ ch = next((s for s in cvp if (s["id"], s["rate"], s["frames"]) == key and s["host"] == "H2" and s["mode"] == "chain" and s["status"] == "ok"), None)
+ if ind and ch and ind.get("processes") and ch.get("processes") and ind.get("pss_mb_total") and ch.get("pss_mb_total"):
+ doc.p("%s at %d Hz: the chain runs in %d process%s against %d, and uses %.2f times the proportional memory of the individual mode."
+ % (ind["title"], key[1], ch["processes"], "" if ch["processes"] == 1 else "es", ind["processes"],
+ ch["pss_mb_total"] / ind["pss_mb_total"]))
+ stress = [s for s in scen if s["id"].startswith("stress")]
+ doc.h3("Under stress")
+ if stress:
+ doc.table(["scenario", "host", "mode", "rate", "processes", "PSS MB", "CPU %", "DSP load %", "xruns"],
+ [[s["title"], s["host"], s.get("mode", ""), str(s["rate"]), str(s.get("processes", "n/a")),
+ num(s.get("pss_mb_total")), num(s.get("cpu_pct"), 2), num(s["dsp_load_avg"], 2), str(s.get("xruns", "n/a"))] for s in stress])
+ else:
+ doc.na("grouped against individual processes under stress")
+ gp = _read(os.path.join(results_dir, "grouped-vs-processes.md"))
+ if gp:
+ doc.p("Device notes, read as written from grouped-vs-processes.md:")
+ doc.quote(gp.strip())
+
+ # 3c Pianoteq
+ doc.h3("Pianoteq 9 at 48 and 96 kHz")
+ rows, src = [], None
+ for rate in (48000, 96000):
+ s = next((x for x in scen if x["id"] == "pianoteq" and x["rate"] == rate and x["status"] == "ok"), None)
+ if s:
+ rows.append([str(rate), str(s["frames"]), dict(HOSTS)[s["host"]], num(s["dsp_load_avg"]), num(s["dsp_load_max"]), str(s["samples"]), rpi["id"]])
+ elif earlier and earlier["runs"].get(rate, {}).get("pianoteq"):
+ e = earlier["runs"][rate]
+ q = e["pianoteq"]
+ rows.append([str(rate), str(e["frames"]), "plugin-hostd (hand run)", num(q["avg"]), num(q["max"]), str(q["n"]), q["src"]])
+ if len(rows) == 2:
+ doc.p("Five-note chords from jack_midiseq into the plugin's MIDI input, DSP load averaged over the samples.")
+ doc.table(["rate", "frames", "host", "DSP load avg %", "max %", "samples", "source"], rows)
+ else:
+ doc.na("Pianoteq 9 at both rates")
+ # hand-run chain3 when the bench doc has none
+ if earlier and not any(s["id"] == "chain3" and s["host"] == "H2" for s in scen):
+ r = [[str(rate), str(e["frames"]), num(e["chain3"]["avg"]), num(e["chain3"]["max"]), str(e["chain3"]["n"]), e["chain3"]["src"]]
+ for rate, e in sorted(earlier["runs"].items()) if e.get("chain3")]
+ if r:
+ doc.p("Hand run of the three-plugin chain (a CLAP compressor, the Calf compressor, a CLAP delay) under plugin-hostd, "
+ "governor %s:" % (earlier.get("governor") or "unrecorded"))
+ doc.table(["rate", "frames", "DSP load avg %", "max %", "samples", "source"], r)
+
+
+def sec_qual(doc, facts, rpi, qpilot, qdocs, coverage):
+ doc.h2("4. Qualification coverage")
+ counts = reasons = None
+ seconds = None
+ src = None
+ if qdocs:
+ counts, reasons, src = qdocs["counts"], qdocs["reasons"], "plugin-qualify documents (%d verdicts)" % qdocs["verdicts"]
+ elif qpilot:
+ counts, reasons, src = qpilot["counts"], None, "qualification run on the Pi 5, host profile openmixer-console (zy-qpilot.out)"
+ if qpilot:
+ seconds = qpilot["seconds"]
+ elif rpi and rpi.get("qualification", {}).get("seconds") is not None:
+ seconds = rpi["qualification"]["seconds"]
+ if not counts:
+ doc.na("verdict counts")
+ else:
+ total = sum(counts.values())
+ doc.p("Source: %s. %d plugins judged%s." % (src, total,
+ ", %s s with %s over %d CLAP files" % (num(seconds, 0), facts["qualify_command"]["value"], facts["clap_files"]["value"]) if seconds else ""))
+ doc.table(["verdict", "plugins", "share"], [[k, str(v), "%.1f%%" % (100.0 * v / total)] for k, v in sorted(counts.items(), key=lambda kv: -kv[1])])
+ hidden = counts.get("unsuitable", 0)
+ doc.p("%d of %d plugins are unsuitable (a measured failure) and are not qualified. The menus hide the plugins pinned "
+ "refused or that crashed or hung; the others are offered with their tag." % (hidden, total))
+ doc.h3("Refused, with reasons")
+ if reasons:
+ doc.table(["deciding reason", "plugins"], [[k, str(v)] for k, v in sorted(reasons.items(), key=lambda kv: -kv[1])])
+ elif qpilot and qpilot["sample_reasons"]:
+ doc.table(["deciding reason (first lines of the run's sample only)", "plugins in sample"], [[k, str(v)] for k, v in qpilot["sample_reasons"].items()])
+ doc.na("the full reason breakdown of the unsuitable plugins (needs the interchange documents from the Pi 5)")
+ else:
+ doc.na("reasons for refused plugins")
+ doc.h3("LV2 hosting coverage of the installed set")
+ if coverage:
+ doc.table(["host", "hosted", "of"], [[k, str(v[0]), str(v[1])] for k, v in coverage.items()])
+ doc.p("Measured on the desk (x86_64 Fedora 44), not on the Pi 5.")
+ else:
+ doc.na("LV2 coverage")
+
+
+def sec_risk(doc, facts):
+ doc.h2("5. Risk and how to revert")
+ doc.ul([
+ "mod-host backend: one variable, ZYNTHIAN_MODHOST_BACKEND, picks what answers MOD-UI on port %s: jack_load "
+ "(Zynthian's own mod-host, as today) or plugin-hostd. The unit keeps its name, mod-host.service, so MOD-UI "
+ "and the MOD-UI engine are untouched. sbin/set_modhost_backend.sh jack_load|plugin-hostd flips it, restarts "
+ "mod-ui and prints each step." % facts["modhost_port"]["value"],
+ "Engine: ZYNTHIAN_PLUGIN_ENGINE=jalv brings the jalv menus back. The jalv entries stay in the engine list, so a "
+ "snapshot that holds a JV processor loads as saved.",
+ "Snapshots: nothing converts at load time. tools/convert_snapshots_to_ph.py is run once by hand, writes "
+ "foo.ph.zss beside foo.zss and never overwrites.",
+ "Defaults: each install step runs once and is recorded in /zynthian/config/openmixer-defaults, so a setting "
+ "changed by hand stays. The install restarts no audio service and says what takes effect at the next start.",
+ "Unchanged: MOD-UI chains keep the MD engine, MIDI tools stay with jalv, Zynthian's own services and presets "
+ "are not touched, and without plugin-hostd installed the menus keep jalv.",
+ ])
+
+
+def sec_try(doc, facts):
+ doc.h2("6. How to try it")
+ doc.ul([
+ "Add the repository and the packages: curl -fsSLO https://www.openmixer.org/deb/openmixer-release.deb, then "
+ "sudo apt install ./openmixer-release.deb and sudo apt install plugin-hostd omx-clap-host omx-clap-scan omx-panel.",
+ "Check out the zynthian-ui branch of the PH engine in /zynthian/zynthian-ui and restart zynthian. Enable one "
+ "LV2 effect in webconf so its (PH) entry shows up, then add a chain with an LV2 and a CLAP (PH) effect.",
+ "Existing Zynthian: check out the zynthian-sys branch in /zynthian/zynthian-sys and run "
+ "scripts/update_zynthian_sys.sh; read the 'OpenMixer defaults' report; a second run reports nothing to change. "
+ "Apply with sbin/set_modhost_backend.sh plugin-hostd or reboot.",
+ "New image: run setup_zynthian.sh, then scripts/omx_prequalify_image.sh on an arm64 machine before taking the "
+ "image, so the image ships qualified. The first boot otherwise runs the qualification sync two minutes after "
+ "boot at the lowest priority.",
+ "To go back: section 5.",
+ ])
+
+
+def sec_ask(doc, facts, prs):
+ doc.h2("7. What we ask of the Zynthian project")
+ doc.p("Review and take the pull requests below, in this order. The sys pull requests come first because the "
+ "engine reads what they install; the second ui pull request is stacked on the first.")
+ if not prs:
+ doc.na("pull request states")
+ return
+ role = {(p["repo"], p["number"]): p["role"] for p in facts["prs"]}
+ doc.table(["repository", "PR", "what", "state", "base"],
+ [[p["repo"], "#%d" % p["number"], role.get((p["repo"], p["number"]), p["title"]), p["state"].lower(), p["baseRefName"]] for p in prs])
+ doc.p(facts["upstream_state"]["value"])
+
+
+def build(args):
+ with open(args.facts) as fh:
+ facts = json.load(fh)
+ docs = load_bench(args.bench)
+ rpi = latest_of_class(docs, "rpi5")
+ others = [d for d in docs if d is not rpi and d.get("machine", {}).get("class") != "rpi5"]
+ earlier = parse_earlier(args.earlier) if args.earlier else None
+ qpilot = parse_qpilot(args.earlier) if args.earlier else None
+ qdocs = parse_qdocs(args.qualification) if args.qualification and os.path.isdir(args.qualification) else None
+ coverage = parse_coverage(args.coverage) if args.coverage else None
+ prs = None
+ if args.prs and os.path.isfile(args.prs):
+ with open(args.prs) as fh:
+ prs = json.load(fh)
+ doc = Doc()
+ doc.h1("plugin-hostd, the PH engine, qualified menus and OpenMixer by default in Zynthian")
+ sec_change(doc, facts, prs)
+ sec_user(doc, facts)
+ sec_measure(doc, facts, rpi, earlier, args.results, others)
+ sec_qual(doc, facts, rpi, qpilot, qdocs, coverage)
+ sec_risk(doc, facts)
+ sec_try(doc, facts)
+ sec_ask(doc, facts, prs)
+ return doc
+
+
+# ---------------------------------------------------------------- renderers
+
+def md_cell(c):
+ return str(c).replace("|", "\\|").replace("\n", " ")
+
+
+def to_md(doc):
+ out = []
+ for b in doc.blocks:
+ k = b[0]
+ if k == "h1": out.append("# " + b[1])
+ elif k == "h2": out.append("## " + b[1])
+ elif k == "h3": out.append("### " + b[1])
+ elif k == "p": out.append(b[1])
+ elif k == "na": out.append("*" + b[1] + "*")
+ elif k == "quote": out.append("\n".join("> " + l for l in b[1].splitlines()))
+ elif k == "ul": out.append("\n".join("- " + i for i in b[1]))
+ elif k == "table":
+ out.append("| " + " | ".join(md_cell(h) for h in b[1]) + " |\n|" + "---|" * len(b[1]) + "\n" +
+ "\n".join("| " + " | ".join(md_cell(c) for c in r) + " |" for r in b[2]))
+ elif k == "bars":
+ out.append("\n".join("- %s: %s%s" % (l, num(v, 2), b[3]) for l, v in b[2]) if b[2] else "")
+ return "\n\n".join(x for x in out if x) + "\n"
+
+
+CSS = """:root{--bg:#fff;--fg:#1d2129;--mut:#5b6573;--line:#d9dee5;--bar:#2b6cb0;--bgq:#f3f5f8}
+@media(prefers-color-scheme:dark){:root{--bg:#14171c;--fg:#e6e9ee;--mut:#9aa4b2;--line:#2c333d;--bar:#6aa7e6;--bgq:#1c2128}}
+body{background:var(--bg);color:var(--fg);font:16px/1.55 system-ui,sans-serif;max-width:60rem;margin:0 auto;padding:1rem 16px 4rem}
+h1{font-size:1.6rem}h2{margin-top:2.5rem;border-bottom:1px solid var(--line);padding-bottom:.2rem}h3{margin-top:1.6rem}
+table{border-collapse:collapse;display:block;overflow-x:auto;font-size:.9rem}th,td{border:1px solid var(--line);padding:.25rem .5rem;text-align:left}
+th{background:var(--bgq)}blockquote{background:var(--bgq);margin:1rem 0;padding:.5rem 1rem;white-space:pre-wrap;border-left:3px solid var(--line)}
+.na{color:var(--mut);font-style:italic}svg text{fill:var(--fg);font-size:11px}.bar{fill:var(--bar)}"""
+
+
+def svg_bars(items):
+ if not items:
+ return ""
+ mx = max(v for _l, v in items) or 1.0
+ w, lab, h = 560, 280, 18
+ rows = []
+ for i, (l, v) in enumerate(items):
+ y = i * h
+ bw = max(1.0, (w - lab - 60) * v / mx)
+ rows.append('
%s
" % e(b[1])) + elif k == "na": out.append('%s
' % e(b[1])) + elif k == "quote": out.append("%s" % e(b[1])) + elif k == "ul": out.append("
%s (%s)
%s" % (e(b[1]), e(b[3]), svg_bars(b[2]))) + out.append("") + return "\n".join(out) + + +def main(argv=None): + here = os.path.dirname(os.path.abspath(__file__)) + home = os.path.expanduser("~/Devel/audio/zynthian-results") + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--bench", nargs="*", default=[os.path.join(home, "data")]) + ap.add_argument("--results", default=home) + ap.add_argument("--earlier", default=os.path.join(home, "data", "earlier")) + ap.add_argument("--qualification", default=os.path.join(home, "data", "qualification")) + ap.add_argument("--coverage", default=os.path.join(home, "data", "lv2-coverage-summary.txt")) + ap.add_argument("--prs", default=os.path.join(home, "data", "prs.json")) + ap.add_argument("--facts", default=os.path.join(here, "facts.json")) + ap.add_argument("--out", default=home) + ap.add_argument("--strict", action="store_true", help="exit 1 when any section lacks data") + ap.add_argument("--fetch-prs", action="store_true", help="refresh --prs with gh and exit") + a = ap.parse_args(argv) + if a.fetch_prs: + with open(a.facts) as fh: + fetch_prs(json.load(fh), a.prs) + print("wrote", a.prs) + return 0 + doc = build(a) + os.makedirs(a.out, exist_ok=True) + with open(os.path.join(a.out, "report.md"), "w") as fh: + fh.write(to_md(doc)) + with open(os.path.join(a.out, "report.html"), "w") as fh: + fh.write(to_html(doc)) + for m in doc.missing: + print("missing:", m, file=sys.stderr) + print("wrote %s/report.md and report.html; %d missing" % (a.out, len(doc.missing))) + return 1 if (a.strict and doc.missing) else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bench/report/test_make_report.py b/bench/report/test_make_report.py new file mode 100644 index 0000000..d53040d --- /dev/null +++ b/bench/report/test_make_report.py @@ -0,0 +1,129 @@ +import json +import os +import tempfile +import unittest + +import make_report as m + +FACTS = os.path.join(os.path.dirname(os.path.abspath(m.__file__)), "facts.json") + + +def scen(sid, host, rate=48000, frames=256, **kw): + d = dict(id=sid, title=sid, host=host, rate=rate, frames=frames, plugins=[], dsp_load_avg=1.5, dsp_load_max=2.0, + samples=10, rss_mb=10.0, add_ms=5.0, recover_ms=None, status="ok", note="") + d.update(kw) + return d + + +def bench(cls, scens, id="rpi5-20261027-1"): + return {"schema": "omx-bench/1", "id": id, "machine": {"class": cls, "model": "x", "governor": "performance"}, + "software": {"plugin-hostd": "0.12.0"}, "scenarios": scens} + + +class Case(unittest.TestCase): + def setUp(self): + self.t = tempfile.TemporaryDirectory() + self.d = self.t.name + self.empty = os.path.join(self.d, "empty") + os.makedirs(self.empty) + + def tearDown(self): + self.t.cleanup() + + def write(self, name, text): + p = os.path.join(self.d, name) + os.makedirs(os.path.dirname(p), exist_ok=True) + with open(p, "w") as fh: + fh.write(text if isinstance(text, str) else json.dumps(text)) + return p + + def report(self, name="report.md"): + with open(os.path.join(self.d, "o", name)) as fh: + return fh.read() + + def run_main(self, *extra, results=None): + return m.main(["--bench", os.path.join(self.d, "b"), "--results", results or self.empty, "--earlier", self.empty, + "--qualification", self.empty, "--coverage", os.path.join(self.d, "none"), + "--prs", os.path.join(self.d, "none"), "--facts", FACTS, "--out", os.path.join(self.d, "o")] + list(extra)) + + def test_strict_fails_with_no_data_and_says_not_measured(self): + self.assertEqual(self.run_main("--strict"), 1) + self.assertIn("not measured yet", self.report()) + + def test_non_strict_exits_zero(self): + self.assertEqual(self.run_main(), 0) + + def test_ci_run_is_never_shown_as_pi_numbers(self): + self.write("b/ci.json", bench("ci", [scen("chain3", "H2", dsp_load_avg=77.77)], "ci-1")) + self.run_main() + self.assertNotIn("77.77", self.report()) + + def test_host_table_and_ratio_values(self): + sc = [scen("chain3", h, dsp_load_avg=v) for h, v in (("H1", 1.11), ("H2", 2.22))] + sc.append(scen("chain3", "H3", status="unsupported", note="no mod-host", dsp_load_avg=None, + dsp_load_max=None, rss_mb=None, samples=0)) + sc += [scen("cvp3", "H2", mode="individual", processes=3, pss_mb_total=30.0, rss_mb_total=60.0, cpu_pct=1.0), + scen("cvp3", "H2", mode="chain", processes=1, pss_mb_total=15.0, rss_mb_total=30.0, cpu_pct=0.5)] + self.write("b/rpi.json", bench("rpi5", sc)) + self.run_main() + md = self.report() + self.assertIn("1.11 (2.00)", md) + self.assertIn("unsupported (no mod-host)", md) + self.assertIn("uses 0.50 times the proportional memory", md) + self.assertIn("