From b4d4d6d6886bd13eab0699aec4232c8a78708abd Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Mon, 26 Oct 2026 23:15:20 +0100 Subject: [PATCH 1/7] Bench: the omx-bench/1 schema and its validator One JSON document per run: the machine, the software, one row per scenario, host, mode, rate and frame count. validate.sh reads the schema and the rules across rows it cannot say (one row per key, a max not below its average, recover_ms only on an ok row) and names the file, the row and the rule of each problem. --- bench/lib/validate.py | 149 ++++++++++++++++++++++++++ bench/schema/omx-bench.v1.schema.json | 109 +++++++++++++++++++ bench/validate.sh | 11 ++ 3 files changed, 269 insertions(+) create mode 100644 bench/lib/validate.py create mode 100644 bench/schema/omx-bench.v1.schema.json create mode 100755 bench/validate.sh diff --git a/bench/lib/validate.py b/bench/lib/validate.py new file mode 100644 index 0000000..73689ec --- /dev/null +++ b/bench/lib/validate.py @@ -0,0 +1,149 @@ +"""Check omx-bench/1 documents: the JSON Schema of bench/schema, read here with the keywords it uses, and the rules +across rows the schema cannot say. One line per problem: file, scenario row, rule, what is wrong. Exit 1 on any.""" + +import json +import re +import sys + + +def type_ok(value, name): + if name == "null": + return value is None + if name == "boolean": + return isinstance(value, bool) + if name == "integer": + return isinstance(value, int) and not isinstance(value, bool) + if name == "number": + return isinstance(value, (int, float)) and not isinstance(value, bool) + if name == "string": + return isinstance(value, str) + if name == "array": + return isinstance(value, list) + if name == "object": + return isinstance(value, dict) + raise ValueError("schema uses an unknown type: " + name) + + +KNOWN = {"$schema", "$id", "title", "type", "const", "enum", "required", "properties", "additionalProperties", "items", + "minItems", "minimum", "minLength", "pattern", "allOf", "if", "then"} + + +def check(value, schema, path, errors): + """append (path, rule, message) for each way value breaks schema; path is a list of keys""" + unknown = set(schema) - KNOWN + if unknown: + raise ValueError("schema uses keywords this validator does not read: " + ", ".join(sorted(unknown))) + rule = "/".join(str(p) for p in path) or "(document)" + if "type" in schema: + names = schema["type"] if isinstance(schema["type"], list) else [schema["type"]] + if not any(type_ok(value, n) for n in names): + errors.append((path, rule + ":type", "is %s, not %s" % (json.dumps(value), " or ".join(names)))) + return + if "const" in schema and value != schema["const"]: + errors.append((path, rule + ":const", "is %s, not %s" % (json.dumps(value), json.dumps(schema["const"])))) + if "enum" in schema and value not in schema["enum"]: + errors.append((path, rule + ":enum", "is %s, not one of %s" % (json.dumps(value), json.dumps(schema["enum"])))) + if isinstance(value, (int, float)) and not isinstance(value, bool) and "minimum" in schema: + if value < schema["minimum"]: + errors.append((path, rule + ":minimum", "is %s, below %s" % (value, schema["minimum"]))) + if isinstance(value, str): + if len(value) < schema.get("minLength", 0): + errors.append((path, rule + ":minLength", "is shorter than %d" % schema["minLength"])) + if "pattern" in schema and not re.search(schema["pattern"], value): + errors.append((path, rule + ":pattern", "%s does not match %s" % (json.dumps(value), schema["pattern"]))) + if isinstance(value, list): + if len(value) < schema.get("minItems", 0): + errors.append((path, rule + ":minItems", "has %d items, fewer than %d" % (len(value), schema["minItems"]))) + if "items" in schema: + for n, item in enumerate(value): + check(item, schema["items"], path + [n], errors) + if isinstance(value, dict): + for key in schema.get("required", []): + if key not in value: + errors.append((path, rule + ":required", "lacks %s" % key)) + props = schema.get("properties", {}) + extra = schema.get("additionalProperties", True) + for key, item in value.items(): + if key in props: + check(item, props[key], path + [key], errors) + elif extra is False: + errors.append((path, rule + ":additionalProperties", "has %s, which the schema does not declare" % key)) + elif isinstance(extra, dict): + check(item, extra, path + [key], errors) + for sub in schema.get("allOf", []): + check(value, sub, path, errors) + if "if" in schema: + probe = [] + check(value, schema["if"], path, probe) + if not probe and "then" in schema: + check(value, schema["then"], path, errors) + + +def row_name(doc, path): + if len(path) >= 2 and path[0] == "scenarios" and isinstance(path[1], int): + try: + s = doc["scenarios"][path[1]] + return "scenario %s %s %s/%s%s" % (s.get("id"), s.get("host"), s.get("rate"), s.get("frames"), + " " + s["mode"] if "mode" in s else "") + except (KeyError, IndexError, TypeError, AttributeError): + return "scenario #%d" % path[1] + return "document" + + +def cross_rules(doc, errors): + """what one row cannot say about itself""" + seen = {} + for n, s in enumerate(doc.get("scenarios", [])): + if not isinstance(s, dict): + continue + key = (s.get("id"), s.get("host"), s.get("mode"), s.get("rate"), s.get("frames")) + if key in seen: + errors.append((["scenarios", n], "rule:unique-row", "repeats row #%d (same id, host, mode, rate, frames)" + % seen[key])) + seen[key] = n + avg, mx = s.get("dsp_load_avg"), s.get("dsp_load_max") + if isinstance(avg, (int, float)) and isinstance(mx, (int, float)) and mx < avg: + errors.append((["scenarios", n], "rule:max-not-below-avg", "dsp_load_max %s is below dsp_load_avg %s" + % (mx, avg))) + if s.get("recover_ms") is not None and s.get("status") != "ok": + errors.append((["scenarios", n], "rule:recover-only-ok", "has recover_ms on a row that is not ok")) + if s.get("status") == "ok" and s.get("plugins") and s.get("add_ms") is None: + errors.append((["scenarios", n], "rule:add-measured", "is ok with plugins and no add_ms")) + if "rss_mb_total" in s and "processes" not in s: + errors.append((["scenarios", n], "rule:totals-name-processes", "has totals and no processes count")) + m = re.match(r"^([a-z0-9]+)-", str(doc.get("id", ""))) + cls = doc.get("machine", {}).get("class") if isinstance(doc.get("machine"), dict) else None + if m and cls and m.group(1) != cls: + errors.append(([], "rule:id-class", "id %s does not start with the machine class %s" % (doc["id"], cls))) + q = doc.get("qualification") + if isinstance(q, dict) and all(isinstance(q.get(k), int) for k in ("plugins", "suitable", "unsuitable")): + if q["suitable"] + q["unsuitable"] != q["plugins"]: + errors.append((["qualification"], "rule:qualification-sum", "suitable + unsuitable is not plugins")) + + +def main(): + schema = json.load(open(sys.argv[1])) + bad = 0 + for name in sys.argv[2:]: + try: + with open(name) as f: + doc = json.load(f) + except (OSError, ValueError) as e: + print("%s: document: rule:json: %s" % (name, e)) + bad += 1 + continue + errors = [] + check(doc, schema, [], errors) + if isinstance(doc, dict): + cross_rules(doc, errors) + for path, rule, msg in errors: + print("%s: %s: %s: %s" % (name, row_name(doc, path), rule, msg)) + if errors: + bad += 1 + else: + print("%s: valid omx-bench/1, %d scenario rows" % (name, len(doc["scenarios"]))) + sys.exit(1 if bad else 0) + + +if __name__ == "__main__": + main() diff --git a/bench/schema/omx-bench.v1.schema.json b/bench/schema/omx-bench.v1.schema.json new file mode 100644 index 0000000..4a980b5 --- /dev/null +++ b/bench/schema/omx-bench.v1.schema.json @@ -0,0 +1,109 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://www.openmixer.org/schema/omx-bench.v1.schema.json", + "title": "omx-bench/1: one benchmark run of plugin hosts on one machine", + "type": "object", + "required": ["schema", "id", "machine", "software", "scenarios"], + "additionalProperties": false, + "properties": { + "schema": { "const": "omx-bench/1" }, + "id": { "type": "string", "pattern": "^(rpi5|desk|ci)-[0-9]{8}-[0-9]+$" }, + "machine": { + "type": "object", + "required": ["class", "model", "cpu", "cores", "ram_mb", "os", "kernel", "governor"], + "additionalProperties": false, + "properties": { + "class": { "enum": ["rpi5", "desk", "ci"] }, + "model": { "type": "string", "minLength": 1 }, + "cpu": { "type": "string", "minLength": 1 }, + "cores": { "type": "integer", "minimum": 1 }, + "ram_mb": { "type": "integer", "minimum": 1 }, + "os": { "type": "string", "minLength": 1 }, + "kernel": { "type": "string", "minLength": 1 }, + "governor": { "type": "string", "minLength": 1 } + } + }, + "software": { + "type": "object", + "required": ["plugin-hostd", "omx-clap-host", "mod-host", "jalv", "jack"], + "additionalProperties": { "type": ["string", "null"] }, + "properties": { + "plugin-hostd": { "type": ["string", "null"] }, + "omx-clap-host": { "type": ["string", "null"] }, + "mod-host": { "type": ["string", "null"] }, + "jalv": { "type": ["string", "null"] }, + "jack": { "type": ["string", "null"] } + } + }, + "scenarios": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["id", "title", "host", "rate", "frames", "plugins", "dsp_load_avg", "dsp_load_max", "samples", + "rss_mb", "add_ms", "recover_ms", "status", "note"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "pattern": "^[a-z0-9-]+$" }, + "title": { "type": "string", "minLength": 1 }, + "host": { "enum": ["H1", "H2", "H3", "H4"] }, + "rate": { "type": "integer", "minimum": 8000 }, + "frames": { "type": "integer", "minimum": 16 }, + "plugins": { "type": "array", "items": { "type": "string", "minLength": 1 } }, + "dsp_load_avg": { "type": ["number", "null"], "minimum": 0 }, + "dsp_load_max": { "type": ["number", "null"], "minimum": 0 }, + "samples": { "type": "integer", "minimum": 0 }, + "rss_mb": { "type": ["number", "null"], "minimum": 0 }, + "add_ms": { "type": ["number", "null"], "minimum": 0 }, + "recover_ms": { "type": ["number", "null"], "minimum": 0 }, + "status": { "enum": ["ok", "unsupported", "failed"] }, + "note": { "type": "string" }, + "mode": { "enum": ["individual", "chain", "shared-process"] }, + "processes": { "type": "integer", "minimum": 0 }, + "rss_mb_total": { "type": "number", "minimum": 0 }, + "pss_mb_total": { "type": "number", "minimum": 0 }, + "cpu_pct": { "type": "number", "minimum": 0 }, + "ctx_switches": { "type": "integer", "minimum": 0 }, + "ctx_involuntary": { "type": "integer", "minimum": 0 } + }, + "allOf": [ + { + "if": { "properties": { "status": { "const": "ok" } } }, + "then": { + "properties": { + "dsp_load_avg": { "type": "number" }, + "dsp_load_max": { "type": "number" }, + "samples": { "minimum": 1 }, + "rss_mb": { "type": "number" } + } + } + }, + { + "if": { "properties": { "status": { "enum": ["unsupported", "failed"] } } }, + "then": { "properties": { "note": { "minLength": 1 } } } + } + ] + } + }, + "coverage": { + "type": "object", + "required": ["set"], + "additionalProperties": { "type": "integer", "minimum": 0 }, + "properties": { "set": { "type": "string", "minLength": 1 } } + }, + "qualification": { + "type": "object", + "required": ["plugins", "seconds", "rates", "quanta", "suitable", "unsuitable"], + "additionalProperties": false, + "properties": { + "plugins": { "type": "integer", "minimum": 0 }, + "files": { "type": "integer", "minimum": 0 }, + "seconds": { "type": "number", "minimum": 0 }, + "rates": { "type": "array", "minItems": 1, "items": { "type": "integer" } }, + "quanta": { "type": "array", "minItems": 1, "items": { "type": "integer" } }, + "suitable": { "type": "integer", "minimum": 0 }, + "unsuitable": { "type": "integer", "minimum": 0 } + } + } + } +} diff --git a/bench/validate.sh b/bench/validate.sh new file mode 100755 index 0000000..dafb3e5 --- /dev/null +++ b/bench/validate.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# validate.sh ...: check omx-bench/1 documents against bench/schema/omx-bench.v1.schema.json and the rules the +# schema cannot say. Exit 1 names the file, the scenario row and the rule of the first problem of each file. +set -euo pipefail + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +if [ $# -lt 1 ]; then + echo "usage: validate.sh ..." >&2 + exit 2 +fi +exec python3 -I "$here/lib/validate.py" "$here/schema/omx-bench.v1.schema.json" "$@" From 693816b9b199f7c29c997994bd8b16fc3c77285f Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Mon, 26 Oct 2026 23:15:30 +0100 Subject: [PATCH 2/7] Bench: the runner and the scenario table run.sh starts a private JACK server per rate and frame count (jackd -n bench -d dummy), each host fresh per row on a port of its own (H1 omx-clap-host, H2 plugin-hostd with require_pins 0, H3 mod-host, H4 one jalv per plugin), wires a sine or jack_midiseq's chords through the plugins in series, and reads jack_cpu_load, RSS, PSS, CPU time and context switches of the host's processes over the window, the add time, and for H2 the time from kill -9 of the worker until its client is back and wired. One controller connection per host is held for the row: the daemon sees a worker's death while a controller is connected. The chains-vs-processes rows run the same N plugins as N workers, one chain, one shared process and N jalv processes. Every process carries the run's mark; the teardown stops them, puts the governor back, and the run exits 3 if a marked process is left. send.sh sends protocol lines by hand. --- bench/lib/bench.py | 992 +++++++++++++++++++++++++++++++++++++++++++ bench/lib/send.sh | 20 + bench/run.sh | 7 + bench/scenarios.json | 33 ++ 4 files changed, 1052 insertions(+) create mode 100644 bench/lib/bench.py create mode 100755 bench/lib/send.sh create mode 100755 bench/run.sh create mode 100644 bench/scenarios.json diff --git a/bench/lib/bench.py b/bench/lib/bench.py new file mode 100644 index 0000000..3d41fe3 --- /dev/null +++ b/bench/lib/bench.py @@ -0,0 +1,992 @@ +"""The benchmark runner behind bench/run.sh: four plugin hosts, one private JACK server per rate and frame count, +one omx-bench/1 document per run. bench/README.md says what each scenario measures.""" + +import argparse +import ctypes +import datetime +import glob +import json +import os +import re +import shutil +import signal +import socket +import subprocess +import sys +import tempfile +import time + +HERE = os.path.dirname(os.path.abspath(__file__)) +BENCH = os.path.dirname(HERE) +CLK_TCK = os.sysconf("SC_CLK_TCK") +PAGE_KB = os.sysconf("SC_PAGE_SIZE") // 1024 +HOSTS = ("H1", "H2", "H3", "H4") +HOST_PROGRAM = {"H1": "omx-clap-host", "H2": "plugin-hostd", "H3": "mod-host", "H4": "jalv"} +MARK = "OMX_BENCH_RUN" +TAG = "omx-bench-%d" % os.getpid() +RECOVER_LIMIT_S = 30.0 +READY_LIMIT_S = 20.0 + + +def log(msg): + print("bench: " + msg, file=sys.stderr, flush=True) + + +class Refused(Exception): + """a row that cannot run here: status unsupported, the message is the note""" + + +class Failed(Exception): + """a row that ran and broke: status failed, the message is the note""" + + +# ---------------------------------------------------------------- processes + + +class Procs: + """every process the run starts: each in a session of its own, each marked with MARK so the last check finds + any that outlives the teardown, whoever its parent is by then""" + + def __init__(self, logdir, server): + self.logdir = logdir + self.server = server + self.live = [] + self.env = dict(os.environ) + self.env[MARK] = TAG + self.env["JACK_DEFAULT_SERVER"] = server + self.env["JACK_NO_AUDIO_RESERVATION"] = "1" + + def spawn(self, argv, name, env=None, stdin=subprocess.DEVNULL): + e = dict(self.env) + e.update(env or {}) + out = open(os.path.join(self.logdir, name + ".log"), "ab") + p = subprocess.Popen(argv, stdin=stdin, stdout=out, stderr=subprocess.STDOUT, env=e, start_new_session=True) + out.close() + p.name = name + self.live.append(p) + return p + + def stop(self, p, grace=3.0): + """SIGTERM to the process group and every descendant, SIGKILL to what is left after grace""" + if p not in self.live: + return + self.live.remove(p) + group = [p.pid] + descendants(p.pid) + signal_all(p.pid, group, signal.SIGTERM) + end = time.monotonic() + grace + while time.monotonic() < end and any(alive(q) for q in group): + reap() + time.sleep(0.02) + signal_all(p.pid, group, signal.SIGKILL) + try: + p.wait(timeout=5) + except subprocess.TimeoutExpired: + pass + reap() + + def stop_all(self): + for p in list(reversed(self.live)): + self.stop(p) + + +def signal_all(pgid, pids, sig): + try: + os.killpg(pgid, sig) + except (ProcessLookupError, PermissionError): + pass + for q in pids: + try: + os.kill(q, sig) + except (ProcessLookupError, PermissionError): + pass + + +def alive(pid): + """running and not a zombie""" + try: + with open("/proc/%d/stat" % pid) as f: + return f.read().rsplit(")", 1)[1].split()[0] != "Z" + except (OSError, IndexError): + return False + + +def reap(): + """collect exited children, orphans included: the runner is the subreaper of everything it starts""" + while True: + try: + pid, _ = os.waitpid(-1, os.WNOHANG) + except ChildProcessError: + return + if pid == 0: + return + + +def ppid_map(): + m = {} + for d in os.listdir("/proc"): + if d.isdigit(): + try: + with open("/proc/%s/stat" % d) as f: + m[int(d)] = int(f.read().rsplit(")", 1)[1].split()[1]) + except (OSError, IndexError, ValueError): + pass + return m + + +def descendants(pid): + m = ppid_map() + out, todo = [], [pid] + while todo: + cur = todo.pop() + kids = [c for c, pp in m.items() if pp == cur] + out += kids + todo += kids + return out + + +def marked(): + """(pid, command) of every live process carrying this run's mark""" + found = [] + needle = ("%s=%s" % (MARK, TAG)).encode() + for d in os.listdir("/proc"): + if not d.isdigit() or int(d) == os.getpid(): + continue + try: + with open("/proc/%s/environ" % d, "rb") as f: + if needle not in f.read().split(b"\0"): + continue + if not alive(int(d)): + continue + with open("/proc/%s/comm" % d) as f: + found.append((int(d), f.read().strip())) + except OSError: + pass + return found + + +# ---------------------------------------------------------------- /proc measurements + + +def proc_sample(pid): + """utime+stime ticks, RSS kB, PSS kB, voluntary and involuntary context switches of one process""" + with open("/proc/%d/stat" % pid) as f: + fields = f.read().rsplit(")", 1)[1].split() + ticks = int(fields[11]) + int(fields[12]) + rss = int(fields[21]) * PAGE_KB + pss = 0 + try: + with open("/proc/%d/smaps_rollup" % pid) as f: + for line in f: + if line.startswith("Pss:"): + pss = int(line.split()[1]) + except OSError: + pss = None + # a process's status counts its main thread only: the switches are summed over its threads + vol = invol = 0 + for status in glob.glob("/proc/%d/task/*/status" % pid): + try: + with open(status) as f: + for line in f: + if line.startswith("voluntary_ctxt_switches:"): + vol += int(line.split()[1]) + elif line.startswith("nonvoluntary_ctxt_switches:"): + invol += int(line.split()[1]) + except OSError: + pass + return {"ticks": ticks, "rss": rss, "pss": pss, "vol": vol, "invol": invol} + + +def samples(pids): + out = {} + for pid in pids: + try: + out[pid] = proc_sample(pid) + except (OSError, IndexError, ValueError): + pass + return out + + +# ---------------------------------------------------------------- JACK + + +def run(argv, env, timeout=10): + return subprocess.run(argv, env=env, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, + timeout=timeout).stdout + + +def ports(env): + """{client: {"audio_in": [...], "audio_out": [...], "midi_in": [...]}} from jack_lsp, in port order""" + out, port = {}, None + props = "" + for line in run(["jack_lsp", "-p", "-t"], env).splitlines(): + if not line.startswith((" ", "\t")): + port, props = line.strip(), "" + continue + text = line.strip() + if text.startswith("properties:"): + props = text + continue + if port is None: + continue + kind = "audio" if "audio" in text else "midi" if "midi" in text else None + if kind: + direction = "in" if "input" in props else "out" + client = port.split(":", 1)[0] + out.setdefault(client, {}).setdefault("%s_%s" % (kind, direction), []).append(port) + return out + + +def connected(env, a, b): + return b in run(["jack_lsp", "-c", a], env).split() + + +# ---------------------------------------------------------------- the protocol + + +class Control: + """one controller connection to a mod-host protocol port, held for the life of the host, as a controller holds + it: NUL-terminated lines, one reply each (the same framing as lib/send.sh)""" + + def __init__(self, port, proc, limit_s=READY_LIMIT_S): + end = time.monotonic() + limit_s + while True: + try: + self.sock = socket.create_connection(("127.0.0.1", port), timeout=2) + break + except OSError: + if proc.poll() is not None: + raise Failed("the host exited at start (exit %s)" % proc.returncode) + if time.monotonic() > end: + raise Failed("no host on port %d after %.0f s" % (port, limit_s)) + time.sleep(0.05) + self.sock.settimeout(60) + + def cmd(self, line): + self.sock.sendall(line.encode() + b"\0") + buf = b"" + while not buf.endswith(b"\0"): + chunk = self.sock.recv(65536) + if not chunk: + raise Failed("the host closed the connection on: " + line) + buf += chunk + return buf[:-1].decode(errors="replace").strip() + + def close(self): + try: + self.sock.close() + except OSError: + pass + + +def free_port(start): + p = start + while True: + s = socket.socket() + try: + s.bind(("127.0.0.1", p)) + s.close() + # a daemon may take p+1 as its feedback port + s = socket.socket() + s.bind(("127.0.0.1", p + 1)) + s.close() + return p + except OSError: + s.close() + p += 2 + + +# ---------------------------------------------------------------- the machine and the software + + +def read(path, default=""): + try: + with open(path, "rb") as f: + return f.read().replace(b"\0", b"").decode(errors="replace").strip() + except OSError: + return default + + +def governors(): + return sorted(glob.glob("/sys/devices/system/cpu/cpu[0-9]*/cpufreq/scaling_governor")) + + +def governor_now(): + vals = sorted({read(p) for p in governors()}) + if not vals: + return "none" + return vals[0] if len(vals) == 1 else "mixed:" + ",".join(vals) + + +def write_sys(path, value): + try: + with open(path, "w") as f: + f.write(value) + return True + except OSError: + pass + if shutil.which("sudo"): + r = subprocess.run(["sudo", "-n", "tee", path], input=value, text=True, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + return r.returncode == 0 + return False + + +def machine(cls): + model = read("/proc/device-tree/model") or " ".join( + x for x in (read("/sys/class/dmi/id/sys_vendor"), read("/sys/class/dmi/id/product_name")) if x) or "unknown" + cpu = "" + for line in read("/proc/cpuinfo").splitlines(): + if line.lower().startswith("model name"): + cpu = line.split(":", 1)[1].strip() + break + if not cpu and shutil.which("lscpu"): + for line in subprocess.run(["lscpu"], stdout=subprocess.PIPE, text=True).stdout.splitlines(): + if line.startswith("Model name:"): + cpu = line.split(":", 1)[1].strip() + break + mem = 0 + for line in read("/proc/meminfo").splitlines(): + if line.startswith("MemTotal:"): + mem = int(line.split()[1]) // 1024 + rel = {} + for line in read("/etc/os-release").splitlines(): + if "=" in line: + k, v = line.split("=", 1) + rel[k] = v.strip('"') + os_name = ("%s %s" % (rel.get("NAME", "Linux").split()[0], rel.get("VERSION_ID", ""))).strip() + return {"class": cls, "model": model, "cpu": cpu or "unknown", "cores": os.cpu_count() or 1, "ram_mb": mem, + "os": os_name, "kernel": os.uname().release, "governor": governor_now()} + + +def package_of(path): + """(package, version) of the installed package that owns path, from dpkg or rpm""" + real = os.path.realpath(path) + for cand in dict.fromkeys([path, real]): + if shutil.which("dpkg-query"): + r = subprocess.run(["dpkg-query", "-S", cand], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) + if r.returncode == 0 and ":" in r.stdout: + pkg = r.stdout.split(":", 1)[0].split(",")[0].strip() + v = subprocess.run(["dpkg-query", "-W", "-f", "${Version}", pkg], stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, text=True).stdout.strip() + return pkg, v or None + if shutil.which("rpm"): + r = subprocess.run(["rpm", "-qf", "--qf", "%{NAME} %{VERSION}-%{RELEASE}\n", cand], stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, text=True) + if r.returncode == 0 and r.stdout.strip(): + pkg, v = r.stdout.splitlines()[0].split(" ", 1) + return pkg, v + return None, None + + +def program_version(name): + path = shutil.which(name) + if not path: + return None + _, v = package_of(path) + if v: + return v + for flag in ("-V", "--version"): + try: + out = subprocess.run([path, flag], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + timeout=5).stdout + except (OSError, subprocess.TimeoutExpired): + continue + m = re.search(r"\d+\.\d+[\w.+~-]*", out) + if m: + return m.group(0) + " (not packaged)" + return "not packaged" + + +def software(plugin_files): + sw = {"plugin-hostd": program_version("plugin-hostd"), "omx-clap-host": program_version("omx-clap-host"), + "mod-host": program_version("mod-host"), "jalv": program_version("jalv"), "jack": program_version("jackd")} + for f in plugin_files: + pkg, v = package_of(f) + if pkg and pkg not in sw: + sw[pkg] = v + return sw + + +# ---------------------------------------------------------------- plugins + + +def clap_dirs(): + dirs = [d for d in os.environ.get("CLAP_PATH", "").split(":") if d] + return dirs + [os.path.expanduser("~/.clap"), "/usr/lib/clap", "/usr/lib64/clap", "/usr/local/lib/clap"] + + +def lv2_dirs(): + dirs = [d for d in os.environ.get("LV2_PATH", "").split(":") if d] + return dirs + [os.path.expanduser("~/.lv2"), "/usr/lib/lv2", "/usr/lib64/lv2", "/usr/local/lib/lv2", + "/usr/lib/%s-linux-gnu/lv2" % os.uname().machine] + + +def clap_uri(spec): + fname, pid = spec.split("#", 1) + for d in clap_dirs(): + if os.path.exists(os.path.join(d, fname)): + return "clap:%s#%s" % (os.path.join(d, fname), pid), os.path.join(d, fname) + return None, None + + +class Catalog: + def __init__(self, table, pianoteq_dir): + self.plugins = table["plugins"] + self.midi = table["midi"] + self.pianoteq_dir = pianoteq_dir + + def name(self, key): + return self.plugins[key]["name"] + + def needs(self, key): + if self.plugins[key].get("needs") == "pianoteq" and not self.pianoteq_dir: + raise Refused("Pianoteq is not on this machine (no --pianoteq-dir)") + + def for_host(self, key, host, fmt=None): + """(uri the host takes, label for the document, note) for one plugin on one host""" + self.needs(key) + p = self.plugins[key] + if host in ("H1", "H2"): + if fmt != "lv2" and "clap" in p: + uri, _ = clap_uri(p["clap"]) + if not uri: + raise Refused("%s is not installed as CLAP" % p["name"]) + return uri, "%s (CLAP)" % p["name"], None + if fmt == "clap": + raise Refused("%s has no CLAP build" % p["name"]) + return "lv2:" + p["lv2"], "%s (LV2)" % p["name"], None + if fmt == "clap": + raise Refused("%s runs LV2 only" % HOST_PROGRAM[host]) + if "lv2" not in p: + raise Refused("%s has no LV2 build" % p["name"]) + return p["lv2"], "%s (LV2)" % p["name"], p["name"] if "clap" in p else None + + def files(self): + out = [] + for p in self.plugins.values(): + if "clap" in p: + _, f = clap_uri(p["clap"]) + if f: + out.append(f) + for d in lv2_dirs(): + for b in ("omx-comp.lv2", "calf.lv2"): + m = os.path.join(d, b, "manifest.ttl") + if os.path.exists(m): + out.append(m) + return list(dict.fromkeys(out)) + + +# ---------------------------------------------------------------- hosts + + +class Host: + """one host program started for one scenario row""" + + def __init__(self, key, procs, env, workdir, port): + self.key, self.procs, self.env, self.workdir, self.port = key, procs, env, workdir, port + self.proc = None + self.ctl = None + self.jalvs = {} + + def start(self): + if self.key == "H1": + self.proc = self.procs.spawn(["omx-clap-host", "-n", "-p", str(self.port)], "H1", self.env) + elif self.key == "H2": + conf = os.path.join(self.workdir, "plugin-hostd.conf") + with open(conf, "w") as f: + f.write("require_pins 0\n") + self.proc = self.procs.spawn(["plugin-hostd", "-n", "-p", str(self.port), "-c", conf], "H2", self.env) + elif self.key == "H3": + self.proc = self.procs.spawn(["mod-host", "-n", "-p", str(self.port)], "H3", self.env) + if self.key != "H4": + self.ctl = Control(self.port, self.proc) + + def roots(self): + if self.key == "H4": + return [p.pid for p in self.jalvs.values() if p.poll() is None] + return [self.proc.pid] if self.proc and self.proc.poll() is None else [] + + def pids(self): + """the processes the host runs: its own and every descendant, the workers of plugin-hostd included""" + out = [] + for r in self.roots(): + out += [r] + descendants(r) + return [p for p in dict.fromkeys(out) if alive(p)] + + def workers(self): + """plugin-hostd's worker_list: [(worker, pid, format, state, place, [instances])]""" + reply = self.ctl.cmd("worker_list").split() + out = [] + for rec in reply[2:]: + f = rec.split(":") + insts = [int(i) for i in f[-1].split(",") if i.isdigit()] + out.append((f[0], int(f[1]), f[2], f[3], ":".join(f[4:-1]), insts)) + return out + + def add(self, uri, inst, placement=None): + """add one plugin; returns the client name""" + client = "effect_%d" % inst + if self.key == "H4": + argv = ["jalv", "-i", "-n", client, uri] + self.jalvs[inst] = self.procs.spawn(argv, "H4-%d" % inst, self.env) + end = time.monotonic() + READY_LIMIT_S + while client not in ports(self.env): + if self.jalvs[inst].poll() is not None: + raise Failed("jalv exited at start for %s" % uri) + if time.monotonic() > end: + raise Failed("jalv shows no JACK client for %s after %.0f s" % (uri, READY_LIMIT_S)) + time.sleep(0.01) + return client + line = "add %s %d" % (uri, inst) + (" " + placement if placement else "") + reply = self.ctl.cmd(line) + if reply != "resp %d" % inst: + raise Failed("%s answered '%s' to: %s" % (HOST_PROGRAM[self.key], reply, line)) + return client + + def connect(self, a, b): + if self.key == "H4": + subprocess.run(["jack_connect", a, b], env=self.env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + return + reply = self.ctl.cmd("connect %s %s" % (a, b)) + if reply != "resp 0": + raise Failed("connect %s %s answered '%s'" % (a, b, reply)) + + def stop(self): + if self.ctl: + self.ctl.close() + for p in list(self.jalvs.values()): + self.procs.stop(p) + if self.proc: + self.procs.stop(self.proc) + + +def wire(host, env, clients, instrument): + """sine (or the chords) into the first client, each client into the next, the last into the playback ports""" + graph = ports(env) + if instrument: + host.connect("seq:out", graph[clients[0]]["midi_in"][0]) + src = [] + else: + src = graph["sine"]["audio_out"] + for c in clients: + ins = graph.get(c, {}).get("audio_in", []) + for k, port in enumerate(ins): + if src: + host.connect(src[min(k, len(src) - 1)], port) + src = graph.get(c, {}).get("audio_out", []) + if not src: + raise Failed("%s has no audio output" % c) + for k, port in enumerate(graph["system"]["audio_in"][:2]): + host.connect(src[min(k, len(src) - 1)], port) + return src + + +# ---------------------------------------------------------------- the runner + + +class Runner: + def __init__(self, args, table): + self.args = args + self.table = table + self.cat = Catalog(table, args.pianoteq_dir) + self.work = tempfile.mkdtemp(prefix="omx-bench-") + self.logdir = args.logs or os.path.join(self.work, "logs") + os.makedirs(self.logdir, exist_ok=True) + self.server = "bench%d" % os.getpid() + self.procs = Procs(self.logdir, self.server) + self.env = dict(self.procs.env) + self.env["XDG_STATE_HOME"] = os.path.join(self.work, "state") + self.env["XDG_RUNTIME_DIR"] = self.env.get("XDG_RUNTIME_DIR") or os.path.join(self.work, "run") + os.makedirs(self.env["XDG_RUNTIME_DIR"], mode=0o700, exist_ok=True) + if args.pianoteq_dir: + self.env["LV2_PATH"] = ":".join([args.pianoteq_dir] + [d for d in lv2_dirs() if os.path.isdir(d)]) + self.saved_governor = {} + self.jackd = None + self.port = 5700 + + # -- governor + + def governor_on(self): + if self.args.keep_governor: + return + paths = governors() + for p in paths: + self.saved_governor[p] = read(p) + ok = paths and all(write_sys(p, "performance") for p in paths) + if not ok: + self.governor_off() + msg = "cannot set the CPU governor to performance (run as root, or with sudo, or pass --keep-governor)" + if self.args.cls != "ci": + raise SystemExit("bench: " + msg) + log(msg + "; class ci goes on with the governor it has: " + governor_now()) + + def governor_off(self): + for p, v in self.saved_governor.items(): + if v and read(p) != v: + write_sys(p, v) + self.saved_governor = {} + + # -- JACK + + def jack_start(self, rate, frames): + self.jack_stop() + self.jackd = self.procs.spawn(["jackd", "-n", self.server, "-d", "dummy", "-r", str(rate), "-p", str(frames)], + "jackd-%d-%d" % (rate, frames)) + end = time.monotonic() + READY_LIMIT_S + while "system" not in ports(self.env): + if self.jackd.poll() is not None or time.monotonic() > end: + raise SystemExit("bench: the private JACK server did not start at %d Hz / %d frames (see %s)" + % (rate, frames, self.logdir)) + time.sleep(0.1) + + def jack_stop(self): + if self.jackd: + self.procs.stop(self.jackd) + self.jackd = None + + def dsp_window(self, seconds): + """jack_cpu_load's one line a second for seconds+1 s; the first is the warm-up and is dropped""" + p = subprocess.Popen(["jack_cpu_load"], env=self.env, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + text=True, start_new_session=True) + time.sleep(seconds + 1.3) + p.send_signal(signal.SIGINT) + try: + out, _ = p.communicate(timeout=5) + except subprocess.TimeoutExpired: + p.kill() + out, _ = p.communicate() + vals = [float(m) for m in re.findall(r"DSP load\s+([0-9.]+)", out)] + return vals[1:seconds + 1] + + # -- rows + + def plan(self): + hosts = self.args.hosts + rows = [] + for s in self.table["scenarios"]: + if self.args.scenarios and s["id"] not in self.args.scenarios: + continue + if s.get("family"): + for m in self.table["modes"][s["family"]]: + if m["host"] in hosts: + rows.append((s, m["host"], m["mode"])) + else: + for h in hosts: + rows.append((s, h, None)) + return rows + + def plugin_list(self, s): + keys = s["plugins"] + if s.get("count"): + keys = [keys[n % len(keys)] for n in range(s["count"])] + elif s.get("repeat"): + keys = keys * s["repeat"] + return keys + + def row(self, s, hkey, mode, rate, frames): + r = {"id": s["id"], "title": s["title"], "host": hkey, "rate": rate, "frames": frames, "plugins": [], + "dsp_load_avg": None, "dsp_load_max": None, "samples": 0, "rss_mb": None, "add_ms": None, + "recover_ms": None, "status": "failed", "note": ""} + if mode: + r["mode"] = mode + host = None + sources = [] + try: + keys = self.plugin_list(s) + if hkey == "H4" and not keys: + raise Refused("jalv runs one plugin per process; it has no host without a plugin") + if s.get("chain") and hkey == "H2": + mode = "chain" + picks = [self.cat.for_host(k, hkey, s.get("format")) for k in keys] + r["plugins"] = [label for _, label, _ in picks] + if not shutil.which(HOST_PROGRAM[hkey]): + raise Refused("%s is not installed" % HOST_PROGRAM[hkey]) + builds = list(dict.fromkeys(n for _, _, n in picks if n)) + notes = ["LV2 builds of " + ", ".join(builds)] if builds else [] + instrument = any(self.cat.plugins[k].get("instrument") for k in keys) + + self.port = free_port(self.port + 2) + wd = tempfile.mkdtemp(dir=self.work) + host = Host(hkey, self.procs, self.env, wd, self.port) + host.start() + if instrument: + sources.append(self.procs.spawn(["jack_midiseq", "seq"] + self.cat.midi["chords"], "midiseq")) + want = "seq" + else: + sources.append(self.procs.spawn(["jack_simple_client", "sine"], "sine")) + want = "sine" + end = time.monotonic() + READY_LIMIT_S + while want not in ports(self.env): + if time.monotonic() > end: + raise Failed("the %s source did not start" % want) + time.sleep(0.05) + for k in range(2): + subprocess.run(["jack_disconnect", "sine:output%d" % (k + 1), "system:playback_%d" % (k + 1)], + env=self.env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + insts = list(range(len(picks))) + placement = "own" if hkey == "H2" and mode == "individual" else None + t0 = time.monotonic() + clients = [host.add(uri, i, placement) for (uri, _, _), i in zip(picks, insts)] + if mode == "chain" and insts: + reply = host.ctl.cmd("chain 1 " + " ".join(map(str, insts))) + if reply != "resp 0": + raise Failed("chain answered '%s'" % reply) + self.chain_formed(host, insts) + add_ms = (time.monotonic() - t0) * 1000 if insts else None + if clients: + graph = ports(self.env) + clients = [c for c in clients if c in graph] + wire(host, self.env, clients, instrument) + if hkey == "H2" and insts: + ws = host.workers() + if mode is None: + mode = "individual" if len(ws) == len(insts) else "shared-process" + if mode == "individual" and len(ws) != len(insts): + notes.append("worker_list shows %d workers for %d instances" % (len(ws), len(insts))) + if mode == "chain" and len(ws) != 1: + notes.append("worker_list shows %d workers for one chain" % len(ws)) + if mode == "chain" and len(clients) > 1: + notes.append("one worker, %d JACK clients" % len(clients)) + elif insts: + mode = mode or ("individual" if hkey == "H4" else "shared-process") + if mode: + r["mode"] = mode + time.sleep(1.0) + + pids = host.pids() + before = samples(pids) + t_start = time.monotonic() + loads = self.dsp_window(self.args.window) + after = samples(pids) + span = time.monotonic() - t_start + if not host.pids() or any(not alive(p) for p in pids): + raise Failed("a host process exited during the window") + if not loads: + raise Failed("jack_cpu_load gave no reading") + r["dsp_load_avg"] = round(sum(loads) / len(loads), 2) + r["dsp_load_max"] = round(max(loads), 2) + r["samples"] = len(loads) + common = [p for p in pids if p in before and p in after] + rss = sum(after[p]["rss"] for p in common) / 1024 + r["rss_mb"] = round(rss, 1) + r["add_ms"] = round(add_ms, 1) if add_ms is not None else None + r["processes"] = len(common) + r["rss_mb_total"] = round(rss, 1) + if all(after[p]["pss"] is not None for p in common): + r["pss_mb_total"] = round(sum(after[p]["pss"] for p in common) / 1024, 1) + ticks = sum(after[p]["ticks"] - before[p]["ticks"] for p in common) + r["cpu_pct"] = round(100.0 * ticks / CLK_TCK / span, 2) + vol = sum(after[p]["vol"] - before[p]["vol"] for p in common) + invol = sum(after[p]["invol"] - before[p]["invol"] for p in common) + r["ctx_switches"] = vol + invol + r["ctx_involuntary"] = invol + r["status"] = "ok" + + if s.get("kill"): + self.kill_and_recover(host, r, insts[0], clients[0]) + r["note"] = "; ".join(notes + ([r["note"]] if r["note"] else [])) + except Refused as e: + r["status"], r["note"] = "unsupported", str(e) + except Failed as e: + r["status"], r["note"] = "failed", str(e) + finally: + if host: + host.stop() + for p in sources: + self.procs.stop(p) + self.assert_only_jack("after %s %s %d/%d" % (s["id"], hkey, rate, frames)) + return r + + def chain_formed(self, host, insts): + want = ",".join(map(str, insts)) + end = time.monotonic() + READY_LIMIT_S + while time.monotonic() < end: + f = host.ctl.cmd("chain_get 1").split() + if len(f) >= 5 and f[2] != "-" and f[4] == want: + return + time.sleep(0.01) + raise Failed("the chain did not form within %.0f s" % READY_LIMIT_S) + + def kill_and_recover(self, host, r, inst, client): + """kill -9 the process that holds the plugin; time until its client is back and wired as before""" + if host.key == "H2": + holder = [w for w in host.workers() if inst in w[5]] + if not holder: + raise Failed("worker_list names no worker for instance %d" % inst) + pid = holder[0][1] + elif host.key == "H4": + pid = host.jalvs[inst].pid + else: + pid = host.proc.pid + graph = ports(self.env) + ins = graph[client]["audio_in"][0] + outs = graph[client]["audio_out"][0] + wired = lambda: connected(self.env, ins, "sine:output1") and connected(self.env, outs, "system:playback_1") + t0 = time.monotonic() + os.kill(pid, signal.SIGKILL) + gone = False + while time.monotonic() - t0 < RECOVER_LIMIT_S: + reap() + if host.key != "H2": + if not alive(pid): + break + elif not gone: + # the old client leaves the graph first; only then can a wired client be the respawned one + gone = not alive(pid) and not wired() + elif wired(): + r["recover_ms"] = round((time.monotonic() - t0) * 1000, 1) + return + time.sleep(0.005) + if host.key == "H2": + raise Failed("the plugin's audio was not back %.0f s after kill -9 of its worker" % RECOVER_LIMIT_S) + reap() + time.sleep(0.5) + if client in ports(self.env): + raise Failed("the client outlived kill -9 of its process") + r["status"] = "failed" + r["note"] = "host process lost" + + def assert_only_jack(self, when): + jack = {self.jackd.pid} if self.jackd and self.jackd.poll() is None else set() + stray = [(p, c) for p, c in marked() if p not in jack] + if stray: + for p, _ in stray: + try: + os.kill(p, signal.SIGKILL) + except OSError: + pass + raise SystemExit("bench: teardown %s left %s" % (when, ", ".join("%d %s" % x for x in stray))) + + # -- the run + + def qualify(self, rates, frames): + if not shutil.which("plugin-qualify"): + raise SystemExit("bench: --qualify needs plugin-qualify") + empty = os.path.join(self.work, "no-lv2") + os.makedirs(empty, exist_ok=True) + env = dict(self.env) + env["LV2_PATH"] = empty + t0 = time.monotonic() + p = subprocess.run(["plugin-qualify", "--all", "--isolate", "--rates", str(rates[0]), "--quanta", + str(frames[0]), "--format", "json"], env=env, stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, text=True) + secs = time.monotonic() - t0 + if p.returncode not in (0, 1): + raise SystemExit("bench: plugin-qualify could not measure (exit %d)" % p.returncode) + recs = json.loads(p.stdout) + good = sum(1 for x in recs if x["verdicts"] and x["verdicts"][0].get("rating") == "suitable") + return {"plugins": len(recs), "files": len({x["binary"]["path"] for x in recs}), "seconds": round(secs, 1), + "rates": [rates[0]], "quanta": [frames[0]], "suitable": good, "unsuitable": len(recs) - good} + + def doc_id(self): + day = datetime.date.today().strftime("%Y%m%d") + os.makedirs(self.args.out, exist_ok=True) + seq = 1 + while os.path.exists(os.path.join(self.args.out, "%s-%s-%d.json" % (self.args.cls, day, seq))): + seq += 1 + return "%s-%s-%d" % (self.args.cls, day, seq) + + def run(self): + self.governor_on() + doc = {"schema": "omx-bench/1", "id": self.doc_id(), "machine": machine(self.args.cls), + "software": software(self.cat.files()), "scenarios": []} + rows = self.plan() + for rate in self.args.rates: + for frames in self.args.frames: + self.jack_start(rate, frames) + for s, h, mode in rows: + r = self.row(s, h, mode, rate, frames) + log("%-8s %s %-14s %6d/%-4d %-11s dsp %s%% rss %s MB add %s ms %s" % ( + r["id"], h, r.get("mode", "-"), rate, frames, r["status"], r["dsp_load_avg"], r["rss_mb"], + r["add_ms"], r["note"])) + doc["scenarios"].append(r) + self.jack_stop() + if self.args.qualify: + doc["qualification"] = self.qualify(self.args.rates, self.args.frames) + path = os.path.join(self.args.out, doc["id"] + ".json") + with open(path + ".tmp", "w") as f: + json.dump(doc, f, indent=2) + f.write("\n") + os.rename(path + ".tmp", path) + return path + + def teardown(self): + """always: every process stopped, the governor put back, then nothing of this run may be left""" + self.procs.stop_all() + self.governor_off() + reap() + time.sleep(0.2) + left = marked() + if not self.args.logs: + shutil.rmtree(self.work, ignore_errors=True) + if left: + for p, _ in left: + try: + os.kill(p, signal.SIGKILL) + except OSError: + pass + log("teardown left %d process(es) of this run: %s" % (len(left), ", ".join("%d %s" % x for x in left))) + return False + log("teardown: no host, no JACK server, no other process of this run is left") + return True + + +def csv_ints(text): + return [int(x) for x in text.split(",") if x] + + +def main(): + ap = argparse.ArgumentParser(prog="bench/run.sh", description="compare four plugin hosts; write omx-bench/1") + ap.add_argument("--class", dest="cls", required=True, choices=["rpi5", "desk", "ci"]) + ap.add_argument("--out", required=True, help="directory of the documents") + ap.add_argument("--rates", type=csv_ints, default=[48000, 96000]) + ap.add_argument("--frames", type=csv_ints, default=[128, 256]) + ap.add_argument("--scenarios", type=lambda t: [x for x in t.split(",") if x], default=None) + ap.add_argument("--hosts", type=lambda t: [x for x in t.split(",") if x], default=list(HOSTS)) + ap.add_argument("--window", type=int, default=10, help="seconds of DSP load and /proc sampling per row") + ap.add_argument("--pianoteq-dir", default=None, help="the directory of Pianoteq's LV2 bundle") + ap.add_argument("--qualify", action="store_true", help="also time plugin-qualify --all --isolate") + ap.add_argument("--keep-governor", action="store_true", help="leave the CPU governor as it is") + ap.add_argument("--logs", default=None, help="keep the host logs in this directory") + args = ap.parse_args() + if set(args.hosts) - set(HOSTS): + ap.error("--hosts takes H1,H2,H3,H4") + table = json.load(open(os.path.join(BENCH, "scenarios.json"))) + known = {s["id"] for s in table["scenarios"]} + if args.scenarios and set(args.scenarios) - known: + ap.error("unknown scenario: " + ", ".join(sorted(set(args.scenarios) - known))) + for prog in ("jackd", "jack_cpu_load", "jack_simple_client", "jack_lsp", "jack_connect"): + if not shutil.which(prog): + ap.error(prog + " is needed (packages jackd2 and jack-example-tools)") + + # orphans of the hosts come back to the runner, which reaps them and sees them in the last check + try: + ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) # PR_SET_CHILD_SUBREAPER + except (OSError, AttributeError): + pass + runner = Runner(args, table) + + def on_signal(sig, _frame): + raise SystemExit("bench: stopped by signal %d" % sig) + + for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP): + signal.signal(sig, on_signal) + code = 0 + try: + path = runner.run() + print(path) + except SystemExit as e: + if e.code not in (None, 0): + print(e.code if isinstance(e.code, str) else "bench: exit %s" % e.code, file=sys.stderr) + code = 1 + finally: + for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP): + signal.signal(sig, signal.SIG_IGN) + if not runner.teardown(): + code = 3 + sys.exit(code) + + +if __name__ == "__main__": + main() diff --git a/bench/lib/send.sh b/bench/lib/send.sh new file mode 100755 index 0000000..7cb926c --- /dev/null +++ b/bench/lib/send.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# send.sh [ ...]: send each line to a mod-host protocol port on 127.0.0.1, NUL-terminated, +# and print its one reply on a line of its own. The runner holds one connection per host, as a controller does; +# this is the same framing for a person or a script. +set -euo pipefail + +if [ $# -lt 2 ]; then + echo "usage: send.sh [ ...]" >&2 + exit 2 +fi +port=$1 +shift +for line in "$@"; do + reply=$(printf '%s\0' "$line" | nc -w 5 127.0.0.1 "$port" | tr -d '\0') + if [ -z "$reply" ]; then + echo "send.sh: no reply from port $port to: $line" >&2 + exit 1 + fi + printf '%s\n' "$reply" +done diff --git a/bench/run.sh b/bench/run.sh new file mode 100755 index 0000000..adc7b39 --- /dev/null +++ b/bench/run.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# The benchmark runner: bench/README.md has the options and what each scenario measures. +set -euo pipefail + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +command -v python3 >/dev/null || { echo "run.sh: python3 is needed" >&2; exit 2; } +exec python3 -I "$here/lib/bench.py" "$@" diff --git a/bench/scenarios.json b/bench/scenarios.json new file mode 100644 index 0000000..025f2a6 --- /dev/null +++ b/bench/scenarios.json @@ -0,0 +1,33 @@ +{ + "plugins": { + "comp": { "name": "omx-comp", "clap": "omx-comp.clap#org.openmixer.comp", "lv2": "urn:openmixer:comp" }, + "delay": { "name": "omx-delay", "clap": "omx-delay.clap#org.openmixer.delay", "lv2": "urn:openmixer:delay" }, + "drive": { "name": "omx-drive", "clap": "omx-drive.clap#org.openmixer.drive", "lv2": "urn:openmixer:drive" }, + "calf": { "name": "Calf Compressor", "lv2": "http://calf.sourceforge.net/plugins/Compressor" }, + "pianoteq": { "name": "Pianoteq 9", "lv2": "https://www.modartt.com/lv2/Pianoteq9", "instrument": true, "needs": "pianoteq" } + }, + "midi": { + "chords": ["80000", "0", "60", "70000", "0", "64", "70000", "0", "67", "70000", "0", "72", "70000", "0", "76", "70000"] + }, + "scenarios": [ + { "id": "idle", "title": "the host running, no plugin", "plugins": [] }, + { "id": "clap1", "title": "one CLAP plugin, sine in", "plugins": ["comp"], "format": "clap" }, + { "id": "lv2-1", "title": "one LV2 plugin, sine in", "plugins": ["calf"] }, + { "id": "chain3", "title": "three plugins in a chain, sine in", "plugins": ["comp", "calf", "delay"], "chain": true }, + { "id": "clap16", "title": "16 instances of one CLAP plugin, sine in", "plugins": ["comp"], "repeat": 16 }, + { "id": "pianoteq", "title": "Pianoteq 9, five-note chords", "plugins": ["pianoteq"] }, + { "id": "kill1", "title": "one plugin, its process killed", "plugins": ["comp"], "kill": true }, + { "id": "cvp3", "title": "chains vs processes, 3 plugins", "plugins": ["comp", "calf", "delay", "drive"], "count": 3, "family": "chains-vs-processes" }, + { "id": "cvp8", "title": "chains vs processes, 8 plugins", "plugins": ["comp", "calf", "delay", "drive"], "count": 8, "family": "chains-vs-processes" }, + { "id": "cvp16", "title": "chains vs processes, 16 plugins", "plugins": ["comp", "calf", "delay", "drive"], "count": 16, "family": "chains-vs-processes" } + ], + "modes": { + "chains-vs-processes": [ + { "host": "H2", "mode": "individual" }, + { "host": "H2", "mode": "chain" }, + { "host": "H1", "mode": "shared-process" }, + { "host": "H3", "mode": "shared-process" }, + { "host": "H4", "mode": "individual" } + ] + } +} From c6d52d005eddc23f90a762415e12e012d703dccd Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Mon, 26 Oct 2026 23:15:30 +0100 Subject: [PATCH 3/7] Bench: break each guard and watch it go red Five broken documents (an undeclared status, an ok row with no DSP load, an unsupported row with no reason, a max below its average, an undeclared field) each fail validate.sh naming file, row and rule. A runner whose host stop leaves omx-clap-host fails after the row; one that leaves jackd fails at the end, exit 3. The runner as it is ends green with nothing left. --- bench/tests/sabotage.sh | 91 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100755 bench/tests/sabotage.sh diff --git a/bench/tests/sabotage.sh b/bench/tests/sabotage.sh new file mode 100755 index 0000000..73e3f25 --- /dev/null +++ b/bench/tests/sabotage.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# Break each guard of the harness on purpose and watch it go red, then run it whole and watch it stay green: +# validator: a shipped example passes; a document with one field broken fails, naming file, scenario and rule +# teardown: a copy of the runner whose host stop leaves the host, and one whose last stop leaves the JACK server, +# both end non-zero naming the process left; the runner as it is ends with nothing left +# The teardown part needs jackd2, jack-example-tools and omx-clap-host: run it with --teardown. +set -euo pipefail + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +fail=0 + +red() { # red