From b756251f0461eeff54b1d65a8dc4d7bbc8275536 Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Mon, 19 Oct 2026 21:12:37 +0200 Subject: [PATCH 1/6] probe: pass --probe-notes and --hold to the CLAP prober (#106) The runner builds clap-qualify's argv itself, so the count an instrument with no voice-info is held at could not reach the prober through it. Both take a positive count; how many the prober can hold is the prober's to say. The test child writes the arguments it was given and prints a latency and a cost for every rate it is asked for, of the shapes clap-qualify prints. --- README.md | 3 ++ doc/plugin-hostd-probe.1 | 7 ++++ src/probe.c | 35 +++++++++++++++++++- tests/probe_child.c | 71 +++++++++++++++++++++++++++++++++++++--- tests/probe_test.py | 16 +++++++++ 5 files changed, 127 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c8e0c6f..d9a6227 100644 --- a/README.md +++ b/README.md @@ -317,6 +317,9 @@ The runner never loads plugin code itself. plugin-hostd-probe run --out probe.json --log probe.stderr.log /usr/lib64/clap/plate.clap org.example.plate plugin-hostd-probe run --out probe.json lv2:http://example.org/plate +A CLAP instrument is timed holding notes down: as many as the voice capacity it declares, or, when it declares none, +the count `--probe-notes N` gives; `--hold N` holds that count whatever it declares. Both are passed to clap-qualify. + An LV2 plugin is measured the way plugin-qualify's LV2 hosting pass measures it today, in two children under the same limits and the same wall clock. The first child (`lv2_measure --pin`) pins the bundle's files without instantiating the plugin. The second measures at the highest of `--rates`, in blocks of `--quantum`, with 30 instantiate and destroy cycles diff --git a/doc/plugin-hostd-probe.1 b/doc/plugin-hostd-probe.1 index 8074f84..8094bdb 100644 --- a/doc/plugin-hostd-probe.1 +++ b/doc/plugin-hostd-probe.1 @@ -41,6 +41,13 @@ Blocks to process per rate. .BI \-\-offer " EXT,..." Host extensions to offer on top of the child's own. .TP +.BI \-\-probe\-notes " n" +The notes a CLAP instrument that declares no voice capacity is timed holding. Passed to clap\-qualify; without it such +an instrument has no cost. +.TP +.BI \-\-hold " n" +The notes a CLAP instrument is timed holding, whatever voice capacity it declares. Passed to clap\-qualify. +.TP .BI \-\-out " file" Where to write the document, atomically. Default standard output. .TP diff --git a/src/probe.c b/src/probe.c index 3109884..c01387b 100644 --- a/src/probe.c +++ b/src/probe.c @@ -414,6 +414,8 @@ typedef struct const char *offer; const char *set; unsigned quantum, blocks; + /* the notes a CLAP instrument is timed holding: when it declares no voice capacity, and whatever it declares */ + unsigned probe_notes, hold; long wall_s, cpu_s, files; long long data_bytes, file_bytes; long grace_ms; @@ -1072,7 +1074,7 @@ static void take_step_records(run_t *r) r->steps.s[r->steps.len] = '\0'; } -/* clap-qualify's arguments: the rates, the block, the offer, the file and the id */ +/* clap-qualify's arguments: the rates, the block, the offer, the notes an instrument holds, the file and the id */ static void clap_argv(const options_t *o, char **argv) { int argc = 0; @@ -1098,6 +1100,22 @@ static void clap_argv(const options_t *o, char **argv) argv[argc++] = "--offer"; argv[argc++] = (char *)o->offer; } + { + static char notes[32], hold[32]; + + if (o->probe_notes) + { + snprintf(notes, sizeof(notes), "%u", o->probe_notes); + argv[argc++] = "--probe-notes"; + argv[argc++] = notes; + } + if (o->hold) + { + snprintf(hold, sizeof(hold), "%u", o->hold); + argv[argc++] = "--hold"; + argv[argc++] = hold; + } + } argv[argc++] = (char *)o->plugin; if (o->id) argv[argc++] = (char *)o->id; @@ -1568,6 +1586,8 @@ int main(int argc, char **argv) " --rates R,R,... sample rates (default %s)\n" " --quantum N block size in frames (default 512) --blocks N blocks per rate\n" " --offer EXT,... host extensions to offer on top of the child's own\n" + " --probe-notes N notes a CLAP instrument declaring no voice capacity is timed holding\n" + " --hold N notes a CLAP instrument is timed holding, whatever it declares\n" " --out FILE the document, written atomically (default stdout)\n" " --log FILE the child's stderr (default discarded)\n" " --reason WHY add, rescan, binary-changed, on-request, auto-measure, reference, ci\n" @@ -1594,6 +1614,19 @@ int main(int argc, char **argv) else if (!strcmp(k, "--quantum")) o.quantum = (unsigned)atoi(v); else if (!strcmp(k, "--blocks")) o.blocks = (unsigned)atoi(v); else if (!strcmp(k, "--offer")) o.offer = v; + else if (!strcmp(k, "--probe-notes") || !strcmp(k, "--hold")) + { + char *end; + long n = strtol(v, &end, 10); + + /* how many the prober can hold is the prober's to say: a count it refuses ends its run as any refusal does */ + if (*end || n < 1 || n > INT_MAX) + { + fprintf(stderr, RUNNER_NAME ": %s takes a positive count of notes\n", k); + return 2; + } + *(k[2] == 'p' ? &o.probe_notes : &o.hold) = (unsigned)n; + } else if (!strcmp(k, "--out")) o.out = v; else if (!strcmp(k, "--log")) o.log = v; else if (!strcmp(k, "--reason")) o.reason = v; diff --git a/tests/probe_child.c b/tests/probe_child.c index d306c58..ce10706 100644 --- a/tests/probe_child.c +++ b/tests/probe_child.c @@ -21,7 +21,14 @@ /* The smallest child the probe runner's tests can run a plugin with: it takes clap-qualify's arguments, loads the * .clap, creates and activates the plugin, writes a step record on file descriptor 3 after each step, runs one block * through process() and prints one JSON line of facts. A plugin that crashes, hangs or leaves a process behind in - * process() (tests/stress_plugin.c, STRESS_FAULT) leaves the records written before. */ + * process() (tests/stress_plugin.c, STRESS_FAULT) leaves the records written before. + * + * The line carries a latency and a cost for every rate of --rates, of the shapes clap-qualify prints, with figures + * made from the rate (a median of rate / 1000 ns per sample), so a figure the runner put at another rate's point is + * seen. PROBE_CHILD_INSTRUMENT makes the plugin an instrument: its latency is taken from a note, its cost holds the + * notes --hold or --probe-notes give (no count: the rate is not timed) and it has a note sweep, counted under the + * interposer when PROBE_CHILD_NOTE_ALLOCATIONS gives the allocations it saw. PROBE_CHILD_SILENT_RATE is a rate where + * the note was not heard, PROBE_CHILD_UNMEASURED_RATE one the cost arm did not time. */ #ifndef _GNU_SOURCE #define _GNU_SOURCE @@ -106,9 +113,51 @@ static void record(const char *step, long rate, long quantum, long since) _exit(2); } +/* the per-rate facts of the line, for every rate of --rates */ +static void put_rates(const char *rates, long quantum, unsigned notes) +{ + const char *instrument = getenv("PROBE_CHILD_INSTRUMENT"), *note_allocs = getenv("PROBE_CHILD_NOTE_ALLOCATIONS"); + long silent = getenv("PROBE_CHILD_SILENT_RATE") ? atol(getenv("PROBE_CHILD_SILENT_RATE")) : 0; + long unmeasured = getenv("PROBE_CHILD_UNMEASURED_RATE") ? atol(getenv("PROBE_CHILD_UNMEASURED_RATE")) : 0; + const char *p; + char *end; + long r; + + printf("\"latency\":{"); + for (p = rates; (r = strtol(p, &end, 10)) > 0; p = *end ? end + 1 : end) + printf("%s\"%ld\":{\"declaredFrames\":0,\"measuredFrames\":%d%s%s}", p == rates ? "" : ",", r, instrument ? 1 : 0, + instrument ? ",\"from\":\"note\"" : "", instrument && r == silent ? ",\"heard\":false" : ""); + printf("},\"threadViolations\":0,\"processErrors\":0},\"rtSafety\":{\"interposer\":false}"); + if (instrument && note_allocs) + printf(",\"noteSweep\":{\"applicable\":true,\"dialect\":\"clap\",\"interposer\":true,\"allocationsInRun\":%ld," + "\"locksInRun\":0,\"syscallsInRun\":0,\"events\":[]}", atol(note_allocs)); + else + printf(",\"noteSweep\":{\"applicable\":%s%s}", instrument ? "true" : "false", instrument ? ",\"dialect\":\"clap\",\"interposer\":false" : ""); + printf(",\"cost\":{\"stimulus\":\"%s\",\"blockFrames\":%ld,\"warmupBlocks\":64,\"percentile\":95,\"perRate\":{", + instrument ? "notes-held" : "noise", quantum); + for (p = rates; (r = strtol(p, &end, 10)) > 0; p = *end ? end + 1 : end) + { + double ns = (double)r / 1000; + + printf("%s\"%ld\":{", p == rates ? "" : ",", r); + if (r == unmeasured) + printf("\"unmeasuredReason\":\"no-figures\",\"unmeasuredDetail\":\"the test asked for none at this rate\""); + else if (instrument && !notes) + printf("\"unmeasuredReason\":\"voices-unknown\",\"unmeasuredDetail\":\"no voice-info and no --probe-notes count to hold\""); + else + printf("\"nsPerSampleMedian\":%g,\"nsPerSampleP95\":%g,\"nsPerSampleMin\":%g,\"nsPerSampleMax\":%g," + "\"warmupNsPerSampleMedian\":%g,\"blocks\":256", ns, ns + 1, ns - 1, ns + 4, ns + 2); + if (instrument && notes) + printf(",\"voices\":{\"held\":%u,\"basis\":\"probe\"}", notes); + printf("}"); + } + printf("}}"); +} + int main(int argc, char **argv) { const char *rates = "48000", *path = NULL, *id = NULL; + unsigned notes = 0, hold = 0; long rate, quantum = 256, t0 = now_ms(); void *lib; const clap_plugin_entry_t *entry; @@ -124,6 +173,10 @@ int main(int argc, char **argv) rates = argv[++a]; else if (!strcmp(argv[a], "--block") && a + 1 < argc) quantum = atol(argv[++a]); + else if (!strcmp(argv[a], "--probe-notes") && a + 1 < argc) + notes = (unsigned)atoi(argv[++a]); + else if (!strcmp(argv[a], "--hold") && a + 1 < argc) + hold = (unsigned)atoi(argv[++a]); else if ((!strcmp(argv[a], "--blocks") || !strcmp(argv[a], "--offer")) && a + 1 < argc) a++; else if (!path) @@ -135,6 +188,16 @@ int main(int argc, char **argv) return 2; if (getenv("PROBE_CHILD_LIMITS")) report_limits(getenv("PROBE_CHILD_LIMITS")); + /* PROBE_CHILD_ARGV names a file to write the arguments the runner gave, one to a line */ + if (getenv("PROBE_CHILD_ARGV")) + { + FILE *f = fopen(getenv("PROBE_CHILD_ARGV"), "w"); + + for (a = 1; f && a < argc; a++) + fprintf(f, "%s\n", argv[a]); + if (f) + fclose(f); + } rate = atol(rates); lib = dlopen(path, RTLD_NOW | RTLD_LOCAL); entry = lib ? dlsym(lib, "clap_entry") : NULL; @@ -167,8 +230,8 @@ int main(int argc, char **argv) plugin->stop_processing(plugin); plugin->deactivate(plugin); plugin->destroy(plugin); - printf("{\"id\":\"%s\",\"format\":\"clap\",\"hostExtensions\":[],\"clap\":{\"headlessOk\":true,\"latency\":{\"%ld\":" - "{\"declaredFrames\":0,\"measuredFrames\":0}},\"threadViolations\":0,\"processErrors\":0},\"rtSafety\":{\"interposer\":false}}\n", - id, rate); + printf("{\"id\":\"%s\",\"format\":\"clap\",\"hostExtensions\":[],\"clap\":{\"headlessOk\":true,", id); + put_rates(rates, quantum, hold ? hold : notes); + printf("}\n"); return 0; } diff --git a/tests/probe_test.py b/tests/probe_test.py index ba27a6f..55818ff 100644 --- a/tests/probe_test.py +++ b/tests/probe_test.py @@ -169,6 +169,22 @@ def the_limits_reach_the_child(): check(doc["machine"]["pinnedCpus"] == str(cpus[0]), "pinnedCpus %s" % doc["machine"]["pinnedCpus"]) +@test +def the_notes_an_instrument_holds_reach_the_clap_child(): + work = tempfile.mkdtemp(prefix="plugin-hostd-probe-argv.") + seen = os.path.join(work, "argv") + code, doc, _, _, _ = run(extra=("--probe-notes", "32", "--hold", "8"), env={"PROBE_CHILD_ARGV": seen}) + check(code == 0, "exit status %d" % code) + args = open(seen).read().split("\n") + check(args[args.index("--probe-notes") + 1] == "32" if "--probe-notes" in args else False, "--probe-notes 32 reached the child: %s" % args) + check(args[args.index("--hold") + 1] == "8" if "--hold" in args else False, "--hold 8 reached the child: %s" % args) + code, doc, _, _, _ = run(env={"PROBE_CHILD_ARGV": seen}) + args = open(seen).read().split("\n") + check("--probe-notes" not in args and "--hold" not in args, "no count is given when none was asked for: %s" % args) + for bad in ("0", "-3", "many"): + p = subprocess.run([PROBE, "run", "--child", CHILD, "--probe-notes", bad, CLAP], capture_output=True, text=True) + check(p.returncode == 2 and p.stdout == "", "--probe-notes %s is refused with no document: exit %d" % (bad, p.returncode)) + # ---------------------------------------------------------------- LV2: the runner drives lv2_measure's passes LV2_CHILD = os.path.abspath(os.environ.get("PROBE_LV2_CHILD", os.path.join(ROOT, "tests", "lv2-probe-child"))) From 971fe1832fbe11a71fb3542810c4f8c10d5b89e2 Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Mon, 19 Oct 2026 21:14:04 +0200 Subject: [PATCH 2/6] probe output 1.1: each point carries the cost timed at its rate, with the notes held (#106) clap-qualify prints a cost per rate, and for an instrument the voices it held while it was timed; the runner copied none of it, so a record judged through the runner had no cost for a CLAP plugin. Each grid point now carries the figures of its own rate, and voices when the line has them. A rate the prober did not time keeps no cost, never a neighbour's. The cost block gains the optional voices object: a minor of the format. A fourth fixture is an instrument, and three broken copies of it are refused. --- README.md | 2 + include/plugin-hostd/protocol.h | 2 +- protocol/probe-output.schema.in | 25 ++ protocol/probe-output.schema.json | 25 ++ src/probe.c | 32 +- tests/fixtures/probe/instrument-notes.json | 296 ++++++++++++++++++ .../fixtures/probe/memory-limit-refused.json | 2 +- tests/fixtures/probe/plugin-crash.json | 2 +- tests/fixtures/probe/success-msi.json | 2 +- tests/probe_schema.py | 19 +- tests/probe_test.py | 58 +++- 11 files changed, 449 insertions(+), 16 deletions(-) create mode 100644 tests/fixtures/probe/instrument-notes.json diff --git a/README.md b/README.md index d9a6227..396e7d4 100644 --- a/README.md +++ b/README.md @@ -319,6 +319,8 @@ The runner never loads plugin code itself. A CLAP instrument is timed holding notes down: as many as the voice capacity it declares, or, when it declares none, the count `--probe-notes N` gives; `--hold N` holds that count whatever it declares. Both are passed to clap-qualify. +Each rate's grid point carries the cost clap-qualify timed at that rate, in nanoseconds per sample, and for an instrument +`voices`, the notes it held and why that many (`declared` or `probe`). A rate it did not time has no cost. An LV2 plugin is measured the way plugin-qualify's LV2 hosting pass measures it today, in two children under the same limits and the same wall clock. The first child (`lv2_measure --pin`) pins the bundle's files without instantiating the diff --git a/include/plugin-hostd/protocol.h b/include/plugin-hostd/protocol.h index 133b4be..eb08793 100644 --- a/include/plugin-hostd/protocol.h +++ b/include/plugin-hostd/protocol.h @@ -116,7 +116,7 @@ #define PHD_PROBE_FORMAT "plugin-hostd.probe-output" #define PHD_PROBE_FORMAT_MAJOR 1 -#define PHD_PROBE_FORMAT_MINOR 0 +#define PHD_PROBE_FORMAT_MINOR 1 #define PHD_PROBE_PROTOCOL_NAME "plugin-hostd.probe-protocol" #define PHD_PROBE_PROTOCOL_VERSION 1 /* version 1 of the probe protocol runs an LV2 plugin as plugin-qualify's LV2 hosting pass does today diff --git a/protocol/probe-output.schema.in b/protocol/probe-output.schema.in index 48b00cd..186025c 100644 --- a/protocol/probe-output.schema.in +++ b/protocol/probe-output.schema.in @@ -820,6 +820,31 @@ }, "samples": { "$ref": "#/$defs/fileRef" + }, + "voices": { + "description": "An instrument's: the notes it held down while it was timed, and why that many.", + "type": "object", + "required": [ + "held", + "basis" + ], + "additionalProperties": false, + "properties": { + "held": { + "type": "integer", + "minimum": 1 + }, + "basis": { + "enum": [ + "declared", + "probe" + ] + }, + "capacity": { + "type": "integer", + "minimum": 0 + } + } } } }, diff --git a/protocol/probe-output.schema.json b/protocol/probe-output.schema.json index ebaf8f6..1946a76 100644 --- a/protocol/probe-output.schema.json +++ b/protocol/probe-output.schema.json @@ -820,6 +820,31 @@ }, "samples": { "$ref": "#/$defs/fileRef" + }, + "voices": { + "description": "An instrument's: the notes it held down while it was timed, and why that many.", + "type": "object", + "required": [ + "held", + "basis" + ], + "additionalProperties": false, + "properties": { + "held": { + "type": "integer", + "minimum": 1 + }, + "basis": { + "enum": [ + "declared", + "probe" + ] + }, + "capacity": { + "type": "integer", + "minimum": 0 + } + } } } }, diff --git a/src/probe.c b/src/probe.c index c01387b..0248067 100644 --- a/src/probe.c +++ b/src/probe.c @@ -1535,6 +1535,32 @@ static void put_lv2_measure(sb_t *b, const jnode_t *m, long rate, unsigned quant sb_printf(b, "]}}"); } +/* the cost arm's figures at one rate, as that rate's point carries them: nanoseconds per sample, and for an instrument + * the notes it held; a rate the arm did not time (an unmeasuredReason in place of the figures) gets no cost, never a + * neighbour's */ +static void put_cost(sb_t *b, const jnode_t *cost, const char *rate) +{ + const jnode_t *at = jget(jget(cost, "perRate"), rate), *voices = jget(at, "voices"); + double median, p95, min, max, blocks, warmup, percentile, held, capacity; + + if (!jnum(at, "nsPerSampleMedian", &median) || !jnum(at, "nsPerSampleP95", &p95) || + !jnum(at, "nsPerSampleMin", &min) || !jnum(at, "nsPerSampleMax", &max) || !jnum(at, "blocks", &blocks) || + !jnum(cost, "warmupBlocks", &warmup) || !jnum(cost, "percentile", &percentile)) + return; + sb_printf(b, ", \"cost\": {\"unit\": \"ns-per-sample\", \"median\": %.9g, \"percentile\": {\"p\": %.9g, \"value\": %.9g}, " + "\"min\": %.9g, \"max\": %.9g, \"blocks\": %.0f, \"warmupBlocks\": %.0f", + median, percentile, p95, min, max, blocks, warmup); + if (jnum(voices, "held", &held) && jstr(voices, "basis")) + { + sb_printf(b, ", \"voices\": {\"held\": %.0f, \"basis\": ", held); + sb_str(b, jstr(voices, "basis")); + if (jnum(voices, "capacity", &capacity)) + sb_printf(b, ", \"capacity\": %.0f", capacity); + sb_printf(b, "}"); + } + sb_printf(b, "}"); +} + /* the rates a run measures at when it is given none: omx-contract's ORACLE_FLOOR_RATES, every kernel oracle's floor */ static const unsigned g_floor_rates[] = OMX_ORACLE_FLOOR_RATES_INIT; @@ -1791,7 +1817,8 @@ int main(int argc, char **argv) { const char *plugin_id = line ? jstr(line, "id") : NULL; const jnode_t *clap = jget(line, "clap"), *integ = jget(line, "integrity"), *rt = jget(line, "rtSafety"), - *notes = jget(line, "noteSweep"), *latency = jget(clap, "latency"), *ext = jget(line, "hostExtensions"); + *notes = jget(line, "noteSweep"), *latency = jget(clap, "latency"), *ext = jget(line, "hostExtensions"), + *cost = jget(line, "cost"); const char *layout_sha = NULL, *unmeasured = NULL; int firstpoint = 1; size_t k; @@ -1958,7 +1985,7 @@ int main(int argc, char **argv) put_step(&doc, &g_steps[i]); } sb_printf(&doc, "\n ]"); - /* the grid: one point per rate the child reported a latency for */ + /* the grid: one point per rate the child reported a latency for, with the cost the child timed at that rate */ if (latency && latency->n) { sb_printf(&doc, ",\n \"grid\": ["); @@ -1971,6 +1998,7 @@ int main(int argc, char **argv) sb_printf(&doc, "%s\n {\"rate\": %ld, \"quantum\": %u, \"outcome\": \"%s\", \"latency\": {\"declaredFrames\": %.0f, " "\"measuredFrames\": %.0f, \"method\": \"impulse-onset\"}", k ? "," : "", atol(latency->key[k]), o.quantum, OUT_OK, decl, meas); + put_cost(&doc, cost, latency->key[k]); if (firstpoint && rt) { double v = 0; diff --git a/tests/fixtures/probe/instrument-notes.json b/tests/fixtures/probe/instrument-notes.json new file mode 100644 index 0000000..c35e890 --- /dev/null +++ b/tests/fixtures/probe/instrument-notes.json @@ -0,0 +1,296 @@ +{ + "format": "plugin-hostd.probe-output", + "formatVersion": "1.1", + "run": { + "id": "msi-20261019T101500Z-3571140", + "startedAt": "2026-10-19T10:15:00+02:00", + "elapsedMs": 96410, + "complete": true, + "outcome": "probe.ok", + "reason": "reference", + "protocol": { + "name": "plugin-hostd.probe-protocol", + "version": 1 + }, + "tools": { + "runner": "plugin-hostd-probe 0.5.0", + "child": "clap-qualify" + }, + "via": "runner" + }, + "plugin": { + "format": "clap", + "id": "org.example.pad", + "name": "Pad", + "vendor": "Example", + "version": "2.0.1" + }, + "binary": { + "path": "/usr/lib64/clap/example-pad.clap", + "sha256": "7d1b0c55e1f3a9b8c2d4e6f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c", + "files": [ + { + "path": "example-pad.clap", + "sha256": "7d1b0c55e1f3a9b8c2d4e6f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c" + } + ] + }, + "machine": { + "class": { + "arch": "x86_64", + "family": "intel", + "coreBand": "17-32", + "clockBand": "4.5GHz+", + "preemptRt": false, + "cstatesLimited": true + }, + "profile": { + "governor": "performance", + "turbo": true, + "smt": true, + "preempt": "PREEMPT_DYNAMIC", + "cpuidle": [ + { + "cpu": 22, + "state": "POLL", + "latencyUs": 0, + "disabled": false, + "usageDelta": 118 + }, + { + "cpu": 22, + "state": "C1_ACPI", + "latencyUs": 1, + "disabled": false, + "usageDelta": 40210 + }, + { + "cpu": 22, + "state": "C2_ACPI", + "latencyUs": 127, + "disabled": true, + "usageDelta": 0 + }, + { + "cpu": 22, + "state": "C3_ACPI", + "latencyUs": 1048, + "disabled": true, + "usageDelta": 0 + } + ], + "meetsProtocol": true, + "declaredName": "omx-live-rig" + }, + "pinnedCpus": "22", + "reference": { + "rig": "msi", + "image": "omx-ref-2026-10" + } + }, + "offer": { + "id": "default", + "realtime": "none" + }, + "limits": { + "set": "default", + "applied": [ + { + "limit": "wall-clock", + "value": 600, + "unit": "s", + "by": "supervisor" + }, + { + "limit": "cpu-time", + "value": 540, + "unit": "s", + "by": "rlimit" + }, + { + "limit": "memory", + "value": 2147483648, + "unit": "bytes", + "by": "cgroup" + }, + { + "limit": "tasks", + "value": 64, + "unit": "count", + "by": "cgroup" + }, + { + "limit": "files", + "value": 256, + "unit": "count", + "by": "rlimit" + }, + { + "limit": "file-size", + "value": 67108864, + "unit": "bytes", + "by": "rlimit" + }, + { + "limit": "core-dump", + "value": 0, + "unit": "bytes", + "by": "rlimit" + }, + { + "limit": "rt-priority", + "value": 0, + "by": "rlimit" + }, + { + "limit": "cpus", + "value": "22", + "by": "affinity" + }, + { + "limit": "no-new-privs", + "by": "prctl" + }, + { + "limit": "network", + "value": "none", + "by": "netns" + } + ], + "unavailable": [] + }, + "steps": [ + { + "step": "hash", + "outcome": "probe.ok", + "elapsedMs": 4 + }, + { + "step": "instantiate", + "outcome": "probe.ok", + "elapsedMs": 31 + }, + { + "step": "lifecycle", + "outcome": "probe.ok", + "elapsedMs": 2210 + }, + { + "step": "latency", + "rate": 48000, + "quantum": 128, + "outcome": "probe.ok", + "elapsedMs": 310 + }, + { + "step": "cost", + "rate": 48000, + "quantum": 128, + "outcome": "probe.ok", + "elapsedMs": 30020 + }, + { + "step": "rt-safety", + "rate": 48000, + "quantum": 128, + "outcome": "probe.ok", + "elapsedMs": 4100 + }, + { + "step": "latency", + "rate": 96000, + "quantum": 128, + "outcome": "probe.ok", + "elapsedMs": 305 + }, + { + "step": "cost", + "rate": 96000, + "quantum": 128, + "outcome": "probe.ok", + "elapsedMs": 30011 + }, + { + "step": "rt-safety", + "rate": 96000, + "quantum": 128, + "outcome": "probe.ok", + "elapsedMs": 4080 + }, + { + "step": "note-sweep", + "outcome": "probe.not-attempted", + "elapsedMs": 0, + "detail": "no note input" + } + ], + "grid": [ + { + "rate": 48000, + "quantum": 128, + "outcome": "probe.ok", + "latency": { + "declaredFrames": 0, + "measuredFrames": 1 + }, + "cost": { + "unit": "ns-per-sample", + "median": 212.4, + "percentile": { + "p": 95, + "value": 237.9 + }, + "min": 201.8, + "max": 446.0, + "blocks": 11250, + "warmupBlocks": 64, + "voices": { + "held": 16, + "basis": "declared", + "capacity": 16 + } + } + }, + { + "rate": 96000, + "quantum": 128, + "outcome": "probe.ok", + "latency": { + "declaredFrames": 0, + "measuredFrames": 1 + }, + "cost": { + "unit": "ns-per-sample", + "median": 208.9, + "percentile": { + "p": 95, + "value": 234.0 + }, + "min": 198.5, + "max": 438.7, + "blocks": 22500, + "warmupBlocks": 64, + "voices": { + "held": 32, + "basis": "probe" + } + } + }, + { + "rate": 192000, + "quantum": 128, + "outcome": "probe.ok", + "latency": { + "declaredFrames": 0, + "measuredFrames": 1 + } + } + ], + "behaviour": { + "lifecycle": { + "cycles": 30, + "instantiated": 30, + "failed": 0 + } + } +} diff --git a/tests/fixtures/probe/memory-limit-refused.json b/tests/fixtures/probe/memory-limit-refused.json index 12238b6..4bd8fe0 100644 --- a/tests/fixtures/probe/memory-limit-refused.json +++ b/tests/fixtures/probe/memory-limit-refused.json @@ -1,6 +1,6 @@ { "format": "plugin-hostd.probe-output", - "formatVersion": "1.0", + "formatVersion": "1.1", "run": { "id": "r1-20261017T232540Z-652b62e", "startedAt": "2026-10-17T23:25:40+02:00", diff --git a/tests/fixtures/probe/plugin-crash.json b/tests/fixtures/probe/plugin-crash.json index add7d52..8ac6cdc 100644 --- a/tests/fixtures/probe/plugin-crash.json +++ b/tests/fixtures/probe/plugin-crash.json @@ -1,6 +1,6 @@ { "format": "plugin-hostd.probe-output", - "formatVersion": "1.0", + "formatVersion": "1.1", "run": { "id": "r1-20261017T231004Z-0b858f1", "startedAt": "2026-10-17T23:10:04+02:00", diff --git a/tests/fixtures/probe/success-msi.json b/tests/fixtures/probe/success-msi.json index 0bf226e..86b7ae1 100644 --- a/tests/fixtures/probe/success-msi.json +++ b/tests/fixtures/probe/success-msi.json @@ -1,6 +1,6 @@ { "format": "plugin-hostd.probe-output", - "formatVersion": "1.0", + "formatVersion": "1.1", "run": { "id": "msi-20261017T221500Z-3569027", "startedAt": "2026-10-17T22:15:00+02:00", diff --git a/tests/probe_schema.py b/tests/probe_schema.py index 2735839..31864de 100644 --- a/tests/probe_schema.py +++ b/tests/probe_schema.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-3.0-or-later -# The probe output (protocol/probe-output.schema.json, generated from include/plugin-hostd/protocol.h) against its three -# fixtures, and against five broken copies of the first: a guard that refuses nothing is decoration, so each +# The probe output (protocol/probe-output.schema.json, generated from include/plugin-hostd/protocol.h) against its four +# fixtures, and against broken copies of the first and of the instrument's: a guard that refuses nothing is decoration, so each # copy must be refused, and at the path the schema names. # probe_schema.py [tree] tree holds the schema (default: this checkout); the fixtures are always this checkout's import copy, json, os, sys @@ -39,7 +39,7 @@ def errors(document): except jsonschema.SchemaError as e: check(False, "the schema is a valid draft 2020-12 schema: %s" % e.message) -for name in ("success-msi", "plugin-crash", "memory-limit-refused"): +for name in ("success-msi", "plugin-crash", "memory-limit-refused", "instrument-notes"): found = errors(fixture(name)) check(not found, "%s validates%s" % (name, "" if not found else ": " + str(found[:3]))) @@ -60,13 +60,20 @@ def go(d): return go -# (what is broken in a copy of the success document, the path the schema must refuse it at, the keyword that refuses) +# (what is broken in a copy of the success document, the path the schema must refuse it at, the keyword that refuses[, +# the fixture broken when it is not the success document]) BROKEN = [ ("a verdict block added", "/", "not", put([], "verdict", {"class": "in-process"})), ("binary.sha256 removed", "/binary", "required", drop(["binary"], "sha256")), ("a step outcome that is no code", "/steps/0/outcome", "enum", put(["steps", 0], "outcome", "probe.exploded")), ("a format version of another major", "/formatVersion", "pattern", put([], "formatVersion", "2.0")), ("a cost percentile of 50", "/grid/0/cost/percentile/p", "exclusiveMinimum", None), + ("an instrument timed holding no note", "/grid/0/cost/voices/held", "minimum", + put(["grid", 0, "cost", "voices"], "held", 0), "instrument-notes"), + ("held notes with a basis that is none of the format's", "/grid/1/cost/voices/basis", "enum", + put(["grid", 1, "cost", "voices"], "basis", "guess"), "instrument-notes"), + ("held notes that do not say why that many", "/grid/1/cost/voices", "required", + drop(["grid", 1, "cost", "voices"], "basis"), "instrument-notes"), ] @@ -74,8 +81,8 @@ def percentile_50(d): d["grid"][0]["cost"]["percentile"]["p"] = 50 -for what, path, keyword, mutate in BROKEN: - d = copy.deepcopy(fixture("success-msi")) +for what, path, keyword, mutate, *broken in BROKEN: + d = copy.deepcopy(fixture(broken[0] if broken else "success-msi")) (mutate or percentile_50)(d) found = errors(d) check((path, keyword) in found, diff --git a/tests/probe_test.py b/tests/probe_test.py index 55818ff..aebdef4 100644 --- a/tests/probe_test.py +++ b/tests/probe_test.py @@ -31,8 +31,9 @@ def check(cond, what): raise AssertionError(what) -def run(fault=None, wall=5, extra=(), timeout=30, env=None): - """the runner over the stress plugin; returns (exit status, document or None, seconds, the files it used)""" +def run(fault=None, wall=5, extra=(), timeout=30, env=None, rates="48000"): + """the runner over the stress plugin; returns (exit status, document or None, seconds, the files it used); rates None + leaves the runner's default rates""" work = tempfile.mkdtemp(prefix="plugin-hostd-probe-test.") out = os.path.join(work, "probe.json") pidfile = os.path.join(work, "forkpid") @@ -41,8 +42,8 @@ def run(fault=None, wall=5, extra=(), timeout=30, env=None): e["STRESS_FAULT"] = fault else: e.pop("STRESS_FAULT", None) - cmd = [PROBE, "run", "--child", CHILD, "--rates", "48000", "--quantum", "256", "--wall-s", str(wall), "--grace-ms", "200", - "--log", os.path.join(work, "child.log"), "--out", out, *extra, CLAP, ID] + cmd = [PROBE, "run", "--child", CHILD, *(("--rates", rates) if rates else ()), "--quantum", "256", "--wall-s", str(wall), + "--grace-ms", "200", "--log", os.path.join(work, "child.log"), "--out", out, *extra, CLAP, ID] start = time.monotonic() try: p = subprocess.run(cmd, env=e, capture_output=True, text=True, timeout=timeout) @@ -61,6 +62,16 @@ def kill_leftover(pidfile): pass +def floor_rates(): + """omx-contract's ORACLE_FLOOR_RATES, the rates a run measures at by default, read from the installed contract""" + base = subprocess.run(["pkg-config", "--variable=contractdir", "omx-contract"], capture_output=True, text=True, check=True).stdout.strip() + for major in sorted(os.listdir(base), reverse=True): + rates = os.path.join(base, major, "omx-contract", "rates.json") + if os.path.exists(rates): + return json.load(open(rates))["ORACLE_FLOOR_RATES"]["values"] + raise AssertionError("omx-contract declares no ORACLE_FLOOR_RATES") + + def valid(doc): errors = [e.message for e in jsonschema.Draft202012Validator(SCHEMA).iter_errors(doc)] check(not errors, "the document does not validate: %s" % errors[:3]) @@ -185,6 +196,45 @@ def the_notes_an_instrument_holds_reach_the_clap_child(): p = subprocess.run([PROBE, "run", "--child", CHILD, "--probe-notes", bad, CLAP], capture_output=True, text=True) check(p.returncode == 2 and p.stdout == "", "--probe-notes %s is refused with no document: exit %d" % (bad, p.returncode)) + +@test +def every_point_carries_the_cost_timed_at_its_own_rate(): + rates = floor_rates() + code, doc, _, _, _ = run(rates=None) + check(code == 0, "exit status %d" % code) + valid(doc) + check([g["rate"] for g in doc["grid"]] == rates, "a point per rate of omx-contract's floor %s: %s" % (rates, doc["grid"])) + for g in doc["grid"]: + c = g.get("cost", {}) + want = {"unit": "ns-per-sample", "median": g["rate"] / 1000, "percentile": {"p": 95, "value": g["rate"] / 1000 + 1}, + "min": g["rate"] / 1000 - 1, "max": g["rate"] / 1000 + 4, "blocks": 256, "warmupBlocks": 64} + check(c == want, "the point at %d carries the figures timed at %d, and no voices for an effect: %s" % (g["rate"], g["rate"], c)) + + +@test +def an_instrument_point_carries_the_notes_it_held_and_an_untimed_rate_has_no_cost(): + rates = floor_rates() + untimed = rates[1] + code, doc, _, _, _ = run(rates=None, extra=("--probe-notes", "32"), + env={"PROBE_CHILD_INSTRUMENT": "1", "PROBE_CHILD_UNMEASURED_RATE": str(untimed)}) + check(code == 0, "exit status %d" % code) + valid(doc) + check(doc["formatVersion"] == "1.1", "formatVersion %s" % doc["formatVersion"]) + check([g["rate"] for g in doc["grid"]] == rates, "a point per rate %s: %s" % (rates, [g["rate"] for g in doc["grid"]])) + for g in doc["grid"]: + if g["rate"] == untimed: + check("cost" not in g, "the rate the child did not time has no cost, not a neighbour's: %s" % g) + continue + c = g.get("cost", {}) + check(c.get("median") == g["rate"] / 1000, "the point at %d carries its own median: %s" % (g["rate"], c)) + check(c.get("voices") == {"held": 32, "basis": "probe"}, "the point at %d carries the notes held: %s" % (g["rate"], c)) + code, doc, _, _, _ = run(env={"PROBE_CHILD_INSTRUMENT": "1"}) + valid(doc) + check(all("cost" not in g for g in doc["grid"]), "an instrument with no count to hold has no cost: %s" % doc["grid"]) + code, doc, _, _, _ = run(extra=("--hold", "8"), env={"PROBE_CHILD_INSTRUMENT": "1"}) + valid(doc) + check(doc["grid"][0]["cost"]["voices"] == {"held": 8, "basis": "probe"}, "--hold 8 is the count held: %s" % doc["grid"][0]) + # ---------------------------------------------------------------- LV2: the runner drives lv2_measure's passes LV2_CHILD = os.path.abspath(os.environ.get("PROBE_LV2_CHILD", os.path.join(ROOT, "tests", "lv2-probe-child"))) From 0470c7f3b9df4a4fac2c2953fbe2917675cf9464 Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Mon, 19 Oct 2026 21:14:52 +0200 Subject: [PATCH 3/6] probe: an LV2 point carries its note sweep, a CLAP point the sweep's counts (#106) lv2_measure prints rtSafety.noteSweep for a plugin with a MIDI input, and put_lv2_measure copied rtSafety without it, so plugin-qualify's judge could not read an LV2 instrument's note sweep from a probe output. One copy now serves both formats: whether the sweep applied, its dialect, and the allocations, locks and syscalls counted while the notes played. The CLAP point kept only the first two, so a CLAP instrument allocating on a note on read clean through the runner; it carries the counts too. --- README.md | 3 ++- src/probe.c | 31 +++++++++++++++++++++---------- tests/lv2_probe_child.c | 12 ++++++++++-- tests/probe_test.py | 23 +++++++++++++++++++++-- 4 files changed, 54 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 396e7d4..2b2906a 100644 --- a/README.md +++ b/README.md @@ -327,7 +327,8 @@ limits and the same wall clock. The first child (`lv2_measure --pin`) pins the b plugin. The second measures at the highest of `--rates`, in blocks of `--quantum`, with 30 instantiate and destroy cycles and 3 swept rt-safety runs. It runs under plugin-qualify's interposer (`rt_interpose.so` beside the child) and the noise stimulus. The document's binary is the plugin's library, with every pinned file of the bundle listed. The rate's grid -point carries what run() did under the interposer, and `behaviour` carries the lifecycles, the threads and the features +point carries what run() did under the interposer and, for a plugin with a MIDI input, its note sweep (what run() did +while notes played), and `behaviour` carries the lifecycles, the threads and the features the plugin requires. A plugin no bundle holds exits 3 with no document. A plugin of which nothing could be pinned exits 2 with no document, because the document has no binary to name. Its refusal is a code on stderr: `: : `, `probe.unpinnable` for a bundle the pin cannot cover. Version 1 runs the quick measurement only: the deep lifecycle pass diff --git a/src/probe.c b/src/probe.c index 0248067..9ed295d 100644 --- a/src/probe.c +++ b/src/probe.c @@ -1472,8 +1472,25 @@ static void put_count(sb_t *b, const jnode_t *o, const char *from, const char *t sb_printf(b, ", \"%s\": %.0f", to, v); } +/* a prober's note sweep, as a point carries it: whether it applied, the dialect it played in, and what process() or run() + * did while it played when the interposer counted */ +static void put_note_sweep(sb_t *b, const jnode_t *ns) +{ + sb_printf(b, ", \"noteSweep\": {\"applicable\": %s", jbool(ns, "applicable", 0) ? "true" : "false"); + if (jstr(ns, "dialect")) + { + sb_printf(b, ", \"dialect\": "); + sb_str(b, jstr(ns, "dialect")); + } + put_count(b, ns, "allocationsInRun", "allocations"); + put_count(b, ns, "locksInRun", "locks"); + put_count(b, ns, "syscallsInRun", "syscalls"); + sb_printf(b, "}"); +} + /* lv2_measure's measurement line: the grid point of its rate and block, with what run() did under the interposer, and - * the plugin's behaviour over the pass (lifecycles, threads, the features it requires); a translation, no judgement */ + * its note sweep, and the plugin's behaviour over the pass (lifecycles, threads, the features it requires); a + * translation, no judgement */ static void put_lv2_measure(sb_t *b, const jnode_t *m, long rate, unsigned quantum) { const jnode_t *rt = jget(m, "rtSafety"), *ev = jget(rt, "events"), *life = jget(m, "lifecycle"), *th = jget(m, "threads"), @@ -1505,6 +1522,8 @@ static void put_lv2_measure(sb_t *b, const jnode_t *m, long rate, unsigned quant sb_printf(b, "]"); } sb_printf(b, "}"); + if (jget(rt, "noteSweep")) + put_note_sweep(b, jget(rt, "noteSweep")); } sb_printf(b, "}\n ],\n \"behaviour\": {"); if (life) @@ -2036,15 +2055,7 @@ int main(int argc, char **argv) } sb_printf(&doc, "}"); if (notes) - { - sb_printf(&doc, ", \"noteSweep\": {\"applicable\": %s", jbool(notes, "applicable", 0) ? "true" : "false"); - if (jstr(notes, "dialect")) - { - sb_printf(&doc, ", \"dialect\": "); - sb_str(&doc, jstr(notes, "dialect")); - } - sb_printf(&doc, "}"); - } + put_note_sweep(&doc, notes); firstpoint = 0; } sb_printf(&doc, "}"); diff --git a/tests/lv2_probe_child.c b/tests/lv2_probe_child.c index 280656f..6928337 100644 --- a/tests/lv2_probe_child.c +++ b/tests/lv2_probe_child.c @@ -28,7 +28,8 @@ * loaded with dlopen. The plugin's own code runs here: a plugin that crashes, hangs or leaves a process behind in run() * (tests/lv2_fixture.c's "stress", STRESS_FAULT) does it in this process, as it would in lv2_measure. The layout pin is * a fixed digest: the runner copies it and computes none. A measurement refuses to run without OMX_STIM=noise, and - * reports the interposer as loaded when LD_PRELOAD names rt_interpose.so, so a test sees what the runner set. */ + * reports the interposer as loaded when LD_PRELOAD names rt_interpose.so, so a test sees what the runner set. Its note + * sweep applies when PROBE_CHILD_NOTE_ALLOCATIONS gives the allocations run() made while the notes played. */ #ifndef _GNU_SOURCE #define _GNU_SOURCE @@ -259,8 +260,15 @@ int main(int argc, char **argv) "\"threads\":{\"before\":%d,\"afterFirstInstantiate\":%d,\"afterAllFreed\":%d,\"leaked\":0}," "\"rtSafety\":{\"interposer\":%s,\"blocks\":64,\"repeats\":%d,\"swept\":false,\"sweepApplicable\":true," "\"allocationsInRun\":0,\"allocationsMin\":0,\"variable\":false,\"locksInRun\":0,\"locksMin\":0," - "\"syscallsInRun\":0,\"syscallsMin\":0,\"events\":[]}}\n", + "\"syscallsInRun\":0,\"syscallsMin\":0,\"events\":[]", uri, rate, block, cycles, instantiated, cycles - instantiated, before, after_first, threads(), getenv("LD_PRELOAD") && strstr(getenv("LD_PRELOAD"), "rt_interpose.so") ? "true" : "false", repeats); + if (getenv("PROBE_CHILD_NOTE_ALLOCATIONS")) + printf(",\"noteSweep\":{\"applicable\":true,\"dialect\":\"midi\",\"blocks\":64,\"blockFrames\":%u,\"noteEvents\":768," + "\"processErrors\":0,\"nonFinite\":false,\"interposer\":true,\"allocationsInRun\":%ld,\"locksInRun\":0," + "\"syscallsInRun\":0,\"events\":[]}", block, atol(getenv("PROBE_CHILD_NOTE_ALLOCATIONS"))); + else + printf(",\"noteSweep\":{\"applicable\":false}"); + printf("}}\n"); return 0; } diff --git a/tests/probe_test.py b/tests/probe_test.py index aebdef4..f1a38ae 100644 --- a/tests/probe_test.py +++ b/tests/probe_test.py @@ -243,12 +243,12 @@ def an_instrument_point_carries_the_notes_it_held_and_an_untimed_rate_has_no_cos REAL_LV2_MEASURE = "/usr/libexec/plugin-qualify/lv2_measure" -def run_lv2(fault=None, uri=LV2_STRESS, child=LV2_CHILD, wall=5, timeout=30, rates="48000,96000"): +def run_lv2(fault=None, uri=LV2_STRESS, child=LV2_CHILD, wall=5, timeout=30, rates="48000,96000", env=None, bundles=LV2_BUNDLES): """the runner over the LV2 stress fixture with the fake lv2_measure (or the real one)""" work = tempfile.mkdtemp(prefix="plugin-hostd-probe-lv2.") out = os.path.join(work, "probe.json") pidfile = os.path.join(work, "forkpid") - e = dict(os.environ, STRESS_FORK_PIDFILE=pidfile, LV2_PATH=LV2_BUNDLES) + e = dict(os.environ, STRESS_FORK_PIDFILE=pidfile, LV2_PATH=bundles, **(env or {})) if fault: e["STRESS_FAULT"] = fault else: @@ -288,6 +288,25 @@ def an_lv2_plugin_is_pinned_then_measured_at_the_highest_rate(): check("threads" in b and "required" in b["features"], "threads and features %s" % b) +@test +def an_lv2_point_carries_the_note_sweep_and_what_run_did_while_the_notes_played(): + code, doc, _, _, err = run_lv2(env={"PROBE_CHILD_NOTE_ALLOCATIONS": "7"}) + check(doc is not None, "no document: exit %d %s" % (code, err)) + valid(doc) + ns = doc["grid"][0].get("noteSweep") + check(ns == {"applicable": True, "dialect": "midi", "allocations": 7, "locks": 0, "syscalls": 0}, "noteSweep %s" % ns) + code, doc, _, _, _ = run_lv2() + valid(doc) + check(doc["grid"][0].get("noteSweep") == {"applicable": False}, "an effect's sweep does not apply: %s" % doc["grid"][0]) + + +@test +def a_clap_point_carries_the_note_sweep_counts(): + code, doc, _, _, _ = run(extra=("--probe-notes", "4"), env={"PROBE_CHILD_INSTRUMENT": "1", "PROBE_CHILD_NOTE_ALLOCATIONS": "3"}) + valid(doc) + ns = doc["grid"][0].get("noteSweep") + check(ns == {"applicable": True, "dialect": "clap", "allocations": 3, "locks": 0, "syscalls": 0}, "noteSweep %s" % ns) + @test def an_lv2_crash_keeps_the_pin_and_names_the_signal(): code, doc, _, _, err = run_lv2("crash") From 33dbd1950df8a57f9b5d7b4a9a2595f53d53db0a Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Mon, 19 Oct 2026 21:15:25 +0200 Subject: [PATCH 4/6] probe: a latency taken from a note says so, and whether the note sounded (#106) clap-qualify marks an instrument's latency as measured from a note, with heard false when nothing sounded. The point kept only the declared and measured frames under method impulse-onset, so an instrument read back looked like an impulse reading with an onset of a frame or more. The point's method is note-onset beside impulse-onset and declared-port, and heard is written on every note-onset point: within format 1.1. --- README.md | 4 +++- protocol/probe-output.schema.in | 7 ++++++- protocol/probe-output.schema.json | 7 ++++++- src/probe.c | 10 ++++++++-- tests/fixtures/probe/instrument-notes.json | 12 +++++++++--- tests/probe_schema.py | 4 ++++ tests/probe_test.py | 13 +++++++++++++ 7 files changed, 49 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 2b2906a..a8b5cb6 100644 --- a/README.md +++ b/README.md @@ -320,7 +320,9 @@ The runner never loads plugin code itself. A CLAP instrument is timed holding notes down: as many as the voice capacity it declares, or, when it declares none, the count `--probe-notes N` gives; `--hold N` holds that count whatever it declares. Both are passed to clap-qualify. Each rate's grid point carries the cost clap-qualify timed at that rate, in nanoseconds per sample, and for an instrument -`voices`, the notes it held and why that many (`declared` or `probe`). A rate it did not time has no cost. +`voices`, the notes it held and why that many (`declared` or `probe`). A rate it did not time has no cost. An instrument has +no input for the impulse, so its latency is measured from a note (`method: note-onset`), with `heard: false` at a rate +where the note did not sound. An LV2 plugin is measured the way plugin-qualify's LV2 hosting pass measures it today, in two children under the same limits and the same wall clock. The first child (`lv2_measure --pin`) pins the bundle's files without instantiating the diff --git a/protocol/probe-output.schema.in b/protocol/probe-output.schema.in index 186025c..04e24b1 100644 --- a/protocol/probe-output.schema.in +++ b/protocol/probe-output.schema.in @@ -745,8 +745,13 @@ "method": { "enum": [ "impulse-onset", - "declared-port" + "declared-port", + "note-onset" ] + }, + "heard": { + "description": "A note-onset latency's: whether the note sounded, so an onset that never came is not read as one.", + "type": "boolean" } } }, diff --git a/protocol/probe-output.schema.json b/protocol/probe-output.schema.json index 1946a76..641710e 100644 --- a/protocol/probe-output.schema.json +++ b/protocol/probe-output.schema.json @@ -745,8 +745,13 @@ "method": { "enum": [ "impulse-onset", - "declared-port" + "declared-port", + "note-onset" ] + }, + "heard": { + "description": "A note-onset latency's: whether the note sounded, so an onset that never came is not read as one.", + "type": "boolean" } } }, diff --git a/src/probe.c b/src/probe.c index 9ed295d..4a6d5bc 100644 --- a/src/probe.c +++ b/src/probe.c @@ -2011,12 +2011,18 @@ int main(int argc, char **argv) for (k = 0; k < latency->n && k < MAX_POINTS; k++) { double decl = 0, meas = 0; + int from_note; jnum(latency->kid[k], "declaredFrames", &decl); jnum(latency->kid[k], "measuredFrames", &meas); + /* an instrument has no input for the impulse: its onset is measured from a note, which may not sound */ + from_note = jstr(latency->kid[k], "from") && !strcmp(jstr(latency->kid[k], "from"), "note"); sb_printf(&doc, "%s\n {\"rate\": %ld, \"quantum\": %u, \"outcome\": \"%s\", \"latency\": {\"declaredFrames\": %.0f, " - "\"measuredFrames\": %.0f, \"method\": \"impulse-onset\"}", - k ? "," : "", atol(latency->key[k]), o.quantum, OUT_OK, decl, meas); + "\"measuredFrames\": %.0f, \"method\": \"%s\"", + k ? "," : "", atol(latency->key[k]), o.quantum, OUT_OK, decl, meas, from_note ? "note-onset" : "impulse-onset"); + if (from_note) + sb_printf(&doc, ", \"heard\": %s", jbool(latency->kid[k], "heard", 1) ? "true" : "false"); + sb_printf(&doc, "}"); put_cost(&doc, cost, latency->key[k]); if (firstpoint && rt) { diff --git a/tests/fixtures/probe/instrument-notes.json b/tests/fixtures/probe/instrument-notes.json index c35e890..6ec8057 100644 --- a/tests/fixtures/probe/instrument-notes.json +++ b/tests/fixtures/probe/instrument-notes.json @@ -231,7 +231,9 @@ "outcome": "probe.ok", "latency": { "declaredFrames": 0, - "measuredFrames": 1 + "measuredFrames": 1, + "method": "note-onset", + "heard": true }, "cost": { "unit": "ns-per-sample", @@ -257,7 +259,9 @@ "outcome": "probe.ok", "latency": { "declaredFrames": 0, - "measuredFrames": 1 + "measuredFrames": 1, + "method": "note-onset", + "heard": true }, "cost": { "unit": "ns-per-sample", @@ -282,7 +286,9 @@ "outcome": "probe.ok", "latency": { "declaredFrames": 0, - "measuredFrames": 1 + "measuredFrames": 1, + "method": "note-onset", + "heard": false } } ], diff --git a/tests/probe_schema.py b/tests/probe_schema.py index 31864de..65d5db7 100644 --- a/tests/probe_schema.py +++ b/tests/probe_schema.py @@ -74,6 +74,10 @@ def go(d): put(["grid", 1, "cost", "voices"], "basis", "guess"), "instrument-notes"), ("held notes that do not say why that many", "/grid/1/cost/voices", "required", drop(["grid", 1, "cost", "voices"], "basis"), "instrument-notes"), + ("a latency taken by a method the format has no name for", "/grid/0/latency/method", "enum", + put(["grid", 0, "latency"], "method", "note-guess"), "instrument-notes"), + ("whether the note was heard written as a word", "/grid/2/latency/heard", "type", + put(["grid", 2, "latency"], "heard", "no"), "instrument-notes"), ] diff --git a/tests/probe_test.py b/tests/probe_test.py index f1a38ae..6104e33 100644 --- a/tests/probe_test.py +++ b/tests/probe_test.py @@ -211,6 +211,19 @@ def every_point_carries_the_cost_timed_at_its_own_rate(): check(c == want, "the point at %d carries the figures timed at %d, and no voices for an effect: %s" % (g["rate"], g["rate"], c)) +@test +def a_latency_taken_from_a_note_says_so_and_whether_the_note_was_heard(): + rates = floor_rates() + silent = rates[-1] + code, doc, _, _, _ = run(rates=None, env={"PROBE_CHILD_INSTRUMENT": "1", "PROBE_CHILD_SILENT_RATE": str(silent)}) + valid(doc) + for g in doc["grid"]: + want = {"declaredFrames": 0, "measuredFrames": 1, "method": "note-onset", "heard": g["rate"] != silent} + check(g["latency"] == want, "the point at %d: %s, wanted %s" % (g["rate"], g["latency"], want)) + code, doc, _, _, _ = run() + check(doc["grid"][0]["latency"] == {"declaredFrames": 0, "measuredFrames": 0, "method": "impulse-onset"}, + "an effect's latency is an impulse's: %s" % doc["grid"][0]["latency"]) + @test def an_instrument_point_carries_the_notes_it_held_and_an_untimed_rate_has_no_cost(): rates = floor_rates() From beb1cf5c64dae9bf23cb6f2399910c1b9136190c Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Mon, 19 Oct 2026 21:17:16 +0200 Subject: [PATCH 5/6] test-probe: run plugin-qualify's own prober and lv2_measure when a build is named (#106) The stand-in children print the prober's shapes; these tests hold the runner to the real ones. With PLUGIN_QUALIFY_BUILD naming a plugin-qualify native/build, the CLAP synth is timed holding 32 notes at every rate of omx-contract's ORACLE_FLOOR_RATES, each point with its own cost, its notes and a latency from a note that sounded; with no count it has no cost at any rate; the synth declaring its voices holds that many. The LV2 synth that allocates on a note counts allocations in its note sweep, its clean twin none. --- BUILDING.md | 6 +++++ tests/probe_test.py | 66 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/BUILDING.md b/BUILDING.md index d8c6db1..7bbda61 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -97,6 +97,12 @@ omx-clap-host does not take them yet (a tempo, FreeMixer/omx-clap-core#88; the n case prints `PENDING` with the issue and checks what the worker answers today. Without `OMX_CLAP_HOST_SRC` the target prints `skip test-lv2-adapter` and does nothing else. +`make test-probe` runs the probe runner over `tests/probe-child` and `tests/lv2-probe-child`, stand-ins that print the +lines of plugin-qualify's prober and `lv2_measure`. With `PLUGIN_QUALIFY_BUILD=/native/build` +(`make -C native all` there) it also runs the real ones over plugin-qualify's instrument fixtures: a CLAP synth timed +holding `--probe-notes` at every rate of omx-contract's `ORACLE_FLOOR_RATES`, and an LV2 synth that allocates on a note +beside its clean twin. Without it those tests print `skip`. + `make sabotage` (`tests/sabotage.py`) breaks the daemon on purpose, one guard at a time, and requires the named test to go red. diff --git a/tests/probe_test.py b/tests/probe_test.py index 6104e33..b9ac48d 100644 --- a/tests/probe_test.py +++ b/tests/probe_test.py @@ -3,6 +3,8 @@ # in process() gives the named outcome, in a document that validates, and the whole tree is gone afterwards. # PROBE= PROBE_CHILD= STRESS_CLAP= probe_test.py # ONLY= runs one test (tests/sabotage.py turns each guard off and runs the test that must go red). +# PLUGIN_QUALIFY_BUILD= also runs its real prober and lv2_measure over its own +# instrument fixtures; without it those tests say they skipped. import json, os, signal, subprocess, sys, tempfile, time try: @@ -385,6 +387,70 @@ def the_real_lv2_measure_is_translated_when_it_is_installed(): "the measurement ran under plugin-qualify's interposer: %s %s" % (doc["run"]["tools"], doc["grid"][0]["rtSafety"])) +# ---------------------------------------------------------------- plugin-qualify's own prober and fixtures + +PQ_BUILD = os.environ.get("PLUGIN_QUALIFY_BUILD") + + +def pq(what): + """a file of the plugin-qualify build, or None (the test says it skipped) when no build is named""" + if not PQ_BUILD: + print("skip %s: PLUGIN_QUALIFY_BUILD names no plugin-qualify native/build" % what) + return None + return os.path.abspath(PQ_BUILD) + + +def run_pq_clap(build, plugin, extra=()): + work = tempfile.mkdtemp(prefix="plugin-hostd-probe-pq.") + out = os.path.join(work, "probe.json") + p = subprocess.run([PROBE, "run", "--child", os.path.join(build, "clap-qualify"), "--quantum", "512", "--wall-s", "300", + "--out", out, *extra, os.path.join(build, "fixtures", plugin)], capture_output=True, text=True, timeout=600) + return p.returncode, json.load(open(out)) if os.path.exists(out) else None, p.stderr + + +@test +def the_real_prober_times_an_instrument_holding_the_probe_notes_at_every_floor_rate(): + build = pq("the_real_prober_times_an_instrument_holding_the_probe_notes_at_every_floor_rate") + if not build: + return + rates = floor_rates() + code, doc, err = run_pq_clap(build, "synth.clap", ("--probe-notes", "32")) + check(doc is not None, "no document: exit %d %s" % (code, err)) + valid(doc) + check([g["rate"] for g in doc["grid"]] == rates, "a point per floor rate %s: %s" % (rates, [g["rate"] for g in doc["grid"]])) + for g in doc["grid"]: + c = g.get("cost", {}) + check(c.get("voices") == {"held": 32, "basis": "probe"} and c.get("median", 0) > 0 and c.get("blocks", 0) > 0, + "the point at %d carries the cost it was timed at, 32 notes held: %s" % (g["rate"], c)) + check(g["latency"].get("method") == "note-onset" and g["latency"].get("heard") is True, + "the point at %d: a latency from a note that sounded: %s" % (g["rate"], g["latency"])) + code, doc, err = run_pq_clap(build, "synth.clap") + valid(doc) + check(all("cost" not in g for g in doc["grid"]), "with no count to hold, no rate has a cost: %s" % doc["grid"]) + code, doc, err = run_pq_clap(build, "synth-voices.clap", ("--probe-notes", "32")) + valid(doc) + for g in doc["grid"]: + v = g.get("cost", {}).get("voices", {}) + check(v.get("basis") == "declared" and v.get("held") == v.get("capacity"), + "an instrument declaring its voices holds that many at %d: %s" % (g["rate"], v)) + + +@test +def the_real_lv2_measure_counts_what_run_does_while_notes_play(): + build = pq("the_real_lv2_measure_counts_what_run_does_while_notes_play") + if not build: + return + seen = {} + for uri in ("urn:plugin-qualify:fixture:lv2-synth-alloc", "urn:plugin-qualify:fixture:lv2-synth"): + code, doc, _, _, err = run_lv2(uri=uri, child=os.path.join(build, "lv2_measure"), wall=120, timeout=300, + bundles=os.path.join(build, "fixtures", "lv2")) + check(doc is not None, "%s: no document: exit %d %s" % (uri, code, err)) + valid(doc) + seen[uri] = doc["grid"][0].get("noteSweep", {}) + check(seen[uri].get("applicable") is True and seen[uri].get("dialect") == "midi", "%s noteSweep %s" % (uri, seen[uri])) + check(seen["urn:plugin-qualify:fixture:lv2-synth-alloc"].get("allocations", 0) > 0, "the allocating synth: %s" % seen) + check(seen["urn:plugin-qualify:fixture:lv2-synth"].get("allocations") == 0, "its clean twin: %s" % seen) + for t in tests: if ONLY and t.__name__ != ONLY: continue From 1c7fa82fb98c1c9c5202230484cda51c905f0c30 Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Mon, 19 Oct 2026 21:30:34 +0200 Subject: [PATCH 6/6] sabotage: break each copy of the runner's cost, notes and latency (#106) Twelve guards, each broken in a scratch copy with the test it must turn red: the two counts not passed to the prober, a point with no cost, with the first rate's or a neighbour's, the notes held left out, an LV2 point without its note sweep, a sweep without its allocations, a note-onset latency called an impulse's, a silent note heard, and the schema without voices or note-onset. --- tests/sabotage.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/sabotage.py b/tests/sabotage.py index 529b5f0..e7be1a9 100644 --- a/tests/sabotage.py +++ b/tests/sabotage.py @@ -440,6 +440,32 @@ ("the LV2 measurement runs without the noise stimulus", "src/probe.c", " setenv(PROBE_LV2_STIMULUS_ENV, PROBE_LV2_STIMULUS, 1);\n", "", "an_lv2_plugin_is_pinned_then_measured_at_the_highest_rate", "probe"), + ("--probe-notes does not reach the CLAP prober", "src/probe.c", " if (o->probe_notes)\n {", " if (0)\n {", + "the_notes_an_instrument_holds_reach_the_clap_child", "probe"), + ("--hold does not reach the CLAP prober", "src/probe.c", " if (o->hold)\n {", " if (0)\n {", + "the_notes_an_instrument_holds_reach_the_clap_child", "probe"), + ("a point carries no cost", "src/probe.c", " put_cost(&doc, cost, latency->key[k]);\n", "", + "every_point_carries_the_cost_timed_at_its_own_rate", "probe"), + ("every point carries the first rate's cost", "src/probe.c", " put_cost(&doc, cost, latency->key[k]);\n", + " put_cost(&doc, cost, latency->key[0]);\n", "every_point_carries_the_cost_timed_at_its_own_rate", "probe"), + ("a rate the prober did not time borrows its neighbour's cost", "src/probe.c", " put_cost(&doc, cost, latency->key[k]);\n", + " put_cost(&doc, cost, latency->key[k ? k - 1 : 0]);\n", + "an_instrument_point_carries_the_notes_it_held_and_an_untimed_rate_has_no_cost", "probe"), + ("a cost leaves out the notes held", "src/probe.c", ' if (jnum(voices, "held", &held) && jstr(voices, "basis"))', " if (0)", + "an_instrument_point_carries_the_notes_it_held_and_an_untimed_rate_has_no_cost", "probe"), + ("an LV2 point leaves out its note sweep", "src/probe.c", + ' if (jget(rt, "noteSweep"))\n put_note_sweep(b, jget(rt, "noteSweep"));\n', "", + "an_lv2_point_carries_the_note_sweep_and_what_run_did_while_the_notes_played", "probe"), + ("a note sweep leaves out its allocations", "src/probe.c", ' put_count(b, ns, "allocationsInRun", "allocations");\n', "", + "a_clap_point_carries_the_note_sweep_counts", "probe"), + ("a latency from a note is called an impulse's", "src/probe.c", 'from_note ? "note-onset" : "impulse-onset"', '"impulse-onset"', + "a_latency_taken_from_a_note_says_so_and_whether_the_note_was_heard", "probe"), + ("a note that did not sound is heard", "src/probe.c", 'jbool(latency->kid[k], "heard", 1) ? "true" : "false"', '"true"', + "a_latency_taken_from_a_note_says_so_and_whether_the_note_was_heard", "probe"), + ("the probe schema has no notes held in a cost", "protocol/probe-output.schema.json", ' "voices": {\n', ' "voices-": {\n', + "instrument-notes validates", "probeschema"), + ("the probe schema has no note-onset latency", "protocol/probe-output.schema.json", ' "note-onset"\n', ' "note-guess"\n', + "instrument-notes validates", "probeschema"), ("the probe schema accepts a verdict", "protocol/probe-output.schema.json", ' "verdict"\n', ' "verdict-"\n', "success-msi validates", "probeschema"), ("the probe schema does not require binary.sha256", "protocol/probe-output.schema.json",