probe-output.schema: the shared enums come from omx-contract's vocabulary (#115)

BUILDING.md

@@ -8,12 +8,19 @@ mod-host-protocol` when it is installed (mod-host's `make install-lib`),
 otherwise from `MOD_HOST_DIR`, a mod-host tree where the library is built if
 missing. The daemon needs libc and that library and nothing else.
 
-The probe runner and the protocol generator also read omx-contract's C header,
-`<omxcontract/omx_contract_limits.h>`, from `pkg-config omx-contract`
-(`omx-contract-devel`, `libomx-contract-dev`; `CONTRACT_CFLAGS=-I<dir>` for one
-that is not installed). The runner's default rates are the contract's
-`ORACLE_FLOOR_RATES`, and the generator refuses to build or run when the
-protocol's parameter scales are not the contract's `PARAM_SCALES`. `make
+The probe runner and the protocol generator also read omx-contract's C headers,
+`<omxcontract/omx_contract_limits.h>` and `<omxcontract/omx_vocabulary.h>`, from
+`pkg-config omx-contract` (`omx-contract-devel`, `libomx-contract-dev`, 2.18.0 or
+later; `CONTRACT_CFLAGS=-I<dir>` for one that is not installed). The runner's
+default rates are the contract's `ORACLE_FLOOR_RATES`, and the generator refuses
+to build or run when the protocol's parameter scales are not the contract's
+`PARAM_SCALES`. The probe output schema takes the words it shares with the
+contract (`MEASUREMENT_LATENCY_METHODS`, `MEASUREMENT_VOICE_BASES`,
+`MACHINE_CORE_BANDS`, `MACHINE_CLOCK_BANDS`, `RESOURCE_THREAD_SCHEDULING`) from
+the vocabulary header: `protocol/probe-output.schema.in` names each as an
+`@OMX_<ITEM>@` token and holds no list of its own. `make test-vocabulary` holds
+each of those enums of the generated schema to the installed contract's
+`omx-vocabulary.schema.json`, word for word and in order. `make
 test-contract` checks both, against the installed contract and against a copy
 of its header with the values changed. Nothing is linked: the header holds
 values only.

Makefile

@@ -134,7 +134,7 @@ clean:
 
 # the daemon against workers that are not plugin hosts at all (tests/fake-host speaks the same protocol and can be
 # made to die on cue): placement, forwarding, ledger, replay, attribution, the storm bound; no jack, no plugin
-test: test-pin test-relay test-alloc test-format test-daemon check-generated check-schema test-probe-schema test-probe test-contract test-consumer test-perturbation check-readme check-own-host-page check-no-pins check-ci-guards test-mod-host-json test-socket
+test: test-pin test-relay test-alloc test-format test-daemon check-generated check-schema test-probe-schema test-vocabulary test-probe test-contract test-consumer test-perturbation check-readme check-own-host-page check-no-pins check-ci-guards test-mod-host-json test-socket
 	MOD_HOST_DIR=$(MOD_HOST_DIR) python3 tests/verbs_contract.py
 
 # the same without the verb table, which is read from a mod-host checkout's README
@@ -292,6 +292,11 @@ test-mod-host-json:
 check-schema:
 	python3 tests/protocol_schema.py protocol/plugin-hostd.json protocol/plugin-hostd.schema.json
 
+# the probe output's words shared with omx-contract are the contract's: each enum the template takes from it matches the
+# installed omx-vocabulary.schema.json, and the template types none of the contract's lists by hand
+test-vocabulary:
+	python3 tests/vocabulary_derived.py
+
 # the probe runner over the stress plugin: crash, hang and a process left behind each give the named outcome in a document
 # that validates
 test-probe: $(PROBE) tests/probe-child tests/stress.clap tests/lv2-probe-child tests/rt_interpose.so $(LV2_FIXTURE)
@@ -307,8 +312,8 @@ tests/rt_interpose.so: tests/rt_interpose_stub.c
 tests/probe-child: tests/probe_child.c
 	$(CC) $(CLAP_CFLAGS) $(CFLAGS) -Wall -Wextra -Werror -o $@ $< -ldl
 
-# what is omx-contract's is read from it: the runner's default rates and the protocol's parameter scales follow a
-# scratch copy of its header with other values
+# what is omx-contract's is read from it: the runner's default rates, the protocol's parameter scales and the probe
+# schema's latency methods follow a scratch copy of its headers with other values
 test-contract: $(PROBE)
 	PROBE=./$(PROBE) python3 tests/contract_derived.py
 

debian/control

@@ -4,7 +4,7 @@ Priority: optional
 Maintainer: Pau Aliagas <linuxnow@gmail.com>
 Build-Depends: debhelper-compat (= 13),
                libmod-host-protocol-dev,
-               libomx-contract-dev (>= 2.16.0~),
+               libomx-contract-dev (>= 2.18.0~),
                omx-clap-host-tests (>= 0.9.0~) <!nocheck>,
                pkgconf,
                python3

packaging/plugin-hostd.spec

@@ -14,8 +14,9 @@ BuildRequires: make
 BuildRequires: diffutils
 BuildRequires: python3
 BuildRequires: pkgconfig(mod-host-protocol)
-# the probe runner's default rates and the protocol's parameter scales are read from omx-contract's C header; a floor only
-BuildRequires: omx-contract-devel >= 2.16.0
+# the probe runner's default rates, the protocol's parameter scales and the probe output's shared words are read from
+# omx-contract's C headers (2.18.0 is the first with omx_vocabulary.h); a floor only
+BuildRequires: omx-contract-devel >= 2.18.0
 # a command of exactly the socket's buffer size, or several commands in one read, needs mod-audio/mod-host#113:
 # FreeMixer's build carrying it (11.1.fmx) or audinux's once it carries it (12); a floor only
 Requires: mod-host-protocol%{?_isa} >= 0.10.6.f14a230-11.1

protocol/probe-output.schema.in

@@ -233,23 +233,10 @@
               "type": "string"
             },
             "coreBand": {
-              "enum": [
-                "1-2",
-                "3-4",
-                "5-8",
-                "9-16",
-                "17-32",
-                "33+"
-              ]
+              "enum": [@OMX_MACHINE_CORE_BANDS@]
             },
             "clockBand": {
-              "enum": [
-                "<1.5GHz",
-                "1.5-2.5GHz",
-                "2.5-3.5GHz",
-                "3.5-4.5GHz",
-                "4.5GHz+"
-              ]
+              "enum": [@OMX_MACHINE_CLOCK_BANDS@]
             },
             "preemptRt": {
               "type": "boolean"
@@ -399,10 +386,7 @@
           "type": "object"
         },
         "realtime": {
-          "enum": [
-            "none",
-            "fifo"
-          ]
+          "enum": [@OMX_RESOURCE_THREAD_SCHEDULING@]
         }
       }
     },
@@ -743,11 +727,7 @@
               "minimum": 0
             },
             "method": {
-              "enum": [
-                "impulse-onset",
-                "declared-port",
-                "note-onset"
-              ]
+              "enum": [@OMX_MEASUREMENT_LATENCY_METHODS@]
             },
             "heard": {
               "description": "A note-onset latency's: whether the note sounded, so an onset that never came is not read as one.",
@@ -840,10 +820,7 @@
                   "minimum": 1
                 },
                 "basis": {
-                  "enum": [
-                    "declared",
-                    "probe"
-                  ]
+                  "enum": [@OMX_MEASUREMENT_VOICE_BASES@]
                 },
                 "capacity": {
                   "type": "integer",

protocol/probe-output.schema.json

@@ -233,23 +233,10 @@
               "type": "string"
             },
             "coreBand": {
-              "enum": [
-                "1-2",
-                "3-4",
-                "5-8",
-                "9-16",
-                "17-32",
-                "33+"
-              ]
+              "enum": ["1-2", "3-4", "5-8", "9-16", "17-32", "33+"]
             },
             "clockBand": {
-              "enum": [
-                "<1.5GHz",
-                "1.5-2.5GHz",
-                "2.5-3.5GHz",
-                "3.5-4.5GHz",
-                "4.5GHz+"
-              ]
+              "enum": ["<1.5GHz", "1.5-2.5GHz", "2.5-3.5GHz", "3.5-4.5GHz", "4.5GHz+"]
             },
             "preemptRt": {
               "type": "boolean"
@@ -399,10 +386,7 @@
           "type": "object"
         },
         "realtime": {
-          "enum": [
-            "none",
-            "fifo"
-          ]
+          "enum": ["none", "fifo"]
         }
       }
     },
@@ -743,11 +727,7 @@
               "minimum": 0
             },
             "method": {
-              "enum": [
-                "impulse-onset",
-                "declared-port",
-                "note-onset"
-              ]
+              "enum": ["impulse-onset", "declared-port", "note-onset"]
             },
             "heard": {
               "description": "A note-onset latency's: whether the note sounded, so an onset that never came is not read as one.",
@@ -840,10 +820,7 @@
                   "minimum": 1
                 },
                 "basis": {
-                  "enum": [
-                    "declared",
-                    "probe"
-                  ]
+                  "enum": ["declared", "probe"]
                 },
                 "capacity": {
                   "type": "integer",

src/probe.c

@@ -67,6 +67,7 @@
 #include <unistd.h>
 
 #include <omxcontract/omx_contract_limits.h>
+#include <omxcontract/omx_vocabulary.h>
 #include <plugin-hostd/pin.h>
 #include <plugin-hostd/protocol.h>
 
@@ -731,8 +732,13 @@ static void put_machine(sb_t *b, const cpu_set_t *pinned, int pinned_by_option)
         khz = atol(text);
     if (read_text("/sys/kernel/realtime", text, sizeof(text)) == 0)
         preempt_rt = atoi(text) == 1;
-    core_band = ncpu <= 2 ? "1-2" : ncpu <= 4 ? "3-4" : ncpu <= 8 ? "5-8" : ncpu <= 16 ? "9-16" : ncpu <= 32 ? "17-32" : "33+";
-    clock_band = khz < 1500000 ? "<1.5GHz" : khz < 2500000 ? "1.5-2.5GHz" : khz < 3500000 ? "2.5-3.5GHz" : khz < 4500000 ? "3.5-4.5GHz" : "4.5GHz+";
+    /* the bands are omx-contract's MACHINE_CORE_BANDS and MACHINE_CLOCK_BANDS */
+    core_band = ncpu <= 2 ? OMX_MACHINE_CORE_BANDS_CORES1_TO2 : ncpu <= 4 ? OMX_MACHINE_CORE_BANDS_CORES3_TO4 :
+                ncpu <= 8 ? OMX_MACHINE_CORE_BANDS_CORES5_TO8 : ncpu <= 16 ? OMX_MACHINE_CORE_BANDS_CORES9_TO16 :
+                ncpu <= 32 ? OMX_MACHINE_CORE_BANDS_CORES17_TO32 : OMX_MACHINE_CORE_BANDS_CORES33_UP;
+    clock_band = khz < 1500000 ? OMX_MACHINE_CLOCK_BANDS_UNDER1500_MHZ : khz < 2500000 ? OMX_MACHINE_CLOCK_BANDS_FROM1500_TO2500_MHZ :
+                 khz < 3500000 ? OMX_MACHINE_CLOCK_BANDS_FROM2500_TO3500_MHZ :
+                 khz < 4500000 ? OMX_MACHINE_CLOCK_BANDS_FROM3500_TO4500_MHZ : OMX_MACHINE_CLOCK_BANDS_FROM4500_MHZ;
     for (i = 0; i < g_nidle; i++)
     {
         if (g_idle_after[i].latency_us > DEEP_IDLE_US && !g_idle_after[i].disabled)
@@ -2071,7 +2077,7 @@ int main(int argc, char **argv)
             }
             sb_printf(&doc, "]");
         }
-        sb_printf(&doc, ", \"realtime\": \"none\"},\n  \"limits\": {\"set\": ");
+        sb_printf(&doc, ", \"realtime\": \"" OMX_RESOURCE_THREAD_SCHEDULING_NONE "\"},\n  \"limits\": {\"set\": ");
         sb_str(&doc, o.set);
         sb_printf(&doc, ",\n    \"applied\": [");
         for (i = 0; i < g_napplied; i++)
@@ -2115,7 +2121,8 @@ int main(int argc, char **argv)
                 from_note = jstr(latency->kid[k], "from") && !strcmp(jstr(latency->kid[k], "from"), "note");
                 sb_printf(&doc, "%s\n    {\"rate\": %ld, \"quantum\": %u, \"outcome\": \"%s\", \"latency\": {\"declaredFrames\": %.0f, "
                                 "\"measuredFrames\": %.0f, \"method\": \"%s\"",
-                          k ? "," : "", atol(latency->key[k]), o.quantum, OUT_OK, decl, meas, from_note ? "note-onset" : "impulse-onset");
+                          k ? "," : "", atol(latency->key[k]), o.quantum, OUT_OK, decl, meas,
+                          from_note ? OMX_MEASUREMENT_LATENCY_METHODS_NOTE_ONSET : OMX_MEASUREMENT_LATENCY_METHODS_IMPULSE_ONSET);
                 if (from_note)
                     sb_printf(&doc, ", \"heard\": %s", jbool(latency->kid[k], "heard", 1) ? "true" : "false");
                 sb_printf(&doc, "}");

tests/contract_derived.py

@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 # What plugin-hostd shares with omx-contract is read from omx-contract, never typed again here: the probe runner's
-# default rates are ORACLE_FLOOR_RATES, and the parameter scales of the protocol are PARAM_SCALES, the same words in
-# the same order. Checked twice: against the installed contract (its data and its C header), and against a scratch
+# default rates are ORACLE_FLOOR_RATES, the parameter scales of the protocol are PARAM_SCALES, the same words in
+# the same order, and the probe output's shared enums are the words of omx_vocabulary.h. Checked twice: against the installed contract (its data and its C header), and against a scratch
 # copy of the header with the values changed, where the runner must follow and the generator must refuse to build.
 # Usage: contract_derived.py (run from the tree; CC, PKG_CONFIG and PROBE from the environment, or their defaults).
 import json, os, re, shutil, subprocess, sys, tempfile
@@ -37,13 +37,13 @@ def contract_data(name):
     raise SystemExit("omx-contract declares no %s" % name)
 
 
-def header():
+def header(name="omx_contract_limits.h"):
     for flag in pkg("--cflags").split() + ["-I/usr/include"]:
         if flag.startswith("-I"):
-            path = os.path.join(flag[2:], "omxcontract", "omx_contract_limits.h")
+            path = os.path.join(flag[2:], "omxcontract", name)
             if os.path.exists(path):
                 return path
-    raise SystemExit("omx-contract's C header is not installed")
+    raise SystemExit("omx-contract's C header %s is not installed" % name)
 
 
 def child_rates(probe, tmp):
@@ -84,6 +84,15 @@ def gen_json(gen):
     return p.returncode, p.stdout, p.stderr
 
 
+def gen_methods(gen):
+    """the latency methods of the probe schema the generator writes from this tree's template"""
+    p = subprocess.run([gen, "probe-schema", os.path.join(ROOT, "protocol", "probe-output.schema.in")], capture_output=True,
+                       text=True, timeout=30)
+    if p.returncode != 0:
+        return None
+    return json.loads(p.stdout)["$defs"]["point"]["properties"]["latency"]["properties"]["method"]["enum"]
+
+
 def main():
     floor = ",".join(str(r) for r in contract_data("ORACLE_FLOOR_RATES")["values"])
     scales = contract_data("PARAM_SCALES")["ids"]
@@ -142,6 +151,30 @@ def main():
         built = build_gen(scratch, os.path.join(tmp, "gen-swapped"))
         check(built.returncode != 0 and "PARAM_SCALES" in built.stderr,
               "the generator refuses scales the contract orders otherwise: rc %d %s" % (built.returncode, built.stderr[-300:]))
+
+        # a scratch vocabulary whose latency methods have a word more: the probe schema the generator writes has it
+        # too, and a word list longer than its count does not build
+        vocab = os.path.join(tmp, "vocab")
+        os.makedirs(os.path.join(vocab, "omxcontract"))
+        shutil.copy(header(), os.path.join(vocab, "omxcontract"))
+        vtext = open(header("omx_vocabulary.h")).read()
+        grown = re.sub(r'(#define OMX_MEASUREMENT_LATENCY_METHODS_INIT \{[^}]*)\s\}', r'\1, "x-onset" }', vtext)
+        check(grown != vtext, "positive control: the scratch vocabulary's latency methods were changed")
+        counted = re.sub(r"(#define OMX_MEASUREMENT_LATENCY_METHODS_COUNT )([0-9]+)u",
+                         lambda m: "%s%du" % (m.group(1), int(m.group(2)) + 1), grown)
+        with open(os.path.join(vocab, "omxcontract", "omx_vocabulary.h"), "w") as f:
+            f.write(counted)
+        built = build_gen(vocab, os.path.join(tmp, "gen-vocab"))
+        check(built.returncode == 0, "the generator builds against the scratch vocabulary: %s" % built.stderr[-400:])
+        if built.returncode == 0:
+            methods = gen_methods(os.path.join(tmp, "gen-vocab"))
+            check(methods is not None and methods[-1:] == ["x-onset"],
+                  "a word the contract adds to the latency methods is in the probe schema: %s" % methods)
+        with open(os.path.join(vocab, "omxcontract", "omx_vocabulary.h"), "w") as f:
+            f.write(grown)
+        built = build_gen(vocab, os.path.join(tmp, "gen-vocab-uncounted"))
+        check(built.returncode != 0 and "MEASUREMENT_LATENCY_METHODS" in built.stderr,
+              "the generator refuses a word list that is not its count: rc %d %s" % (built.returncode, built.stderr[-300:]))
     finally:
         shutil.rmtree(tmp, ignore_errors=True)
     print("contract: %s" % ("FAILED (%d)" % failures if failures else "ok"))