Declare the daemon's verbs in its contract directory, ship it in plugin-hostd-contract (#131)

.github/workflows/ci.yml

@@ -217,22 +217,34 @@ jobs:
             rpm -qp --requires "$rpm" | tee requires.txt
             grep -q 'libmod-host-protocol.so.0' requires.txt
             grep -qx 'mod-host-protocol(.*) >= 0.10.6.f14a230-11.1' requires.txt
+            grep -qx "plugin-hostd-contract = $(rpm -qp --qf '%{VERSION}-%{RELEASE}' "$rpm")" requires.txt
             rpm -qp --recommends "$rpm" | tee recommends.txt
             grep -q '^mod-host' recommends.txt
             grep -q '^omx-clap-host' recommends.txt
           done
-          # the development package: the header a consumer includes, its pkg-config file, the schema of the JSON
+          # the development package: the headers a consumer includes, its pkg-config file, the schema of the JSON
           for rpm in rpms/plugin-hostd-devel-[0-9]*.noarch.rpm; do
             rpm -qlp "$rpm"
             rpm -qp --qf '[%{FILEMODES:perms} %{FILENAMES}\n]' "$rpm" | grep -v '^d' | cut -d' ' -f2 | sort > files.txt
             printf '%s\n' /usr/include/plugin-hostd/protocol.h /usr/include/plugin-hostd/pin.h /usr/share/pkgconfig/plugin-hostd.pc \
+              /usr/include/plugin-hostd/plugin_hostd_host_verbs.h /usr/include/plugin-hostd/plugin_hostd_vocabulary.h \
               /usr/share/plugin-hostd/protocol.schema.json /usr/share/plugin-hostd/probe-output.schema.json /usr/share/plugin-hostd/preset.schema.json \
-              /usr/share/omx-contract/contract.d/3/plugin-hostd/MANIFEST.json \
+              /usr/share/licenses/plugin-hostd-devel/COPYING | sort > expected.txt
+            diff -u expected.txt files.txt
+            rpm -qp --requires "$rpm" | grep -qx "plugin-hostd-contract = $(rpm -qp --qf '%{VERSION}-%{RELEASE}' "$rpm")"
+          done
+          # the contract package: the contract directory's files, which readers read at run time, noarch
+          for rpm in rpms/plugin-hostd-contract-[0-9]*.noarch.rpm; do
+            rpm -qlp "$rpm"
+            rpm -qp --qf '[%{FILEMODES:perms} %{FILENAMES}\n]' "$rpm" | grep -v '^d' | cut -d' ' -f2 | sort > files.txt
+            printf '%s\n' /usr/share/omx-contract/contract.d/3/plugin-hostd/MANIFEST.json \
               /usr/share/omx-contract/contract.d/3/plugin-hostd/formats/clap.json \
               /usr/share/omx-contract/contract.d/3/plugin-hostd/formats/lv2.json \
-              /usr/share/licenses/plugin-hostd-devel/COPYING | sort > expected.txt
+              /usr/share/omx-contract/contract.d/3/plugin-hostd/verbs.json \
+              /usr/share/licenses/plugin-hostd-contract/COPYING | sort > expected.txt
             diff -u expected.txt files.txt
           done
+          ls rpms/plugin-hostd-contract-[0-9]*.noarch.rpm
           # the tests package: the fake worker, where plugin-qualify's pins-file test finds it
           for rpm in rpms/plugin-hostd-tests-[0-9]*.${{ matrix.arch }}.rpm; do
             rpm -qlp "$rpm"
@@ -257,6 +269,7 @@ jobs:
             mod-host-rpms/mod-host-[0-9]*.rpm
             mod-host-rpms/mod-host-protocol-[0-9]*.rpm
             rpms/plugin-hostd-[0-9]*.${{ matrix.arch }}.rpm
+            rpms/plugin-hostd-contract-[0-9]*.noarch.rpm
             rpms/plugin-hostd-tests-[0-9]*.${{ matrix.arch }}.rpm
           packages: python3
 
@@ -335,6 +348,14 @@ jobs:
 
       - uses: FreeMixer/.github/.github/actions/build-deb@v1
 
+      # the data packages plugin-hostd-contract depends on are the channel's, not Debian's: the clean container
+      # installs them from here, like the protocol library
+      - name: Fetch the contract data packages of the channel
+        run: |
+          set -euo pipefail
+          mkdir -p dep-debs
+          (cd dep-debs && apt-get download omx-clap-core-contract omx-contract-data)
+
       # what the smoke test runs the installed package against, built here for the architecture
       - name: Build the smoke test fixtures
         run: |
@@ -350,6 +371,8 @@ jobs:
           path: |
             debs/*.deb
             dep-debs/libmod-host-protocol0_*.deb
+            dep-debs/omx-clap-core-contract_*.deb
+            dep-debs/omx-contract-data_*.deb
             smoke
           if-no-files-found: error
           retention-days: 1
@@ -379,11 +402,25 @@ jobs:
         run: |
           set -euo pipefail
           apt-get update -qq
-          apt-get install -y --no-install-recommends ./debs/plugin-hostd_*.deb ./debs/plugin-hostd-dev_*.deb ./debs/plugin-hostd-tests_*.deb ./dep-debs/libmod-host-protocol0_*.deb python3
+          apt-get install -y --no-install-recommends ./debs/plugin-hostd_*.deb ./debs/plugin-hostd-contract_*.deb ./debs/plugin-hostd-dev_*.deb ./debs/plugin-hostd-tests_*.deb ./dep-debs/libmod-host-protocol0_*.deb ./dep-debs/omx-clap-core-contract_*.deb ./dep-debs/omx-contract-data_*.deb python3
           dpkg -L plugin-hostd-dev | tee dev.txt
           grep -qx /usr/include/plugin-hostd/protocol.h dev.txt
+          grep -qx /usr/include/plugin-hostd/plugin_hostd_host_verbs.h dev.txt
+          grep -qx /usr/include/plugin-hostd/plugin_hostd_vocabulary.h dev.txt
           grep -qx /usr/include/plugin-hostd/pin.h dev.txt
           grep -qx /usr/share/pkgconfig/plugin-hostd.pc dev.txt
+          ! grep -q /usr/share/omx-contract/ dev.txt
+          # the contract directory's files are the noarch contract package's, which the daemon and the headers depend on
+          test "$(dpkg-deb -f debs/plugin-hostd-contract_*.deb Architecture)" = all
+          dpkg -L plugin-hostd-contract | tee contract.txt
+          for f in MANIFEST.json formats/clap.json formats/lv2.json verbs.json; do
+            grep -qx "/usr/share/omx-contract/contract.d/3/plugin-hostd/$f" contract.txt
+          done
+          dpkg-deb -f debs/plugin-hostd-contract_*.deb Breaks Replaces | tee breaks.txt
+          test "$(grep -c '^\(Breaks\|Replaces\): plugin-hostd-dev (<< ' breaks.txt)" = 2
+          for p in plugin-hostd plugin-hostd-dev; do
+            dpkg-deb -f debs/${p}_*.deb Depends | grep -q 'plugin-hostd-contract (= '
+          done
           dpkg -L plugin-hostd | grep -qx /usr/share/plugin-hostd/protocol.json
           dpkg -L plugin-hostd | grep -qx /usr/share/plugin-hostd/mod-host.json
           dpkg -L plugin-hostd-tests | tee tests.txt

BUILDING.md

@@ -10,7 +10,7 @@ missing. The daemon needs libc and that library and nothing else.
 
 The probe runner and the protocol generator also read omx-contract's C headers,
 `<omxcontract/omx_contract_limits.h>` and `<omxcontract/omx_vocabulary.h>`, from
-`pkg-config omx-contract` (`omx-contract-devel`, `libomx-contract-dev`, 2.18.0 or
+`pkg-config omx-contract` (`omx-contract-devel`, `libomx-contract-dev`, 3.3.0 or
 later; `CONTRACT_CFLAGS=-I<dir>` for one that is not installed). The runner's
 default rates are the contract's `ORACLE_FLOOR_RATES`, and the generator refuses
 to build or run when the protocol's parameter scales are not the contract's
@@ -46,6 +46,23 @@ The tables in README.md marked `BEGIN GENERATED protocol:...`, the key list of t
 headers the build compiles against (`pkg-config --cflags mod-host-protocol`, or `MOD_HOST_DIR/src`), by
 `tools/mod-host-json.py`; `make test-mod-host-json` checks that it follows them.
 
+The daemon's verbs
+------------------
+
+The verbs the daemon adds to mod-host's protocol, and the codes they answer, are declared in `data/verbs.json`, a
+`verbs` item of omx-contract's contract directory (schema 3.3.0 or later). `omx-contract render` writes them into
+`include/plugin-hostd/plugin_hostd_host_verbs.h` and `plugin_hostd_vocabulary.h`, which are committed and which
+`protocol.h` includes: `make render-verbs` rewrites them and `make check-rendered` fails on any drift. Both run the
+`omx-contract` command (package `omx-contract`); the build itself does not. A refusal number omx-clap-host answers
+too (mod-host's `-3` and `-902`, and the pin, parameter and note numbers) is named once, by omx-clap-core's
+`reply.host.*` codes: `data/verbs.json` requires `omx-clap-core >= 0.22.0`, so both commands and
+`make test-verbs-declared` read omx-clap-core's package of the contract directory (`omx-clap-core-contract`), or
+the first directory of `OMX_CONTRACT_PATH` that holds one.
+
+`make install-contract` installs `data/` as the package `plugin-hostd` of the contract directory. The noarch package
+`plugin-hostd-contract` carries those files, and `plugin-hostd` and `plugin-hostd-devel` (`plugin-hostd-dev`) require
+it at the same version.
+
 Tests
 -----
 
@@ -59,9 +76,9 @@ ledger, replay to the byte, attribution, quarantine, the storm bound,
 
 The verbs are read from their declarations, never spelled: mod-host's from the
 command formats of `mod-host.h`, the daemon's own from
-`include/plugin-hostd/protocol.h`. `tests/verbs_contract.py` fails on a C string
+`data/verbs.json` through the rendered header. `tests/verbs_contract.py` fails on a C string
 that starts with one, and `tests/perturbation.py` renames a verb in a scratch
-copy of each header and requires the rebuilt daemon to answer the new name and
+copy of `data/verbs.json` and of `mod-host.h` and requires the rebuilt daemon to answer the new name and
 not the old.
 
     make test-jack OMX_CLAP_HOST=<omx-clap-host> [MOD_HOST=<mod-host> LV2_DIR=<lv2 path> LV2_URI=<a stereo effect> LV2_BUNDLE=<its bundle dir> LV2_PARAM=<a control taking 0.25>]

Makefile

@@ -60,6 +60,10 @@ SRC = src/main.c src/conf.c src/proc.c src/supervisor.c src/verbs.c src/pins.c s
       src/descriptors.c src/describe.c src/transforms.c src/json.c
 OBJ = $(SRC:.c=.o)
 
+# the C names of the daemon's verbs and reply codes, rendered from data/verbs.json (render-verbs below); protocol.h
+# includes them
+RENDERED = include/plugin-hostd/plugin_hostd_host_verbs.h include/plugin-hostd/plugin_hostd_vocabulary.h
+
 # default build
 all: $(PROG) $(PROBE)
 
@@ -69,7 +73,7 @@ $(PROG): $(OBJ) $(PROTOCOL_LIB)
 # the probe runner: it loads no plugin and links nothing of the daemon or of mod-host
 # describe reads the installed table of a format by default (docs/formats.md)
 PROBE_SRC = src/probe.c src/json.c src/describe.c src/transforms.c
-$(PROBE): $(PROBE_SRC) src/json.h src/describe.h src/transforms.h include/plugin-hostd/protocol.h include/plugin-hostd/pin.h
+$(PROBE): $(PROBE_SRC) src/json.h src/describe.h src/transforms.h include/plugin-hostd/protocol.h $(RENDERED) include/plugin-hostd/pin.h
 	$(CC) -Iinclude -Isrc $(CONTRACT_CFLAGS) $(CFLAGS) -D_GNU_SOURCE -DPROBE_VERSION='"$(VERSION)"' \
 	  -DPROBE_CONTRACTDIR='"$(CONTRACTDIR)"' -DPROBE_CONTRACT_MAJOR='"$(CONTRACT_SCHEMA_MAJOR)"' -o $@ $(PROBE_SRC) -lm
 
@@ -79,7 +83,7 @@ $(PROTOCOL_LIB):
 endif
 
 # the declared protocol is compiled into every object
-$(OBJ): include/plugin-hostd/protocol.h include/plugin-hostd/pin.h
+$(OBJ): include/plugin-hostd/protocol.h $(RENDERED) include/plugin-hostd/pin.h
 
 src/descriptors.o: src/descriptors.h src/describe.h src/json.h src/formats.h src/supervisor.h
 
@@ -115,14 +119,15 @@ install_protocol:
 	install -d $(DESTDIR)$(DATADIR)/plugin-hostd/pins.d
 	install -d $(DESTDIR)$(INCLUDEDIR)/plugin-hostd
 	install -m 644 include/plugin-hostd/protocol.h $(DESTDIR)$(INCLUDEDIR)/plugin-hostd/protocol.h
+	install -m 644 $(RENDERED) $(DESTDIR)$(INCLUDEDIR)/plugin-hostd/
 	install -m 644 include/plugin-hostd/pin.h $(DESTDIR)$(INCLUDEDIR)/plugin-hostd/pin.h
 	install -d $(DESTDIR)$(DATADIR)/pkgconfig
 	sed -e 's,@PREFIX@,$(PREFIX),g' -e 's,@VERSION@,$(VERSION),g' protocol/plugin-hostd.pc.in > $(DESTDIR)$(DATADIR)/pkgconfig/plugin-hostd.pc
 	chmod 644 $(DESTDIR)$(DATADIR)/pkgconfig/plugin-hostd.pc
 
-# the format tables this tree owns (data/formats/, docs/formats.md), installed as the package plugin-hostd into
-# omx-contract's contract directory, <contractdir>/<schema major>/plugin-hostd/formats/{clap,lv2}.json, with a
-# MANIFEST.json naming this version and the tree they came from; plugin-hostd-devel ships them. contractdir and the
+# the format tables (data/formats/, docs/formats.md) and the verbs (data/verbs.json) this tree owns, installed as the
+# package plugin-hostd into omx-contract's contract directory, <contractdir>/<schema major>/plugin-hostd/, with a
+# MANIFEST.json naming this version and the tree they came from; plugin-hostd-contract ships them. contractdir and the
 # install script are omx-contract-devel's; a checkout of omx-contract (OMX_CONTRACT_DIR) stands in for the script when
 # it is not installed.
 CONTRACT_SCHEMA_MAJOR = 3
@@ -133,6 +138,19 @@ install-contract:
 	@test -n "$(INSTALL_CONTRACT_DIR)" || { echo "install-contract: no install-contract-dir.sh: install omx-contract-devel, or set OMX_CONTRACT_DIR to a checkout" >&2; exit 1; }
 	sh $(INSTALL_CONTRACT_DIR) $(PROG) $(VERSION) $(CONTRACT_SCHEMA_MAJOR) data $(DESTDIR)$(CONTRACTDIR) data
 
+# the daemon's verbs and reply codes (data/verbs.json), rendered by omx-contract as the package installs them, beside
+# the packages of OMX_CONTRACT_PATH and of the installed contract directory: the C names every source and protocol.h
+# read (OMX_VERB_<VERB>, its _FMT, _REPLY and _CODES_INIT). `make render-verbs` writes them, `make check-rendered`
+# fails on any drift.
+OMX_CONTRACT ?= omx-contract
+.PHONY: render-verbs check-rendered
+render-verbs check-rendered:
+	@test -n "$(INSTALL_CONTRACT_DIR)" || { echo "$@: no install-contract-dir.sh: install omx-contract-devel, or set OMX_CONTRACT_DIR to a checkout" >&2; exit 1; }
+	@set -e; tmp=$$(mktemp -d); trap 'rm -rf $$tmp' EXIT; \
+	sh $(INSTALL_CONTRACT_DIR) $(PROG) $(VERSION) $(CONTRACT_SCHEMA_MAJOR) data $$tmp data >/dev/null; \
+	OMX_CONTRACT_PATH=$$tmp$(if $(OMX_CONTRACT_PATH),:$(OMX_CONTRACT_PATH)):$(CONTRACTDIR) \
+	  $(OMX_CONTRACT) render --target c --package $(PROG) --installed --out include/plugin-hostd $(if $(filter check-rendered,$@),--check)
+
 # the tables cover their format (tests/format_coverage.py, docs/formats.md#the-coverage-test): every term of
 # omx-contract's installed TERM files and every row with a from, once per format, each field one row; then its sabotage
 # arms, each red naming its field. CONTRACT_SCHEMA_MAJOR=<n> reads the TERM files of another major
@@ -178,7 +196,7 @@ FUZZ_INCS      := -Iinclude -Isrc -Ifuzz $(PROTOCOL_CFLAGS)
 # reply harness includes src/relay.c itself
 FUZZ_SRC_command := src/verbs.c fuzz/recorder.c
 FUZZ_SRC_reply   :=
-FUZZ_DEPS        := fuzz/fuzz.h fuzz/syntax.h fuzz/recorder.h src/relay.c src/verbs.h include/plugin-hostd/protocol.h
+FUZZ_DEPS        := fuzz/fuzz.h fuzz/syntax.h fuzz/recorder.h src/relay.c src/verbs.h include/plugin-hostd/protocol.h $(RENDERED)
 
 $(FUZZ_BUILD)/fuzz_%: fuzz/fuzz_%.c $(FUZZ_DEPS) $(FUZZ_SRC_command)
 	@mkdir -p $(dir $@)
@@ -236,13 +254,19 @@ clean:
 
 # the daemon against workers that are not plugin hosts at all (tests/fake-host speaks the same protocol and can be
 # made to die on cue): placement, forwarding, ledger, replay, attribution, the storm bound; no jack, no plugin
-test: test-pin test-transforms test-format-coverage test-describe test-relay test-alloc test-format test-daemon check-generated check-schema test-probe-schema test-vocabulary test-probe test-contract test-consumer test-perturbation check-readme check-own-host-page check-no-pins check-ci-guards test-mod-host-json test-socket check-verb-table check-preset-schema test-install-schemas
+test: test-pin test-transforms test-format-coverage test-describe test-relay test-alloc test-format test-daemon test-verbs-declared check-rendered check-generated check-schema test-probe-schema test-vocabulary test-probe test-contract test-consumer test-perturbation check-readme check-own-host-page check-no-pins check-ci-guards test-mod-host-json test-socket check-verb-table check-preset-schema test-install-schemas
 	MOD_HOST_DIR=$(MOD_HOST_DIR) python3 tests/verbs_contract.py
 
 # the same without the verb table, which is read from a mod-host checkout's README
 test-daemon: $(PROG) $(FAKE)
 	PLUGIN_HOSTD=./$(PROG) FAKE_HOST=./$(FAKE) python3 tests/daemon_test.py
 
+# the declared verbs on the daemon's socket (tests/verbs_declared.py): each answers its declared reply and codes, the
+# daemon answers no undeclared word itself, and a verb both hosts owe answers in one shape from both; omx-clap-core's
+# declarations come from OMX_CONTRACT_PATH or the installed contract directory
+test-verbs-declared: $(PROG) $(PROBE) $(FAKE)
+	PLUGIN_HOSTD=./$(PROG) PROBE=./$(PROBE) FAKE_HOST=./$(FAKE) python3 tests/verbs_declared.py
+
 # panel_get on the wire against the installed omx-clap-scan (omx-clap-core 0.20.0 or later) and real plugins: one that
 # publishes a panel (omx-plugins-clap's omx-delay) and one that publishes none (omx-clap-host-tests' fake compressor)
 test-panel-wire: $(PROG) $(PROBE) $(FAKE)
@@ -288,7 +312,7 @@ $(TRANSFORMS_TEST): tests/transforms_test.c src/transforms.c src/transforms.h
 test-relay: $(RELAY_TEST)
 	./$(RELAY_TEST)
 
-$(RELAY_TEST): tests/relay_test.c src/relay.c src/relay.h include/plugin-hostd/protocol.h
+$(RELAY_TEST): tests/relay_test.c src/relay.c src/relay.h include/plugin-hostd/protocol.h $(RENDERED)
 	$(CC) -Iinclude $(CFLAGS) -Werror -Drealloc=relay_test_realloc -c -o tests/relay-test-relay.o src/relay.c
 	$(CC) -Isrc $(CFLAGS) -Werror -o $@ tests/relay_test.c tests/relay-test-relay.o -pthread
 # the relay's gesture table and lv2.c's refusal and pick tables when memory runs out: each is built with its realloc
@@ -297,7 +321,7 @@ test-alloc: $(GESTURE_ALLOC_TEST) $(LV2_ALLOC_TEST)
 	./$(GESTURE_ALLOC_TEST)
 	./$(LV2_ALLOC_TEST)
 
-$(GESTURE_ALLOC_TEST): tests/gesture_alloc_test.c src/relay.c src/relay.h include/plugin-hostd/protocol.h
+$(GESTURE_ALLOC_TEST): tests/gesture_alloc_test.c src/relay.c src/relay.h include/plugin-hostd/protocol.h $(RENDERED)
 	$(CC) -Iinclude -Isrc $(CFLAGS) -Werror -Drealloc=alloc_test_realloc -o $@ tests/gesture_alloc_test.c -pthread
 
 $(LV2_ALLOC_TEST): tests/lv2_alloc_test.c src/lv2.c src/formats.h
@@ -361,7 +385,7 @@ tests/jack_levels: tests/jack_levels.c
 GEN = tools/protocol-gen
 GENERATED = README.md doc/plugin-hostd.1 protocol/plugin-hostd.json protocol/mod-host.json protocol/probe-output.schema.json
 
-$(GEN): tools/protocol-gen.c include/plugin-hostd/protocol.h
+$(GEN): tools/protocol-gen.c include/plugin-hostd/protocol.h $(RENDERED)
 	$(CC) -Wall -Wextra -Werror -std=gnu99 -Iinclude $(CONTRACT_CFLAGS) -o $@ tools/protocol-gen.c
 
 gen: $(GEN)

README.md

@@ -72,11 +72,15 @@ Building from source: see [BUILDING.md](BUILDING.md). More about FreeMixer at <h
 Protocol
 --------
 
-The verbs the daemon adds, the placement syntax, the error codes, the feedback events and the
-settings are declared once in `include/plugin-hostd/protocol.h`; the tables here, the man page's
-key list and `protocol/plugin-hostd.json` are generated from it. A C program includes the installed
-header (`pkg-config --cflags plugin-hostd`, package `plugin-hostd-devel` or `plugin-hostd-dev`); anything
-else reads `/usr/share/plugin-hostd/protocol.json`. mod-host's own protocol, which the LV2 workers speak
+The verbs the daemon adds are declared once in omx-contract's contract directory, with their formats, their replies,
+their refusal codes and what each host owes them: `/usr/share/omx-contract/contract.d/3/plugin-hostd/verbs.json`,
+package `plugin-hostd-contract`, beside the format tables. A refusal number omx-clap-host answers too carries
+omx-clap-core's `reply.host.*` code (package `omx-clap-core-contract`), the others plugin-hostd's
+`reply.plugin-hostd.*`: one name per number. The placement syntax, the error codes, the feedback events
+and the settings are declared once in `include/plugin-hostd/protocol.h`, which includes the verbs as
+`plugin_hostd_host_verbs.h` renders them; the tables here, the man page's key list and `protocol/plugin-hostd.json`
+are generated from both. A C program includes the installed header (`pkg-config --cflags plugin-hostd`, package
+`plugin-hostd-devel` or `plugin-hostd-dev`); anything else reads `/usr/share/plugin-hostd/protocol.json`. mod-host's own protocol, which the LV2 workers speak
 (its command and feedback templates, its parse refusals and its reply codes), is installed beside it as
 `/usr/share/plugin-hostd/mod-host.json`, written from the mod-host headers the daemon is built against.
 The development package installs the schemas beside them: `protocol.schema.json` for the protocol JSON,