Planner batch: history record (L1), planner (L2), bench modes (L6), CI savings, README section

.github/workflows/ci.yml

@@ -17,7 +17,44 @@ on:
 permissions:
   contents: read
 
+concurrency:
+  # A new push to a pull request cancels the run it supersedes. Every other run (main, a tag) has a group of its own
+  # and is never cancelled; the run id keeps two pushes to main from replacing each other while one is queued.
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
+  # The packages and their smoke tests are built from the whole tree but fuzz/, bench/, the top-level pages and the licence:
+  # a pull request that changes only those skips them (the jobs below that need this one). main and tags run them always.
+  changes:
+    name: what the pull request changes
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    outputs:
+      packages: ${{ steps.diff.outputs.packages }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+
+      # The checkout of a pull request is its merge commit and the first parent is the base, so this diff is what the
+      # pull request adds. Any other event, or a diff that cannot be read, says true: main and tags always run it.
+      - id: diff
+        env:
+          EVENT: ${{ github.event_name }}
+        run: |
+          set -euo pipefail
+          if [ "$EVENT" != pull_request ] || ! changed=$(git diff --name-only HEAD^1 HEAD) || [ -z "$changed" ]; then
+            echo "packages=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          printf '%s\n' "$changed"
+          if grep -qvE '^(fuzz/|bench/|[^/]*\.md$|COPYING$|LICENSE$|\.github/workflows/fuzz\.yml$)' <<<"$changed"; then
+            echo "packages=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "packages=false" >> "$GITHUB_OUTPUT"
+          fi
+
   private-refs:
     name: no tracked file points into a private repository
     runs-on: ubuntu-latest
@@ -31,7 +68,7 @@ jobs:
   test:
     name: build and test
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    timeout-minutes: 60
     container:
       image: fedora:44
       # the jack test runs the daemon and its workers in a user, net, pid and mount namespace of their own
@@ -172,12 +209,14 @@ jobs:
 
   rpm:
     name: RPMs (fc44, ${{ matrix.arch }}), installed in a clean root
+    needs: changes
+    if: needs.changes.outputs.packages == 'true'
     runs-on: ${{ matrix.arch == 'aarch64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
     timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
-        arch: [x86_64, aarch64]
+        arch: ${{ fromJSON(github.event_name == 'pull_request' && '["x86_64"]' || '["x86_64", "aarch64"]') }}
     container:
       image: fedora:44
       # the clean root gets a /proc and a /dev of its own
@@ -235,6 +274,8 @@ jobs:
             printf '%s\n' /usr/share/omx-contract/contract.d/3/plugin-hostd/MANIFEST.json \
               /usr/share/omx-contract/contract.d/3/plugin-hostd/formats/clap.json \
               /usr/share/omx-contract/contract.d/3/plugin-hostd/formats/lv2.json \
+              /usr/share/omx-contract/contract.d/3/plugin-hostd/history.json \
+              /usr/share/omx-contract/contract.d/3/plugin-hostd/grouping.json \
               /usr/share/omx-contract/contract.d/3/plugin-hostd/verbs.json \
               /usr/share/licenses/plugin-hostd-contract/COPYING | sort > expected.txt
             diff -u expected.txt files.txt
@@ -312,13 +353,15 @@ jobs:
 
   deb:
     name: DEBs (${{ matrix.suite }}, ${{ matrix.arch }}), built and linted
+    needs: changes
+    if: needs.changes.outputs.packages == 'true'
     runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
     timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
         suite: [bookworm, trixie]
-        arch: [amd64, arm64]
+        arch: ${{ fromJSON(github.event_name == 'pull_request' && '["amd64"]' || '["amd64", "arm64"]') }}
     defaults:
       run:
         shell: bash
@@ -380,7 +423,7 @@ jobs:
       fail-fast: false
       matrix:
         suite: [bookworm, trixie]
-        arch: [amd64, arm64]
+        arch: ${{ fromJSON(github.event_name == 'pull_request' && '["amd64"]' || '["amd64", "arm64"]') }}
     # nothing but the distribution: a dependency the package forgot to declare is missing here
     defaults:
       run:
@@ -407,7 +450,7 @@ jobs:
           # the contract directory's files are the noarch contract package's, which the daemon and the headers depend on
           test "$(dpkg-deb -f debs/plugin-hostd-contract_*.deb Architecture)" = all
           dpkg -L plugin-hostd-contract | tee contract.txt
-          for f in MANIFEST.json formats/clap.json formats/lv2.json verbs.json; do
+          for f in MANIFEST.json formats/clap.json formats/lv2.json grouping.json history.json verbs.json; do
             grep -qx "/usr/share/omx-contract/contract.d/3/plugin-hostd/$f" contract.txt
           done
           dpkg-deb -f debs/plugin-hostd-contract_*.deb Breaks Replaces | tee breaks.txt
@@ -456,13 +499,15 @@ jobs:
   # `lv2_worker omx-clap-host`, and --audio then measures the gain of a sine that went through that instance.
   own-host-page:
     name: own-host page (${{ matrix.suite }}, ${{ matrix.arch }}), run as written in a clean container
+    needs: changes
+    if: needs.changes.outputs.packages == 'true'
     runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
     timeout-minutes: 20
     strategy:
       fail-fast: false
       matrix:
         suite: [bookworm, trixie]
-        arch: [amd64, arm64]
+        arch: ${{ fromJSON(github.event_name == 'pull_request' && '["amd64"]' || '["amd64", "arm64"]') }}
     defaults:
       run:
         shell: bash
@@ -495,10 +540,13 @@ jobs:
   # for; its publish and release steps do not run here.
   release-dry-run:
     name: release workflow, dry run
+    needs: changes
+    if: needs.changes.outputs.packages == 'true'
     permissions:
       contents: write
       issues: write
       actions: read
     uses: ./.github/workflows/release.yml
     with:
       publish: false
+      arm: false  # aarch64/arm64 build at release only

.github/workflows/fuzz.yml

@@ -14,9 +14,48 @@ on:
 permissions:
   contents: read
 
+concurrency:
+  # A new push to a pull request cancels the run it supersedes. Every other run (main, a tag) has a group of its own
+  # and is never cancelled; the run id keeps two pushes to main from replacing each other while one is queued.
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
+  # A pull request fuzzes only when it changes what the harnesses are built from or read (the Makefile's FUZZ_DEPS, the
+  # protocol and the pages the seed corpus is cut from); main runs it always.
+  changes:
+    name: what the pull request changes
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    outputs:
+      fuzz: ${{ steps.diff.outputs.fuzz }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+
+      # The checkout of a pull request is its merge commit and the first parent is the base, so this diff is what the
+      # pull request adds. Any other event, or a diff that cannot be read, says true: main and tags always run it.
+      - id: diff
+        env:
+          EVENT: ${{ github.event_name }}
+        run: |
+          set -euo pipefail
+          if [ "$EVENT" != pull_request ] || ! changed=$(git diff --name-only HEAD^1 HEAD) || [ -z "$changed" ]; then
+            echo "fuzz=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          printf '%s\n' "$changed"
+          if grep -qE '^(fuzz/|src/|include/|protocol/|data/|Makefile$|README\.md$|docs/own-host\.md$|packaging/plugin-hostd\.spec$|\.github/workflows/fuzz\.yml$)' <<<"$changed"; then
+            echo "fuzz=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "fuzz=false" >> "$GITHUB_OUTPUT"
+          fi
+
   fuzz:
     name: fuzz the command parser and the reply stream
+    needs: changes
+    if: needs.changes.outputs.fuzz == 'true'
     runs-on: ubuntu-latest
     # a backstop: fuzz-run spends 60 s on each harness and fuzz-sabotage runs seeds only, so the job ends well inside it
     timeout-minutes: 30

.github/workflows/release.yml

@@ -14,11 +14,19 @@ on:
         description: 'Sign and publish; only a run on a tag publishes'
         type: boolean
         default: false
+      arm:
+        description: 'Build the aarch64/arm64 packages too; a pull request dry run passes false'
+        type: boolean
+        default: true
   workflow_call:
     inputs:
       publish:
         type: boolean
         default: false
+      arm:
+        type: boolean
+        default: true
+        description: 'Build the aarch64/arm64 packages too; the pull-request dry run passes false (arm builds run at release only)'
 
 permissions:
   contents: write
@@ -31,6 +39,7 @@ jobs:
     with:
       package-name: plugin-hostd
       publish: ${{ github.event_name == 'push' || inputs.publish == true }}
+      arch-matrix: ${{ (github.event_name != 'push' && inputs.arm == false) && '["x86_64"]' || '["x86_64", "aarch64"]' }}
     secrets: inherit
 
   # after the RPMs: both publish into one tree, and a push that loses a race is refused
@@ -41,4 +50,5 @@ jobs:
     with:
       package-name: plugin-hostd
       publish: ${{ github.event_name == 'push' || inputs.publish == true }}
+      arch-matrix: ${{ (github.event_name != 'push' && inputs.arm == false) && '["amd64"]' || '["amd64", "arm64"]' }}
     secrets: inherit

.gitignore

@@ -8,6 +8,8 @@ tests/host_scenarios
 tools/protocol-gen
 *.new
 tests/pin-test
+tests/history-test
+tests/proc-test
 tests/relay-test
 tests/relay-test-relay.o
 tests/format-test
@@ -22,3 +24,5 @@ tests/lv2-alloc-lv2.o
 tests/transforms-test
 /build/
 fuzz/findings/
+tests/planner-test
+tests/qualcost-test

Makefile

@@ -7,6 +7,10 @@ VERSION = $(shell sed -n "s/^Version: *//p" packaging/plugin-hostd.spec)
 FAKE = tests/fake-host
 FORMAT_TEST = tests/format-test
 PIN_TEST = tests/pin-test
+HISTORY_TEST = tests/history-test
+PROC_TEST = tests/proc-test
+PLANNER_TEST = tests/planner-test
+QUALCOST_TEST = tests/qualcost-test
 RELAY_TEST = tests/relay-test
 GESTURE_ALLOC_TEST = tests/gesture-alloc-test
 LV2_ALLOC_TEST = tests/lv2-alloc-test
@@ -57,7 +61,7 @@ LDFLAGS += -Wl,--no-undefined
 
 # source and object files
 SRC = src/main.c src/conf.c src/proc.c src/supervisor.c src/verbs.c src/pins.c src/relay.c src/formats.c src/lv2.c \
-      src/descriptors.c src/describe.c src/transforms.c src/json.c
+      src/descriptors.c src/describe.c src/transforms.c src/json.c src/fsutil.c src/machine.c src/history.c src/planner.c src/qualcost.c
 OBJ = $(SRC:.c=.o)
 
 # the C names of the daemon's verbs and reply codes, rendered from data/verbs.json (render-verbs below); protocol.h
@@ -72,8 +76,8 @@ $(PROG): $(OBJ) $(PROTOCOL_LIB)
 
 # the probe runner: it loads no plugin and links nothing of the daemon or of mod-host
 # describe reads the installed table of a format by default (docs/formats.md)
-PROBE_SRC = src/probe.c src/json.c src/describe.c src/transforms.c
-$(PROBE): $(PROBE_SRC) src/json.h src/describe.h src/transforms.h include/plugin-hostd/protocol.h $(RENDERED) include/plugin-hostd/pin.h
+PROBE_SRC = src/probe.c src/json.c src/describe.c src/transforms.c src/machine.c
+$(PROBE): $(PROBE_SRC) src/json.h src/describe.h src/transforms.h src/machine.h include/plugin-hostd/protocol.h $(RENDERED) include/plugin-hostd/pin.h
 	$(CC) -Iinclude -Isrc $(CONTRACT_CFLAGS) $(CFLAGS) -D_GNU_SOURCE -DPROBE_VERSION='"$(VERSION)"' \
 	  -DPROBE_CONTRACTDIR='"$(CONTRACTDIR)"' -DPROBE_CONTRACT_MAJOR='"$(CONTRACT_SCHEMA_MAJOR)"' -o $@ $(PROBE_SRC) -lm
 
@@ -85,6 +89,9 @@ endif
 # the declared protocol is compiled into every object
 $(OBJ): include/plugin-hostd/protocol.h $(RENDERED) include/plugin-hostd/pin.h
 
+# the host class of the history is the probe's machine class: the contract's band words
+src/machine.o: INCS += $(CONTRACT_CFLAGS)
+
 src/descriptors.o: src/descriptors.h src/describe.h src/json.h src/formats.h src/supervisor.h
 
 # meta-rule to generate the object files
@@ -251,12 +258,12 @@ fuzz-corpus-check:
 
 # clean rule
 clean:
-	@rm -f src/*.o $(PROG) $(PROBE) tests/probe-child tests/lv2-probe-child tests/rt_interpose.so $(FAKE) $(PIN_TEST) $(RELAY_TEST) tests/relay-test-relay.o $(FORMAT_TEST) $(GESTURE_ALLOC_TEST) $(LV2_ALLOC_TEST) tests/lv2-alloc-lv2.o $(TRANSFORMS_TEST) $(GEN) tests/stress.clap tests/jack_levels tests/host_scenarios $(LV2_FIXTURE)
+	@rm -f src/*.o $(PROG) $(PROBE) tests/probe-child tests/lv2-probe-child tests/rt_interpose.so $(FAKE) $(PIN_TEST) $(HISTORY_TEST) $(PROC_TEST) $(PLANNER_TEST) $(QUALCOST_TEST) $(RELAY_TEST) tests/relay-test-relay.o $(FORMAT_TEST) $(GESTURE_ALLOC_TEST) $(LV2_ALLOC_TEST) tests/lv2-alloc-lv2.o $(TRANSFORMS_TEST) $(GEN) tests/stress.clap tests/jack_levels tests/host_scenarios $(LV2_FIXTURE)
 	@rm -rf build
 
 # the daemon against workers that are not plugin hosts at all (tests/fake-host speaks the same protocol and can be
 # made to die on cue): placement, forwarding, ledger, replay, attribution, the storm bound; no jack, no plugin
-test: test-protocol-lib test-pin test-transforms test-format-coverage test-describe test-relay test-alloc test-format test-daemon test-verbs-declared check-rendered check-generated check-schema test-probe-schema test-vocabulary test-probe test-contract test-consumer test-perturbation check-readme check-own-host-page check-no-pins check-ci-guards test-mod-host-json test-socket check-verb-table check-preset-schema test-install-schemas
+test: test-protocol-lib test-pin test-history test-proc test-planner test-transforms test-format-coverage test-describe test-relay test-alloc test-format test-daemon test-verbs-declared check-rendered check-generated check-schema test-probe-schema test-vocabulary test-probe test-contract test-consumer test-perturbation check-readme check-own-host-page check-no-pins check-ci-guards test-mod-host-json test-socket check-verb-table check-preset-schema test-install-schemas
 	MOD_HOST_DIR=$(MOD_HOST_DIR) python3 tests/verbs_contract.py
 
 # the protocol clients plugin-hostd-tests ships (tests/lib/), against a peer that frames by NUL, and their sabotage
@@ -295,7 +302,7 @@ test-format: $(FORMAT_TEST) $(FAKE)
 	PLUGIN_HOSTD_FORMAT_TEST_DIR=$${TMPDIR:-$${XDG_RUNTIME_DIR:-.}} ./$(FORMAT_TEST) ./$(FAKE)
 
 # the daemon's parts without its main and its verbs: the test calls the supervisor as the verbs do
-FORMAT_OBJ = src/conf.o src/proc.o src/supervisor.o src/pins.o src/relay.o src/formats.o src/lv2.o
+FORMAT_OBJ = src/conf.o src/proc.o src/supervisor.o src/pins.o src/fsutil.o src/machine.o src/history.o src/relay.o src/formats.o src/lv2.o src/planner.o src/qualcost.o src/json.o
 
 $(FORMAT_TEST): tests/format_test.c $(FORMAT_OBJ) $(PROTOCOL_LIB)
 	$(CC) $(INCS) -Isrc $(CFLAGS) -Werror -o $@ tests/format_test.c $(FORMAT_OBJ) $(PROTOCOL_LIBS) -pthread -lm
@@ -307,6 +314,33 @@ test-pin: $(PIN_TEST)
 $(PIN_TEST): tests/pin_test.c include/plugin-hostd/pin.h
 	$(CC) -Iinclude $(CFLAGS) -Werror -o $@ tests/pin_test.c
 
+# the running history (src/history.c, the grouping-planner design note): its line grammar, its file, what its
+# counts and its summary come to, against records written out by hand; and the host class it is kept under (src/machine.c)
+test-history: $(HISTORY_TEST)
+	./$(HISTORY_TEST)
+
+$(HISTORY_TEST): tests/history_test.c src/history.c src/history.h src/fsutil.c src/fsutil.h src/machine.c src/machine.h include/plugin-hostd/protocol.h $(RENDERED)
+	$(CC) -Iinclude -Isrc $(CONTRACT_CFLAGS) $(CFLAGS) -Werror -o $@ tests/history_test.c src/history.c src/fsutil.c src/machine.c -lm
+
+# src/proc.c's readers of /proc: the text the kernel writes, and this process's own files
+test-proc: $(PROC_TEST)
+	./$(PROC_TEST)
+
+$(PROC_TEST): tests/proc_test.c src/proc.c src/proc.h src/conf.h include/plugin-hostd/protocol.h $(RENDERED)
+	$(CC) $(INCS) -Isrc $(CFLAGS) -Werror -o $@ tests/proc_test.c src/proc.c -pthread -lm
+
+# the grouping planner (src/planner.c, the grouping-planner design note), on its own: each reason it gives, the order its
+# rules decide in, the text of a plan, and that the same members give the same bytes
+test-planner: $(PLANNER_TEST) $(QUALCOST_TEST)
+	./$(PLANNER_TEST)
+	./$(QUALCOST_TEST)
+
+$(QUALCOST_TEST): tests/qualcost_test.c src/qualcost.c src/qualcost.h src/json.c src/json.h
+	$(CC) -Isrc $(CFLAGS) -Werror -o $@ tests/qualcost_test.c src/qualcost.c src/json.c -lm
+
+$(PLANNER_TEST): tests/planner_test.c src/planner.c src/planner.h include/plugin-hostd/protocol.h $(RENDERED)
+	$(CC) -Iinclude -Isrc $(CFLAGS) -Werror -o $@ tests/planner_test.c src/planner.c
+
 # the transforms the format tables name (src/transforms.c, docs/formats.md): every one and its inverse over its vectors,
 # and a lookup that is not one to one refused
 test-transforms: $(TRANSFORMS_TEST)