From 18e549a30610c02111168a71eb4ff7fd5afde69b Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Tue, 27 Oct 2026 09:37:01 +0100 Subject: [PATCH 01/16] ci: cancel superseded pull request runs, skip package and fuzz runs a pull request cannot change A new push to a pull request cancels the run it supersedes (concurrency in ci.yml and fuzz.yml); main and tag runs have a group of their own and are never cancelled. A changes job reads the pull request's diff. The RPM, DEB, own-host page and release dry-run jobs run on a pull request only when it touches something besides fuzz/, bench/, the top-level pages and the licence; the fuzz job only when it touches fuzz/, src/, include/, protocol/, data/, the Makefile, the pages the corpus is cut from or the spec. main and tags run everything. No workflow-level paths filter, so the required check build and test always reports. aarch64 and arm64 build at release only on a pull request, as in omx-plugins: release.yml takes an arm input (default true) that the dry run sets false. tests/ci_guards.py holds the concurrency rules and the absence of a paths filter. --- .github/workflows/ci.yml | 54 ++++++++++++++++++++++++++++++++--- .github/workflows/fuzz.yml | 39 +++++++++++++++++++++++++ .github/workflows/release.yml | 10 +++++++ tests/ci_guards.py | 46 +++++++++++++++++++++++++++++ 4 files changed, 145 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b8b29b2..b092a50 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,44 @@ on: permissions: contents: read +concurrency: + # A new push to a pull request cancels the run it supersedes. Every other run (main, a tag) has a group of its own + # and is never cancelled; the run id keeps two pushes to main from replacing each other while one is queued. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: + # The packages and their smoke tests are built from the whole tree but fuzz/, bench/, the top-level pages and the licence: + # a pull request that changes only those skips them (the jobs below that need this one). main and tags run them always. + changes: + name: what the pull request changes + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + packages: ${{ steps.diff.outputs.packages }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + + # The checkout of a pull request is its merge commit and the first parent is the base, so this diff is what the + # pull request adds. Any other event, or a diff that cannot be read, says true: main and tags always run it. + - id: diff + env: + EVENT: ${{ github.event_name }} + run: | + set -euo pipefail + if [ "$EVENT" != pull_request ] || ! changed=$(git diff --name-only HEAD^1 HEAD) || [ -z "$changed" ]; then + echo "packages=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + printf '%s\n' "$changed" + if grep -qvE '^(fuzz/|bench/|[^/]*\.md$|COPYING$|LICENSE$|\.github/workflows/fuzz\.yml$)' <<<"$changed"; then + echo "packages=true" >> "$GITHUB_OUTPUT" + else + echo "packages=false" >> "$GITHUB_OUTPUT" + fi + private-refs: name: no tracked file points into a private repository runs-on: ubuntu-latest @@ -172,12 +209,14 @@ jobs: rpm: name: RPMs (fc44, ${{ matrix.arch }}), installed in a clean root + needs: changes + if: needs.changes.outputs.packages == 'true' runs-on: ${{ matrix.arch == 'aarch64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} timeout-minutes: 30 strategy: fail-fast: false matrix: - arch: [x86_64, aarch64] + arch: ${{ fromJSON(github.event_name == 'pull_request' && '["x86_64"]' || '["x86_64", "aarch64"]') }} container: image: fedora:44 # the clean root gets a /proc and a /dev of its own @@ -308,13 +347,15 @@ jobs: deb: name: DEBs (${{ matrix.suite }}, ${{ matrix.arch }}), built and linted + needs: changes + if: needs.changes.outputs.packages == 'true' runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} timeout-minutes: 30 strategy: fail-fast: false matrix: suite: [bookworm, trixie] - arch: [amd64, arm64] + arch: ${{ fromJSON(github.event_name == 'pull_request' && '["amd64"]' || '["amd64", "arm64"]') }} defaults: run: shell: bash @@ -376,7 +417,7 @@ jobs: fail-fast: false matrix: suite: [bookworm, trixie] - arch: [amd64, arm64] + arch: ${{ fromJSON(github.event_name == 'pull_request' && '["amd64"]' || '["amd64", "arm64"]') }} # nothing but the distribution: a dependency the package forgot to declare is missing here defaults: run: @@ -450,13 +491,15 @@ jobs: # `lv2_worker omx-clap-host`, and --audio then measures the gain of a sine that went through that instance. own-host-page: name: own-host page (${{ matrix.suite }}, ${{ matrix.arch }}), run as written in a clean container + needs: changes + if: needs.changes.outputs.packages == 'true' runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} timeout-minutes: 20 strategy: fail-fast: false matrix: suite: [bookworm, trixie] - arch: [amd64, arm64] + arch: ${{ fromJSON(github.event_name == 'pull_request' && '["amd64"]' || '["amd64", "arm64"]') }} defaults: run: shell: bash @@ -489,6 +532,8 @@ jobs: # for; its publish and release steps do not run here. release-dry-run: name: release workflow, dry run + needs: changes + if: needs.changes.outputs.packages == 'true' permissions: contents: write issues: write @@ -496,3 +541,4 @@ jobs: uses: ./.github/workflows/release.yml with: publish: false + arm: false # aarch64/arm64 build at release only diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index bda7cb4..577c9fe 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -14,9 +14,48 @@ on: permissions: contents: read +concurrency: + # A new push to a pull request cancels the run it supersedes. Every other run (main, a tag) has a group of its own + # and is never cancelled; the run id keeps two pushes to main from replacing each other while one is queued. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: + # A pull request fuzzes only when it changes what the harnesses are built from or read (the Makefile's FUZZ_DEPS, the + # protocol and the pages the seed corpus is cut from); main runs it always. + changes: + name: what the pull request changes + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + fuzz: ${{ steps.diff.outputs.fuzz }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + + # The checkout of a pull request is its merge commit and the first parent is the base, so this diff is what the + # pull request adds. Any other event, or a diff that cannot be read, says true: main and tags always run it. + - id: diff + env: + EVENT: ${{ github.event_name }} + run: | + set -euo pipefail + if [ "$EVENT" != pull_request ] || ! changed=$(git diff --name-only HEAD^1 HEAD) || [ -z "$changed" ]; then + echo "fuzz=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + printf '%s\n' "$changed" + if grep -qE '^(fuzz/|src/|include/|protocol/|data/|Makefile$|README\.md$|docs/own-host\.md$|packaging/plugin-hostd\.spec$|\.github/workflows/fuzz\.yml$)' <<<"$changed"; then + echo "fuzz=true" >> "$GITHUB_OUTPUT" + else + echo "fuzz=false" >> "$GITHUB_OUTPUT" + fi + fuzz: name: fuzz the command parser and the reply stream + needs: changes + if: needs.changes.outputs.fuzz == 'true' runs-on: ubuntu-latest # a backstop: fuzz-run spends 60 s on each harness and fuzz-sabotage runs seeds only, so the job ends well inside it timeout-minutes: 30 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a310fb4..4b268c9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,11 +14,19 @@ on: description: 'Sign and publish; only a run on a tag publishes' type: boolean default: false + arm: + description: 'Build the aarch64/arm64 packages too; a pull request dry run passes false' + type: boolean + default: true workflow_call: inputs: publish: type: boolean default: false + arm: + type: boolean + default: true + description: 'Build the aarch64/arm64 packages too; the pull-request dry run passes false (arm builds run at release only)' permissions: contents: write @@ -31,6 +39,7 @@ jobs: with: package-name: plugin-hostd publish: ${{ github.event_name == 'push' || inputs.publish == true }} + arch-matrix: ${{ (github.event_name != 'push' && inputs.arm == false) && '["x86_64"]' || '["x86_64", "aarch64"]' }} secrets: inherit # after the RPMs: both publish into one tree, and a push that loses a race is refused @@ -41,4 +50,5 @@ jobs: with: package-name: plugin-hostd publish: ${{ github.event_name == 'push' || inputs.publish == true }} + arch-matrix: ${{ (github.event_name != 'push' && inputs.arm == false) && '["amd64"]' || '["amd64", "arm64"]' }} secrets: inherit diff --git a/tests/ci_guards.py b/tests/ci_guards.py index b549d65..89b91c1 100644 --- a/tests/ci_guards.py +++ b/tests/ci_guards.py @@ -49,6 +49,33 @@ def scan(root): return found +# A pull request's older run is cancelled when a newer push supersedes it, and nothing else ever is: ci.yml and fuzz.yml +# carry a concurrency group whose cancel-in-progress is true on a pull request only, release.yml (a tag publishes from +# it) carries none. A workflow-level paths: filter on ci.yml or fuzz.yml would leave a required check that never reports, +# so the filters are the changes job's and the trigger has none. +CANCEL_LINE = "cancel-in-progress: ${{ github.event_name == 'pull_request' }}" +PATHS_FILTER = re.compile(r"^ {4}paths(-ignore)?:", re.M) + + +def cancel_scan(root): + """every finding as 'path what'""" + found = [] + d = os.path.join(root, ".github", "workflows") + for name in ("ci.yml", "fuzz.yml"): + path = os.path.join(d, name) + if not os.path.exists(path): + continue + text = open(path).read() + if not re.search(r"^concurrency:", text, re.M) or CANCEL_LINE not in text: + found.append("%s has no concurrency group that cancels on a pull request only (%s)" % (name, CANCEL_LINE)) + if PATHS_FILTER.search(text): + found.append("%s filters its trigger by paths: a required check that never reports blocks the pull request; use the changes job" % name) + release = os.path.join(d, "release.yml") + if os.path.exists(release) and re.search(r"^\s*(concurrency|cancel-in-progress):", open(release).read(), re.M): + found.append("release.yml has a concurrency group: a run on a tag must never be cancelled") + return found + + def main(): tmp = tempfile.mkdtemp(prefix="ci-guards.", dir=os.environ.get("TMPDIR") or ROOT) try: @@ -66,7 +93,26 @@ def main(): finally: shutil.rmtree(tmp, ignore_errors=True) print("ok the scan sees a planted warn-only else and a planted missing else, and passes one that exits 1") + bad = tempfile.mkdtemp(prefix="ci-guards.", dir=os.environ.get("TMPDIR") or ROOT) + try: + wf = os.path.join(bad, ".github", "workflows") + os.makedirs(wf) + open(os.path.join(wf, "ci.yml"), "w").write("on:\n pull_request:\n paths: [src/]\nconcurrency:\n cancel-in-progress: true\n") + open(os.path.join(wf, "fuzz.yml"), "w").write("on:\n pull_request:\n") + open(os.path.join(wf, "release.yml"), "w").write("concurrency:\n group: x\n") + seen = cancel_scan(bad) + if len(seen) != 4: + print("FAIL the cancel scan misses a planted fault: saw %s" % seen) + return 1 + finally: + shutil.rmtree(bad, ignore_errors=True) + print("ok the cancel scan sees a planted paths filter, a missing group, a cancelling group and a release group") failed = 0 + for f in cancel_scan(ROOT): + print("FAIL %s" % f) + failed += 1 + if not failed: + print("ok ci.yml and fuzz.yml cancel a superseded pull request run only, release.yml never, no workflow paths filter") for f in scan(ROOT): print("FAIL %s" % f) failed += 1 From c640b492fbe36ead28ea0fa1b352abda91a45b8f Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Tue, 27 Oct 2026 06:38:28 +0100 Subject: [PATCH 02/16] Move the directory and fsync helpers of the pin store into src/fsutil.c The running-history file is written the way the pin store is: a temporary file beside it, renamed over it, the directory fsynced. Both writers now share the two helpers instead of a second copy (#184). --- Makefile | 4 ++-- src/fsutil.c | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++++ src/fsutil.h | 31 ++++++++++++++++++++++++ src/pins.c | 43 +++------------------------------ 4 files changed, 103 insertions(+), 42 deletions(-) create mode 100644 src/fsutil.c create mode 100644 src/fsutil.h diff --git a/Makefile b/Makefile index af03f88..5995028 100644 --- a/Makefile +++ b/Makefile @@ -57,7 +57,7 @@ LDFLAGS += -Wl,--no-undefined # source and object files SRC = src/main.c src/conf.c src/proc.c src/supervisor.c src/verbs.c src/pins.c src/relay.c src/formats.c src/lv2.c \ - src/descriptors.c src/describe.c src/transforms.c src/json.c + src/descriptors.c src/describe.c src/transforms.c src/json.c src/fsutil.c OBJ = $(SRC:.c=.o) # the C names of the daemon's verbs and reply codes, rendered from data/verbs.json (render-verbs below); protocol.h @@ -295,7 +295,7 @@ test-format: $(FORMAT_TEST) $(FAKE) PLUGIN_HOSTD_FORMAT_TEST_DIR=$${TMPDIR:-$${XDG_RUNTIME_DIR:-.}} ./$(FORMAT_TEST) ./$(FAKE) # the daemon's parts without its main and its verbs: the test calls the supervisor as the verbs do -FORMAT_OBJ = src/conf.o src/proc.o src/supervisor.o src/pins.o src/relay.o src/formats.o src/lv2.o +FORMAT_OBJ = src/conf.o src/proc.o src/supervisor.o src/pins.o src/fsutil.o src/relay.o src/formats.o src/lv2.o $(FORMAT_TEST): tests/format_test.c $(FORMAT_OBJ) $(PROTOCOL_LIB) $(CC) $(INCS) -Isrc $(CFLAGS) -Werror -o $@ tests/format_test.c $(FORMAT_OBJ) $(PROTOCOL_LIBS) -pthread -lm diff --git a/src/fsutil.c b/src/fsutil.c new file mode 100644 index 0000000..935284a --- /dev/null +++ b/src/fsutil.c @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * This file is part of plugin-hostd. + * + * plugin-hostd is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * plugin-hostd is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with plugin-hostd. If not, see . + * + * Copyright (C) 2026 Pau Aliagas + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "fsutil.h" + +int fs_mkdir_parents(char *path) +{ + char *slash; + + for (slash = path + 1; (slash = strchr(slash, '/')); slash++) + { + *slash = '\0'; + if (mkdir(path, 0700) != 0 && errno != EEXIST) + { + *slash = '/'; + return -1; + } + *slash = '/'; + } + return 0; +} + +void fs_sync_dir(const char *path) +{ + char dir[PATH_MAX + 8], *slash; + int fd; + + snprintf(dir, sizeof(dir), "%s", path); + slash = strrchr(dir, '/'); + if (!slash) + snprintf(dir, sizeof(dir), "."); + else if (slash == dir) + dir[1] = '\0'; + else + *slash = '\0'; + fd = open(dir, O_RDONLY | O_DIRECTORY); + if (fd >= 0) + { + fsync(fd); + close(fd); + } +} diff --git a/src/fsutil.h b/src/fsutil.h new file mode 100644 index 0000000..182887b --- /dev/null +++ b/src/fsutil.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * This file is part of plugin-hostd. + * + * plugin-hostd is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * plugin-hostd is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with plugin-hostd. If not, see . + * + * Copyright (C) 2026 Pau Aliagas + */ + +#ifndef FSUTIL_H +#define FSUTIL_H + +/* the parent directories of `path`, each made mode 0700 when missing; `path` is written through and restored. 0, or -1 + * with errno set */ +int fs_mkdir_parents(char *path); + +/* fsync the directory that holds `path`, so a rename into it survives a power cut */ +void fs_sync_dir(const char *path); + +#endif diff --git a/src/pins.c b/src/pins.c index 85cc54a..f4ec48b 100644 --- a/src/pins.c +++ b/src/pins.c @@ -35,6 +35,7 @@ #include #include "formats.h" +#include "fsutil.h" #include "pins.h" #define SCHEME_MAX 32 @@ -287,23 +288,6 @@ static size_t line_size(const pin_t *p) return size; } -static int mkdir_parents(char *path) -{ - char *slash; - - for (slash = path + 1; (slash = strchr(slash, '/')); slash++) - { - *slash = '\0'; - if (mkdir(path, 0700) != 0 && errno != EEXIST) - { - *slash = '/'; - return -1; - } - *slash = '/'; - } - return 0; -} - /* Lines of the store this version could not parse. They are kept verbatim and written back by every save, so a corrupt * line is never dropped by the next save, and the uri they name (when one can be read) is never learned again: the pin * that line held is the operator's to repair or remove by hand. A store that could not be read at all blocks every @@ -356,27 +340,6 @@ static void held_free(void) g_store_unreadable = 0; } -static void sync_dir(const char *path) -{ - char dir[PATH_MAX + 8], *slash; - int fd; - - snprintf(dir, sizeof(dir), "%s", path); - slash = strrchr(dir, '/'); - if (!slash) - snprintf(dir, sizeof(dir), "."); - else if (slash == dir) - dir[1] = '\0'; - else - *slash = '\0'; - fd = open(dir, O_RDONLY | O_DIRECTORY); - if (fd >= 0) - { - fsync(fd); - close(fd); - } -} - /* the pins the daemon learned, to the store: a temporary file of its own beside it (mkstemp, so two daemons on one * store never share a name), written, fsynced and renamed over it, then the directory fsynced. A failure is announced * on stderr, with what it costs, and leaves the pins in memory */ @@ -396,7 +359,7 @@ static void store_save(void) } snprintf(tmp, sizeof(tmp), "%s.XXXXXX", g_store); fd = -1; - if (mkdir_parents(tmp) != 0 || (fd = mkstemp(tmp)) < 0 || !(file = fdopen(fd, "w"))) + if (fs_mkdir_parents(tmp) != 0 || (fd = mkstemp(tmp)) < 0 || !(file = fdopen(fd, "w"))) { int err = errno; @@ -423,7 +386,7 @@ static void store_save(void) unlink(tmp); return; } - sync_dir(g_store); + fs_sync_dir(g_store); } /* where a file of pin lines goes, and what a line outside the grammar costs there */ From 90f62b9d204f38bfec381fc8d2fb10ce3a064429 Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Tue, 27 Oct 2026 06:54:52 +0100 Subject: [PATCH 03/16] Take the machine class from one place, src/machine.c The probe wrote the core and clock bands from the contract's words inline. The daemon needs the same class (arch/cores/clock) to key what it keeps of a plugin by host class, so the bands and the class move into src/machine.c and the probe calls it (#184). --- Makefile | 9 +++++--- src/machine.c | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/machine.h | 35 +++++++++++++++++++++++++++++ src/probe.c | 10 +++------ 4 files changed, 106 insertions(+), 10 deletions(-) create mode 100644 src/machine.c create mode 100644 src/machine.h diff --git a/Makefile b/Makefile index 5995028..55c67f7 100644 --- a/Makefile +++ b/Makefile @@ -57,7 +57,7 @@ LDFLAGS += -Wl,--no-undefined # source and object files SRC = src/main.c src/conf.c src/proc.c src/supervisor.c src/verbs.c src/pins.c src/relay.c src/formats.c src/lv2.c \ - src/descriptors.c src/describe.c src/transforms.c src/json.c src/fsutil.c + src/descriptors.c src/describe.c src/transforms.c src/json.c src/fsutil.c src/machine.c OBJ = $(SRC:.c=.o) # the C names of the daemon's verbs and reply codes, rendered from data/verbs.json (render-verbs below); protocol.h @@ -72,8 +72,8 @@ $(PROG): $(OBJ) $(PROTOCOL_LIB) # the probe runner: it loads no plugin and links nothing of the daemon or of mod-host # describe reads the installed table of a format by default (docs/formats.md) -PROBE_SRC = src/probe.c src/json.c src/describe.c src/transforms.c -$(PROBE): $(PROBE_SRC) src/json.h src/describe.h src/transforms.h include/plugin-hostd/protocol.h $(RENDERED) include/plugin-hostd/pin.h +PROBE_SRC = src/probe.c src/json.c src/describe.c src/transforms.c src/machine.c +$(PROBE): $(PROBE_SRC) src/json.h src/describe.h src/transforms.h src/machine.h include/plugin-hostd/protocol.h $(RENDERED) include/plugin-hostd/pin.h $(CC) -Iinclude -Isrc $(CONTRACT_CFLAGS) $(CFLAGS) -D_GNU_SOURCE -DPROBE_VERSION='"$(VERSION)"' \ -DPROBE_CONTRACTDIR='"$(CONTRACTDIR)"' -DPROBE_CONTRACT_MAJOR='"$(CONTRACT_SCHEMA_MAJOR)"' -o $@ $(PROBE_SRC) -lm @@ -85,6 +85,9 @@ endif # the declared protocol is compiled into every object $(OBJ): include/plugin-hostd/protocol.h $(RENDERED) include/plugin-hostd/pin.h +# the host class of the history is the probe's machine class: the contract's band words +src/machine.o: INCS += $(CONTRACT_CFLAGS) + src/descriptors.o: src/descriptors.h src/describe.h src/json.h src/formats.h src/supervisor.h # meta-rule to generate the object files diff --git a/src/machine.c b/src/machine.c new file mode 100644 index 0000000..eaa4083 --- /dev/null +++ b/src/machine.c @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * This file is part of plugin-hostd. + * + * plugin-hostd is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * plugin-hostd is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with plugin-hostd. If not, see . + * + * Copyright (C) 2026 Pau Aliagas + */ + +#include +#include +#include +#include +#include + +#include + +#include "machine.h" + +void machine_bands(long ncpu, long khz, const char **core_band, const char **clock_band) +{ + *core_band = ncpu <= 2 ? OMX_MACHINE_CORE_BANDS_CORES1_TO2 : ncpu <= 4 ? OMX_MACHINE_CORE_BANDS_CORES3_TO4 : + ncpu <= 8 ? OMX_MACHINE_CORE_BANDS_CORES5_TO8 : ncpu <= 16 ? OMX_MACHINE_CORE_BANDS_CORES9_TO16 : + ncpu <= 32 ? OMX_MACHINE_CORE_BANDS_CORES17_TO32 : OMX_MACHINE_CORE_BANDS_CORES33_UP; + *clock_band = khz < 1500000 ? OMX_MACHINE_CLOCK_BANDS_UNDER1500_MHZ : + khz < 2500000 ? OMX_MACHINE_CLOCK_BANDS_FROM1500_TO2500_MHZ : + khz < 3500000 ? OMX_MACHINE_CLOCK_BANDS_FROM2500_TO3500_MHZ : + khz < 4500000 ? OMX_MACHINE_CLOCK_BANDS_FROM3500_TO4500_MHZ : OMX_MACHINE_CLOCK_BANDS_FROM4500_MHZ; +} + +int machine_class(char *out, size_t size) +{ + struct utsname u; + char text[64] = ""; + const char *core, *clock; + long khz = 0; + FILE *f; + int n; + + if (uname(&u) != 0) + snprintf(u.machine, sizeof(u.machine), "unknown"); + if ((f = fopen("/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq", "r"))) + { + if (fgets(text, sizeof(text), f)) + khz = atol(text); + fclose(f); + } + machine_bands(sysconf(_SC_NPROCESSORS_ONLN), khz, &core, &clock); + n = snprintf(out, size, "%s/%s/%s", u.machine, core, clock); + return n > 0 && (size_t)n < size ? 0 : -1; +} diff --git a/src/machine.h b/src/machine.h new file mode 100644 index 0000000..6c5620f --- /dev/null +++ b/src/machine.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * This file is part of plugin-hostd. + * + * plugin-hostd is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * plugin-hostd is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with plugin-hostd. If not, see . + * + * Copyright (C) 2026 Pau Aliagas + */ + +#ifndef MACHINE_H +#define MACHINE_H + +#include + +/* The machine class the daemon's probe writes (omx-contract's MACHINE_CORE_BANDS and MACHINE_CLOCK_BANDS): the band of + * `ncpu` cores and of `khz` kHz, the highest clock the first core states (0 when it states none). The words are the + * contract's. */ +void machine_bands(long ncpu, long khz, const char **core_band, const char **clock_band); + +/* arch/cores/clock of this machine, as one word (x86_64/5-8/2.5-3.5GHz): uname's machine, the online cores' band and + * cpu0's cpuinfo_max_freq's band. 0, or -1 when `size` is too small */ +int machine_class(char *out, size_t size); + +#endif diff --git a/src/probe.c b/src/probe.c index 06be7f3..9d36d80 100644 --- a/src/probe.c +++ b/src/probe.c @@ -81,6 +81,7 @@ #include "describe.h" #include "json.h" +#include "machine.h" /* ---------------------------------------------------------------- options */ @@ -414,13 +415,8 @@ static void put_machine(sb_t *b, const cpu_set_t *pinned, int pinned_by_option) khz = atol(text); if (read_text("/sys/kernel/realtime", text, sizeof(text)) == 0) preempt_rt = atoi(text) == 1; - /* the bands are omx-contract's MACHINE_CORE_BANDS and MACHINE_CLOCK_BANDS */ - core_band = ncpu <= 2 ? OMX_MACHINE_CORE_BANDS_CORES1_TO2 : ncpu <= 4 ? OMX_MACHINE_CORE_BANDS_CORES3_TO4 : - ncpu <= 8 ? OMX_MACHINE_CORE_BANDS_CORES5_TO8 : ncpu <= 16 ? OMX_MACHINE_CORE_BANDS_CORES9_TO16 : - ncpu <= 32 ? OMX_MACHINE_CORE_BANDS_CORES17_TO32 : OMX_MACHINE_CORE_BANDS_CORES33_UP; - clock_band = khz < 1500000 ? OMX_MACHINE_CLOCK_BANDS_UNDER1500_MHZ : khz < 2500000 ? OMX_MACHINE_CLOCK_BANDS_FROM1500_TO2500_MHZ : - khz < 3500000 ? OMX_MACHINE_CLOCK_BANDS_FROM2500_TO3500_MHZ : - khz < 4500000 ? OMX_MACHINE_CLOCK_BANDS_FROM3500_TO4500_MHZ : OMX_MACHINE_CLOCK_BANDS_FROM4500_MHZ; + /* the bands are omx-contract's MACHINE_CORE_BANDS and MACHINE_CLOCK_BANDS, the daemon's host class too */ + machine_bands(ncpu, khz, &core_band, &clock_band); for (i = 0; i < g_nidle; i++) { if (g_idle_after[i].latency_us > DEEP_IDLE_US && !g_idle_after[i].disabled) From 096da9ea2eec6182028c51637394583601fde60e Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Tue, 27 Oct 2026 06:55:08 +0100 Subject: [PATCH 04/16] Keep a running history of each plugin on this machine One line per plugin uri, binary digest and host class, in a file of the daemon's own: $XDG_STATE_HOME/plugin-hostd/history, else $HOME/.local/state/plugin-hostd/history, mode 0600, written to a temporary file and renamed. The keys are the words of PLUGIN_HOSTD_HISTORY_KEYS in data/history.json, the first line is '# plugin-hostd history/1'. A line this version cannot read, or with a key it does not know, is kept as it is by every save and not read; a file of another version, or one that cannot be read, is neither read nor overwritten. src/history.c holds the grammar, the windows (CPU and Pss alone, counted apart in a group), the crash, restart and quarantine counts and the summary the planner will read. The settings history_file, history_window_s, history_save_s and host_class are declared. tests/history_test.c checks it against lines written out by hand, with 26 sabotage arms in tests/sabotage.py (#184). --- .gitignore | 1 + Makefile | 17 +- README.md | 4 + data/history.json | 73 ++ doc/plugin-hostd.1 | 2 +- .../plugin-hostd/plugin_hostd_vocabulary.h | 19 + include/plugin-hostd/protocol.h | 28 +- protocol/plugin-hostd.json | 4 + src/history.c | 659 +++++++++++++++++ src/history.h | 91 +++ tests/history_test.c | 660 ++++++++++++++++++ tests/sabotage.py | 131 +++- 12 files changed, 1681 insertions(+), 8 deletions(-) create mode 100644 data/history.json create mode 100644 src/history.c create mode 100644 src/history.h create mode 100644 tests/history_test.c diff --git a/.gitignore b/.gitignore index 14fbe1f..3018b3c 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ tests/host_scenarios tools/protocol-gen *.new tests/pin-test +tests/history-test tests/relay-test tests/relay-test-relay.o tests/format-test diff --git a/Makefile b/Makefile index 55c67f7..7038e9a 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ VERSION = $(shell sed -n "s/^Version: *//p" packaging/plugin-hostd.spec) FAKE = tests/fake-host FORMAT_TEST = tests/format-test PIN_TEST = tests/pin-test +HISTORY_TEST = tests/history-test RELAY_TEST = tests/relay-test GESTURE_ALLOC_TEST = tests/gesture-alloc-test LV2_ALLOC_TEST = tests/lv2-alloc-test @@ -57,7 +58,7 @@ LDFLAGS += -Wl,--no-undefined # source and object files SRC = src/main.c src/conf.c src/proc.c src/supervisor.c src/verbs.c src/pins.c src/relay.c src/formats.c src/lv2.c \ - src/descriptors.c src/describe.c src/transforms.c src/json.c src/fsutil.c src/machine.c + src/descriptors.c src/describe.c src/transforms.c src/json.c src/fsutil.c src/machine.c src/history.c OBJ = $(SRC:.c=.o) # the C names of the daemon's verbs and reply codes, rendered from data/verbs.json (render-verbs below); protocol.h @@ -254,12 +255,12 @@ fuzz-corpus-check: # clean rule clean: - @rm -f src/*.o $(PROG) $(PROBE) tests/probe-child tests/lv2-probe-child tests/rt_interpose.so $(FAKE) $(PIN_TEST) $(RELAY_TEST) tests/relay-test-relay.o $(FORMAT_TEST) $(GESTURE_ALLOC_TEST) $(LV2_ALLOC_TEST) tests/lv2-alloc-lv2.o $(TRANSFORMS_TEST) $(GEN) tests/stress.clap tests/jack_levels tests/host_scenarios $(LV2_FIXTURE) + @rm -f src/*.o $(PROG) $(PROBE) tests/probe-child tests/lv2-probe-child tests/rt_interpose.so $(FAKE) $(PIN_TEST) $(HISTORY_TEST) $(RELAY_TEST) tests/relay-test-relay.o $(FORMAT_TEST) $(GESTURE_ALLOC_TEST) $(LV2_ALLOC_TEST) tests/lv2-alloc-lv2.o $(TRANSFORMS_TEST) $(GEN) tests/stress.clap tests/jack_levels tests/host_scenarios $(LV2_FIXTURE) @rm -rf build # the daemon against workers that are not plugin hosts at all (tests/fake-host speaks the same protocol and can be # made to die on cue): placement, forwarding, ledger, replay, attribution, the storm bound; no jack, no plugin -test: test-protocol-lib test-pin test-transforms test-format-coverage test-describe test-relay test-alloc test-format test-daemon test-verbs-declared check-rendered check-generated check-schema test-probe-schema test-vocabulary test-probe test-contract test-consumer test-perturbation check-readme check-own-host-page check-no-pins check-ci-guards test-mod-host-json test-socket check-verb-table check-preset-schema test-install-schemas +test: test-protocol-lib test-pin test-history test-transforms test-format-coverage test-describe test-relay test-alloc test-format test-daemon test-verbs-declared check-rendered check-generated check-schema test-probe-schema test-vocabulary test-probe test-contract test-consumer test-perturbation check-readme check-own-host-page check-no-pins check-ci-guards test-mod-host-json test-socket check-verb-table check-preset-schema test-install-schemas MOD_HOST_DIR=$(MOD_HOST_DIR) python3 tests/verbs_contract.py # the protocol clients plugin-hostd-tests ships (tests/lib/), against a peer that frames by NUL, and their sabotage @@ -298,7 +299,7 @@ test-format: $(FORMAT_TEST) $(FAKE) PLUGIN_HOSTD_FORMAT_TEST_DIR=$${TMPDIR:-$${XDG_RUNTIME_DIR:-.}} ./$(FORMAT_TEST) ./$(FAKE) # the daemon's parts without its main and its verbs: the test calls the supervisor as the verbs do -FORMAT_OBJ = src/conf.o src/proc.o src/supervisor.o src/pins.o src/fsutil.o src/relay.o src/formats.o src/lv2.o +FORMAT_OBJ = src/conf.o src/proc.o src/supervisor.o src/pins.o src/fsutil.o src/machine.o src/history.o src/relay.o src/formats.o src/lv2.o $(FORMAT_TEST): tests/format_test.c $(FORMAT_OBJ) $(PROTOCOL_LIB) $(CC) $(INCS) -Isrc $(CFLAGS) -Werror -o $@ tests/format_test.c $(FORMAT_OBJ) $(PROTOCOL_LIBS) -pthread -lm @@ -310,6 +311,14 @@ test-pin: $(PIN_TEST) $(PIN_TEST): tests/pin_test.c include/plugin-hostd/pin.h $(CC) -Iinclude $(CFLAGS) -Werror -o $@ tests/pin_test.c +# the running history (src/history.c, docs/design/notes/2026-10-27-grouping-planner.md): its line grammar, its file, what its +# counts and its summary come to, against records written out by hand +test-history: $(HISTORY_TEST) + ./$(HISTORY_TEST) + +$(HISTORY_TEST): tests/history_test.c src/history.c src/history.h src/fsutil.c src/fsutil.h include/plugin-hostd/protocol.h $(RENDERED) + $(CC) -Iinclude -Isrc $(CFLAGS) -Werror -o $@ tests/history_test.c src/history.c src/fsutil.c -lm + # the transforms the format tables name (src/transforms.c, docs/formats.md): every one and its inverse over its vectors, # and a lookup that is not one to one refused test-transforms: $(TRANSFORMS_TEST) diff --git a/README.md b/README.md index 94d3162..235333d 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,8 @@ The settings live in a file, `-c`, `$PLUGIN_HOSTD_CONF` or | `descriptor_dir` | `$OMX_DESCRIPTOR_DIR`, else `` | the scanned descriptors, one file per plugin binary (@-.json, the hash the binary's SHA-256), written by the scan of an add and read by descriptor_get; empty is $XDG_STATE_HOME/omx/descriptors, else /var/lib/omx/descriptors run as root (a system service), else $HOME/.local/state/omx/descriptors; its variable is for tests only | | `qualification_dir` | `$OMX_QUALIFICATION_DIR`, else `` | plugin-qualify's qualification records (verdict, cost, version tested), kept apart from the descriptors; empty is $XDG_STATE_HOME/omx/qualification, else /var/lib/omx/qualification run as root (a system service), else $HOME/.local/state/omx/qualification; its variable is for tests only | | `probe` | `plugin-hostd-probe` | the scanner an add runs, as describe , in a child of its own once per plugin binary; its stdout is the descriptor | +| `history_file` | `` | the running history: what each plugin did on this machine, kept per plugin, binary and host class; empty is $XDG_STATE_HOME/plugin-hostd/history, else $HOME/.local/state/plugin-hostd/history; local to this machine, mode 0600, written by the daemon alone, and deleted by history_clear | +| `host_class` | `` | the host class the history is kept under, as arch/cores/clock (x86_64/5-8/2.5-3.5GHz); empty is the class of this machine, as the probe writes it; no white space | | `ready_timeout_ms` | 5000 ms | a worker that does not accept in this long is a failed spawn | | `rpc_timeout_ms` | 5000 ms | a worker that does not answer in this long is killed | | `backoff_base_ms` | 250 ms | respawn backoff, doubling per death in the window | @@ -126,6 +128,8 @@ The settings live in a file, `-c`, `$PLUGIN_HOSTD_CONF` or | `checkpoint_ms` | 5000 ms | a quiet interval with a changed ledger writes a checkpoint | | `idle_ms` | 25 ms | the period of the idle tick: reap, respawn, checkpoint | | `pool_max` | 8 instances | instances in a pool before a sibling opens | +| `history_window_s` | 60 s | the length of one history window: a worker's CPU and memory are read and folded into the history of its plugins once per window, and a window the plugin was up less than half of is not counted; 0 keeps no history | +| `history_save_s` | 300 s | the history is written at most once this often when it changed, and at quit | | `require_pins` | 1 | 1: add admits only a plugin that has a pin (see pin_mode), its files hashed before any worker sees it, and a plugin of a format whose pin holds only in a world of its own gets a worker of its own, its world the plugin's files (a default that resolves to a pool is placed own); 0: no pin is checked | diff --git a/data/history.json b/data/history.json new file mode 100644 index 0000000..e6514dc --- /dev/null +++ b/data/history.json @@ -0,0 +1,73 @@ +{ + "$schema": "../schema/omx-contract.schema.json", + "requires": { + "omx-contract-schema": ">= 3.3.0" + }, + "PLUGIN_HOSTD_HISTORY_KEYS": { + "kind": "words", + "since": "0.13.0", + "doc": "The keys of a line of the running-history file, in the order the daemon writes them (docs/design/notes/2026-10-27-grouping-planner.md, The history record): history key=value ..., after the header line '# plugin-hostd history/1'. The value of every key is a whole number or a decimal, never negative. A line with a key outside these words is kept as it is by every save and never read.", + "words": { + "first": { + "word": "first", + "doc": "UTC seconds of the first window." + }, + "last": { + "word": "last", + "doc": "UTC seconds of the last window." + }, + "windowsSolo": { + "word": "windows_solo", + "doc": "Windows the plugin ran in a worker of its own." + }, + "windowsGrouped": { + "word": "windows_grouped", + "doc": "Windows the plugin ran in a worker of two or more." + }, + "cpuP95Solo": { + "word": "cpu_p95_solo", + "doc": "Percent of one core, the worker's utime plus stime over a window, an estimate of the 95th percentile of the windows run alone." + }, + "cpuMaxSolo": { + "word": "cpu_max_solo", + "doc": "Percent of one core, the largest window run alone." + }, + "pssMbSolo": { + "word": "pss_mb_solo", + "doc": "MiB, the Pss of the worker's smaps_rollup, the largest window run alone." + }, + "xrunWindowsSolo": { + "word": "xrun_windows_solo", + "doc": "Windows run alone in which JACK counted at least one xrun." + }, + "xrunWindowsGrouped": { + "word": "xrun_windows_grouped", + "doc": "Windows run in a group in which JACK counted at least one xrun." + }, + "crashesSolo": { + "word": "crashes_solo", + "doc": "Worker deaths of a worker of its own that the daemon attributed to the plugin." + }, + "crashesGrouped": { + "word": "crashes_grouped", + "doc": "Worker deaths of a group that the daemon attributed to the plugin." + }, + "restartsGrouped": { + "word": "restarts_grouped", + "doc": "Worker deaths of a group the plugin was a member of and was not the culprit of." + }, + "quarantines": { + "word": "quarantines", + "doc": "Times the supervisor quarantined the plugin." + }, + "lastCrash": { + "word": "last_crash", + "doc": "UTC seconds of the last crash the plugin was blamed for alone." + }, + "lastCrashGrouped": { + "word": "last_crash_grouped", + "doc": "UTC seconds of the last crash the plugin was blamed for in a group." + } + } + } +} diff --git a/doc/plugin-hostd.1 b/doc/plugin-hostd.1 index eb0b7e7..602be28 100644 --- a/doc/plugin-hostd.1 +++ b/doc/plugin-hostd.1 @@ -72,7 +72,7 @@ or in one .\" BEGIN GENERATED protocol:man-keys .RI \(dq key " value" \(dq -per line: mod_host, clap_host, lv2_worker, lv2_path, state_root, pin_mode, pin_store, pin_file, pins_dir, descriptor_dir, qualification_dir, probe, ready_timeout_ms, rpc_timeout_ms, backoff_base_ms, backoff_max_ms, storm_deaths, storm_window_ms, suspect_window_ms, checkpoint_ms, idle_ms, pool_max, require_pins. +per line: mod_host, clap_host, lv2_worker, lv2_path, state_root, pin_mode, pin_store, pin_file, pins_dir, descriptor_dir, qualification_dir, probe, history_file, host_class, ready_timeout_ms, rpc_timeout_ms, backoff_base_ms, backoff_max_ms, storm_deaths, storm_window_ms, suspect_window_ms, checkpoint_ms, idle_ms, pool_max, history_window_s, history_save_s, require_pins. .\" END GENERATED protocol:man-keys .PP The development package installs the schemas of the documents the daemon and its probe write or read, beside diff --git a/include/plugin-hostd/plugin_hostd_vocabulary.h b/include/plugin-hostd/plugin_hostd_vocabulary.h index 48b93bb..7aa5bcc 100644 --- a/include/plugin-hostd/plugin_hostd_vocabulary.h +++ b/include/plugin-hostd/plugin_hostd_vocabulary.h @@ -10,6 +10,25 @@ * literal of its own. */ +/* PLUGIN_HOSTD_HISTORY_KEYS (data/history.json) */ +#define OMX_PLUGIN_HOSTD_HISTORY_KEYS_FIRST "first" ///< UTC seconds of the first window. +#define OMX_PLUGIN_HOSTD_HISTORY_KEYS_LAST "last" ///< UTC seconds of the last window. +#define OMX_PLUGIN_HOSTD_HISTORY_KEYS_WINDOWS_SOLO "windows_solo" ///< Windows the plugin ran in a worker of its own. +#define OMX_PLUGIN_HOSTD_HISTORY_KEYS_WINDOWS_GROUPED "windows_grouped" ///< Windows the plugin ran in a worker of two or more. +#define OMX_PLUGIN_HOSTD_HISTORY_KEYS_CPU_P95_SOLO "cpu_p95_solo" ///< Percent of one core, the worker's utime plus stime over a window, an estimate of the 95th percentile of the windows run alone. +#define OMX_PLUGIN_HOSTD_HISTORY_KEYS_CPU_MAX_SOLO "cpu_max_solo" ///< Percent of one core, the largest window run alone. +#define OMX_PLUGIN_HOSTD_HISTORY_KEYS_PSS_MB_SOLO "pss_mb_solo" ///< MiB, the Pss of the worker's smaps_rollup, the largest window run alone. +#define OMX_PLUGIN_HOSTD_HISTORY_KEYS_XRUN_WINDOWS_SOLO "xrun_windows_solo" ///< Windows run alone in which JACK counted at least one xrun. +#define OMX_PLUGIN_HOSTD_HISTORY_KEYS_XRUN_WINDOWS_GROUPED "xrun_windows_grouped" ///< Windows run in a group in which JACK counted at least one xrun. +#define OMX_PLUGIN_HOSTD_HISTORY_KEYS_CRASHES_SOLO "crashes_solo" ///< Worker deaths of a worker of its own that the daemon attributed to the plugin. +#define OMX_PLUGIN_HOSTD_HISTORY_KEYS_CRASHES_GROUPED "crashes_grouped" ///< Worker deaths of a group that the daemon attributed to the plugin. +#define OMX_PLUGIN_HOSTD_HISTORY_KEYS_RESTARTS_GROUPED "restarts_grouped" ///< Worker deaths of a group the plugin was a member of and was not the culprit of. +#define OMX_PLUGIN_HOSTD_HISTORY_KEYS_QUARANTINES "quarantines" ///< Times the supervisor quarantined the plugin. +#define OMX_PLUGIN_HOSTD_HISTORY_KEYS_LAST_CRASH "last_crash" ///< UTC seconds of the last crash the plugin was blamed for alone. +#define OMX_PLUGIN_HOSTD_HISTORY_KEYS_LAST_CRASH_GROUPED "last_crash_grouped" ///< UTC seconds of the last crash the plugin was blamed for in a group. +#define OMX_PLUGIN_HOSTD_HISTORY_KEYS_COUNT 15u +#define OMX_PLUGIN_HOSTD_HISTORY_KEYS_INIT { "first", "last", "windows_solo", "windows_grouped", "cpu_p95_solo", "cpu_max_solo", "pss_mb_solo", "xrun_windows_solo", "xrun_windows_grouped", "crashes_solo", "crashes_grouped", "restarts_grouped", "quarantines", "last_crash", "last_crash_grouped" } + /* PLUGIN_HOSTD_REPLY_CODES (data/verbs.json) */ #define OMX_REPLY_PLUGIN_HOSTD_PLACEMENT_INVALID "reply.plugin-hostd.placement-invalid" ///< PHD_ERR_PLACEMENT_INVALID: not a placement, or not valid here #define OMX_REPLY_PLUGIN_HOSTD_PLACEMENT_INVALID_RESP (-501) diff --git a/include/plugin-hostd/protocol.h b/include/plugin-hostd/protocol.h index 9f4fc1b..1d8f390 100644 --- a/include/plugin-hostd/protocol.h +++ b/include/plugin-hostd/protocol.h @@ -972,6 +972,22 @@ static inline int phd_is_note_verb(const char *verb) #define PHD_DEFAULT_PIN_MODE PHD_PIN_MODE_FIRST_USE #define PHD_DEFAULT_PINS_DIR "/usr/share/plugin-hostd/pins.d" #define PHD_DEFAULT_PROBE "plugin-hostd-probe" +#define PHD_DEFAULT_HISTORY_WINDOW_S 60 +#define PHD_DEFAULT_HISTORY_SAVE_S 300 + +/* The running history (docs/design/notes/2026-10-27-grouping-planner.md, The history record): what each plugin did on + * this machine, one line per in a file of the daemon's own, whose keys are the words + * of PLUGIN_HOSTD_HISTORY_KEYS (data/history.json). The first line is PHD_HISTORY_HEADER; the file is local, mode 0600, + * and only the daemon writes it. The summary history_get answers reads crashes within PHD_HISTORY_KEEP_DAYS of + * last_crash, and marks a plugin raises_xruns once it has PHD_HISTORY_MIN_WINDOWS grouped windows and its share of xrun + * windows grouped exceeds its share solo by PHD_HISTORY_XRUN_MARGIN_PCT percentage points, or exceeds + * PHD_HISTORY_XRUN_MARGIN_PCT percent when it never ran alone. */ +#define PHD_HISTORY_NAME "plugin-hostd/history" +#define PHD_HISTORY_HEADER "# plugin-hostd history/1" +#define PHD_HISTORY_RECORD "history" +#define PHD_HISTORY_KEEP_DAYS 30 +#define PHD_HISTORY_MIN_WINDOWS 30 +#define PHD_HISTORY_XRUN_MARGIN_PCT 5 /* The directories of what the scan of an add gathers: state, kept across a reboot. An empty setting is the default: * /, where the state home is $XDG_STATE_HOME, else /var/lib for a daemon run as root (a system @@ -1028,7 +1044,12 @@ static inline int phd_is_note_verb(const char *verb) "version tested), kept apart from the descriptors; " PHD_DIR_DEFAULT_TEXT(PHD_DIR_QUALIFICATION_SUBDIR) \ "; its variable is for tests only") \ X(probe, path, PHD_DEFAULT_PROBE, "", "the scanner an add runs, as describe , in a child of its " \ - "own once per plugin binary; its stdout is the descriptor") + "own once per plugin binary; its stdout is the descriptor") \ + X(history_file, path, "", "", "the running history: what each plugin did on this machine, kept per plugin, binary and " \ + "host class; empty is $XDG_STATE_HOME/" PHD_HISTORY_NAME ", else $HOME/.local/state/" PHD_HISTORY_NAME "; local to " \ + "this machine, mode 0600, written by the daemon alone, and deleted by history_clear") \ + X(host_class, word, "", "", "the host class the history is kept under, as arch/cores/clock (x86_64/5-8/2.5-3.5GHz); " \ + "empty is the class of this machine, as the probe writes it; no white space") /* A whole-number setting: X(key, default, unit, meaning); 0 turns off where the meaning says so */ #define PHD_CONF_INTS(X) \ @@ -1043,6 +1064,11 @@ static inline int phd_is_note_verb(const char *verb) X(checkpoint_ms, PHD_DEFAULT_CHECKPOINT_MS, "ms", "a quiet interval with a changed ledger writes a checkpoint") \ X(idle_ms, PHD_DEFAULT_IDLE_MS, "ms", "the period of the idle tick: reap, respawn, checkpoint") \ X(pool_max, PHD_DEFAULT_POOL_MAX, "instances", "instances in a pool before a sibling opens") \ + X(history_window_s, PHD_DEFAULT_HISTORY_WINDOW_S, "s", \ + "the length of one history window: a worker's CPU and memory are read and folded into the history of its plugins " \ + "once per window, and a window the plugin was up less than half of is not counted; 0 keeps no history") \ + X(history_save_s, PHD_DEFAULT_HISTORY_SAVE_S, "s", "the history is written at most once this often when it changed, " \ + "and at quit") \ X(require_pins, PHD_DEFAULT_REQUIRE_PINS, "", \ "1: add admits only a plugin that has a pin (see pin_mode), its files hashed before any worker sees it, and a plugin of a " \ "format whose pin holds only in a world of its own gets a worker of its own, its world the plugin's files (a " \ diff --git a/protocol/plugin-hostd.json b/protocol/plugin-hostd.json index 637f3bc..06ca8a3 100644 --- a/protocol/plugin-hostd.json +++ b/protocol/plugin-hostd.json @@ -231,6 +231,8 @@ {"key": "descriptor_dir", "type": "string", "default": "", "env": "OMX_DESCRIPTOR_DIR", "meaning": "the scanned descriptors, one file per plugin binary (@-.json, the hash the binary's SHA-256), written by the scan of an add and read by descriptor_get; empty is $XDG_STATE_HOME/omx/descriptors, else /var/lib/omx/descriptors run as root (a system service), else $HOME/.local/state/omx/descriptors; its variable is for tests only"}, {"key": "qualification_dir", "type": "string", "default": "", "env": "OMX_QUALIFICATION_DIR", "meaning": "plugin-qualify's qualification records (verdict, cost, version tested), kept apart from the descriptors; empty is $XDG_STATE_HOME/omx/qualification, else /var/lib/omx/qualification run as root (a system service), else $HOME/.local/state/omx/qualification; its variable is for tests only"}, {"key": "probe", "type": "string", "default": "plugin-hostd-probe", "env": "", "meaning": "the scanner an add runs, as describe , in a child of its own once per plugin binary; its stdout is the descriptor"}, + {"key": "history_file", "type": "string", "default": "", "env": "", "meaning": "the running history: what each plugin did on this machine, kept per plugin, binary and host class; empty is $XDG_STATE_HOME/plugin-hostd/history, else $HOME/.local/state/plugin-hostd/history; local to this machine, mode 0600, written by the daemon alone, and deleted by history_clear"}, + {"key": "host_class", "type": "string", "default": "", "env": "", "meaning": "the host class the history is kept under, as arch/cores/clock (x86_64/5-8/2.5-3.5GHz); empty is the class of this machine, as the probe writes it; no white space"}, {"key": "ready_timeout_ms", "type": "int", "default": 5000, "unit": "ms", "meaning": "a worker that does not accept in this long is a failed spawn"}, {"key": "rpc_timeout_ms", "type": "int", "default": 5000, "unit": "ms", "meaning": "a worker that does not answer in this long is killed"}, {"key": "backoff_base_ms", "type": "int", "default": 250, "unit": "ms", "meaning": "respawn backoff, doubling per death in the window"}, @@ -241,6 +243,8 @@ {"key": "checkpoint_ms", "type": "int", "default": 5000, "unit": "ms", "meaning": "a quiet interval with a changed ledger writes a checkpoint"}, {"key": "idle_ms", "type": "int", "default": 25, "unit": "ms", "meaning": "the period of the idle tick: reap, respawn, checkpoint"}, {"key": "pool_max", "type": "int", "default": 8, "unit": "instances", "meaning": "instances in a pool before a sibling opens"}, + {"key": "history_window_s", "type": "int", "default": 60, "unit": "s", "meaning": "the length of one history window: a worker's CPU and memory are read and folded into the history of its plugins once per window, and a window the plugin was up less than half of is not counted; 0 keeps no history"}, + {"key": "history_save_s", "type": "int", "default": 300, "unit": "s", "meaning": "the history is written at most once this often when it changed, and at quit"}, {"key": "require_pins", "type": "int", "default": 1, "unit": "", "meaning": "1: add admits only a plugin that has a pin (see pin_mode), its files hashed before any worker sees it, and a plugin of a format whose pin holds only in a world of its own gets a worker of its own, its world the plugin's files (a default that resolves to a pool is placed own); 0: no pin is checked"} ] } diff --git a/src/history.c b/src/history.c new file mode 100644 index 0000000..96c81bf --- /dev/null +++ b/src/history.c @@ -0,0 +1,659 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * This file is part of plugin-hostd. + * + * plugin-hostd is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * plugin-hostd is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with plugin-hostd. If not, see . + * + * Copyright (C) 2026 Pau Aliagas + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "fsutil.h" +#include "history.h" + +#define HISTORY_LINE_MAX 8192 +#define VERSION_PREFIX "# plugin-hostd history/" +#define SECONDS_PER_DAY 86400 + +/* The keys are the words of PLUGIN_HOSTD_HISTORY_KEYS in their order; what each holds is the type here, one row per + * word (tests/history_test.c holds the two to each other). */ +typedef enum { K_COUNT, K_TIME, K_PERCENT, K_MIB } kind_t; + +static const char *const g_words[] = OMX_PLUGIN_HOSTD_HISTORY_KEYS_INIT; +#define NKEYS OMX_PLUGIN_HOSTD_HISTORY_KEYS_COUNT + +static const struct { kind_t kind; size_t offset; } g_keys[] = { + { K_TIME, offsetof(history_rec_t, first) }, + { K_TIME, offsetof(history_rec_t, last) }, + { K_COUNT, offsetof(history_rec_t, windows_solo) }, + { K_COUNT, offsetof(history_rec_t, windows_grouped) }, + { K_PERCENT, offsetof(history_rec_t, cpu_p95_solo) }, + { K_PERCENT, offsetof(history_rec_t, cpu_max_solo) }, + { K_MIB, offsetof(history_rec_t, pss_mb_solo) }, + { K_COUNT, offsetof(history_rec_t, xrun_windows_solo) }, + { K_COUNT, offsetof(history_rec_t, xrun_windows_grouped) }, + { K_COUNT, offsetof(history_rec_t, crashes_solo) }, + { K_COUNT, offsetof(history_rec_t, crashes_grouped) }, + { K_COUNT, offsetof(history_rec_t, restarts_grouped) }, + { K_COUNT, offsetof(history_rec_t, quarantines) }, + { K_TIME, offsetof(history_rec_t, last_crash) }, + { K_TIME, offsetof(history_rec_t, last_crash_grouped) }, +}; + +_Static_assert(sizeof(g_keys) / sizeof(g_keys[0]) == NKEYS, "one type per word of PLUGIN_HOSTD_HISTORY_KEYS"); + +/* one line of the file, in the file's order: a record, or a line this version keeps and does not read */ +typedef struct ENTRY_T { + history_rec_t rec; + char *raw; /* the line as it was, NULL for a record */ +} entry_t; + +static entry_t *g_entry; +static int g_n; +static char g_path[PATH_MAX]; +static char *g_class; +static int g_save_s; +static int g_dirty, g_blocked, g_started; +static long long g_saved_at; +static int g_saved_once; + +/* ---------------------------------------------------------------- the line grammar */ + +static void rec_free(history_rec_t *r) +{ + free(r->uri); + free(r->klass); +} + +static void entry_free(entry_t *e) +{ + rec_free(&e->rec); + free(e->raw); +} + +static void *field(history_rec_t *r, int key) +{ + return (char *)r + g_keys[key].offset; +} + +static int key_of(const char *word) +{ + int n; + + for (n = 0; n < (int)NKEYS; n++) + if (!strcmp(word, g_words[n])) + return n; + return -1; +} + +/* a value as the file writes it: a whole number, or a decimal; never negative, never a word */ +static int number_ok(const char *text, kind_t kind, void *out) +{ + char *end; + + if (!*text || *text == '-' || *text == '+' || isspace((unsigned char)*text)) + return 0; + errno = 0; + if (kind == K_COUNT || kind == K_TIME) + { + long long v = strtoll(text, &end, 10); + + if (*end || errno) + return 0; + *(long long *)out = v; + } + else + { + double v = strtod(text, &end); + + if (*end || errno || !isfinite(v)) + return 0; + *(double *)out = v; + } + return 1; +} + +/* a record from the words of a line; 0, or -1 when the line is outside the grammar or holds a key this version does + * not know. `line` is cut up. */ +static int rec_parse(char *line, history_rec_t *r) +{ + char *tok[4 + NKEYS], *save = NULL, *t; + unsigned seen = 0; + int ntok = 0, n; + + memset(r, 0, sizeof(*r)); + for (t = strtok_r(line, " \t\r\n", &save); t && ntok < (int)(sizeof(tok) / sizeof(tok[0])); t = strtok_r(NULL, " \t\r\n", &save)) + tok[ntok++] = t; + if (t || ntok < 4 || strcmp(tok[0], PHD_HISTORY_RECORD) || !phd_pin_hex_valid(tok[2])) + return -1; + for (n = 4; n < ntok; n++) + { + char *eq = strchr(tok[n], '='); + int key; + + if (!eq) + return -1; + *eq = '\0'; + key = key_of(tok[n]); + if (key < 0 || (seen & (1u << key)) || !number_ok(eq + 1, g_keys[key].kind, field(r, key))) + return -1; + seen |= 1u << key; + } + r->uri = strdup(tok[1]); + r->klass = strdup(tok[3]); + memcpy(r->digest, tok[2], PHD_SHA256_HEX_LEN + 1); + if (!r->uri || !r->klass) + { + rec_free(r); + return -1; + } + return 0; +} + +/* " key=value" for every key, in the order of the words */ +static void keys_write(FILE *out, const history_rec_t *r) +{ + int n; + + for (n = 0; n < (int)NKEYS; n++) + { + const void *v = (const char *)r + g_keys[n].offset; + + switch (g_keys[n].kind) + { + case K_PERCENT: + fprintf(out, " %s=%.2f", g_words[n], *(const double *)v); + break; + case K_MIB: + fprintf(out, " %s=%.1f", g_words[n], *(const double *)v); + break; + default: + fprintf(out, " %s=%lld", g_words[n], *(const long long *)v); + } + } +} + +char *history_line(const history_rec_t *r) +{ + char *buf = NULL; + size_t size = 0; + FILE *mem = open_memstream(&buf, &size); + + if (!mem) + return NULL; + fprintf(mem, "%s %s %s %s", PHD_HISTORY_RECORD, r->uri, r->digest, r->klass); + keys_write(mem, r); + fputc('\n', mem); + fclose(mem); + return buf; +} + +/* ---------------------------------------------------------------- the table */ + +static entry_t *entry_add(void) +{ + entry_t *grown = realloc(g_entry, (g_n + 1) * sizeof(entry_t)); + + if (!grown) + return NULL; + g_entry = grown; + memset(&g_entry[g_n], 0, sizeof(entry_t)); + return &g_entry[g_n++]; +} + +static history_rec_t *rec_at(const char *uri, const char *digest, const char *klass) +{ + int n; + + for (n = 0; n < g_n; n++) + if (!g_entry[n].raw && !strcmp(g_entry[n].rec.uri, uri) && !strcmp(g_entry[n].rec.digest, digest) && + !strcmp(g_entry[n].rec.klass, klass)) + return &g_entry[n].rec; + return NULL; +} + +/* the record of this class, made when there is none; NULL when the history is unavailable or out of memory */ +static history_rec_t *rec_get(const char *uri, const char *digest) +{ + history_rec_t *r; + entry_t *e; + + if (!g_started || g_blocked || !uri || !*uri || !phd_pin_hex_valid(digest) || strpbrk(uri, " \t\r\n")) + return NULL; + if ((r = rec_at(uri, digest, g_class))) + return r; + if (!(e = entry_add())) + return NULL; + e->rec.uri = strdup(uri); + e->rec.klass = strdup(g_class); + memcpy(e->rec.digest, digest, PHD_SHA256_HEX_LEN + 1); + if (!e->rec.uri || !e->rec.klass) + { + entry_free(e); + g_n--; + return NULL; + } + return &e->rec; +} + +const history_rec_t *history_find(const char *uri, const char *digest) +{ + return g_started && !g_blocked && uri && digest ? rec_at(uri, digest, g_class) : NULL; +} + +int history_available(void) +{ + return g_started && !g_blocked; +} + +/* ---------------------------------------------------------------- reading and writing the file */ + +static const char *path_of(const char *configured) +{ + const char *state = getenv("XDG_STATE_HOME"), *home = getenv("HOME"); + + if (*configured) + snprintf(g_path, sizeof(g_path), "%s", configured); + else if (state && *state) + snprintf(g_path, sizeof(g_path), "%s/" PHD_HISTORY_NAME, state); + else if (home && *home) + snprintf(g_path, sizeof(g_path), "%s/.local/state/" PHD_HISTORY_NAME, home); + else + g_path[0] = '\0'; + return g_path; +} + +static void block(const char *why) +{ + fprintf(stderr, "plugin-hostd: the history %s %s; it is neither read nor overwritten, and none is kept until the " + "operator repairs or removes it\n", g_path, why); + g_blocked = 1; +} + +static void held_add(const char *line, size_t n) +{ + entry_t *e = entry_add(); + + if (e && !(e->raw = strndup(line, n))) + g_n--; +} + +static void file_read(void) +{ + char *line = NULL; + size_t cap = 0; + ssize_t len; + FILE *file = fopen(g_path, "r"); + int lineno = 0, header = 0; + + if (!file) + { + if (errno != ENOENT && errno != ENOTDIR) + { + char why[160]; + + snprintf(why, sizeof(why), "can't be read: %s", strerror(errno)); + block(why); + } + return; + } + while ((len = getline(&line, &cap, file)) >= 0) + { + size_t n = len; + char copy[HISTORY_LINE_MAX]; + history_rec_t rec; + + lineno++; + while (n && (line[n - 1] == '\n' || line[n - 1] == '\r')) + n--; + if (!header) + { + if (!n) + continue; + if (n != strlen(PHD_HISTORY_HEADER) || strncmp(line, PHD_HISTORY_HEADER, n)) + { + block(!strncmp(line, VERSION_PREFIX, strlen(VERSION_PREFIX)) ? "has a version this daemon does not know" + : "is not a history file"); + break; + } + header = 1; + continue; + } + if (!n) + continue; + if (line[0] == '#') + { + held_add(line, n); + continue; + } + if (n >= sizeof(copy)) + { + fprintf(stderr, "plugin-hostd: %s:%d: the line is too long; it is kept as it is and not read\n", g_path, lineno); + held_add(line, n); + continue; + } + memcpy(copy, line, n); + copy[n] = '\0'; + if (rec_parse(copy, &rec) != 0) + { + fprintf(stderr, "plugin-hostd: %s:%d: not a history line, or a key this version does not know; it is kept as " + "it is and not read\n", g_path, lineno); + held_add(line, n); + } + else if (rec_at(rec.uri, rec.digest, rec.klass)) + { + fprintf(stderr, "plugin-hostd: %s:%d: a second line of one plugin binary on one host class; it is kept as it " + "is and not read\n", g_path, lineno); + rec_free(&rec); + held_add(line, n); + } + else + { + entry_t *e = entry_add(); + + if (e) + e->rec = rec; + else + rec_free(&rec); + } + } + if (!g_blocked && ferror(file)) + { + char why[160]; + + snprintf(why, sizeof(why), "can't be read: %s", strerror(errno)); + block(why); + } + free(line); + fclose(file); +} + +/* the history to the file: a temporary file of its own beside it (mkstemp, mode 0600), written, fsynced and renamed over + * it, then the directory fsynced. A failure is announced and leaves the history in memory */ +static int file_write(void) +{ + char tmp[PATH_MAX + 16]; + FILE *file; + int n, fd = -1, ok; + + if (!*g_path) + return -1; + snprintf(tmp, sizeof(tmp), "%s.XXXXXX", g_path); + if (fs_mkdir_parents(tmp) != 0 || (fd = mkstemp(tmp)) < 0 || !(file = fdopen(fd, "w"))) + { + int err = errno; + + if (fd >= 0) + { + close(fd); + unlink(tmp); + } + fprintf(stderr, "plugin-hostd: can't write the history %s: %s; it is kept in memory and written again at the next " + "save\n", g_path, strerror(err)); + return -1; + } + fprintf(file, "%s\n", PHD_HISTORY_HEADER); + for (n = 0; n < g_n; n++) + { + char *line = g_entry[n].raw ? NULL : history_line(&g_entry[n].rec); + + if (g_entry[n].raw) + fprintf(file, "%s\n", g_entry[n].raw); + else if (line) + fputs(line, file); + free(line); + } + ok = fflush(file) == 0 && !ferror(file) && fsync(fileno(file)) == 0; + ok = fclose(file) == 0 && ok; + if (!ok || rename(tmp, g_path) != 0) + { + fprintf(stderr, "plugin-hostd: can't write the history %s: %s; it is kept in memory and written again at the next " + "save\n", g_path, strerror(errno)); + unlink(tmp); + return -1; + } + fs_sync_dir(g_path); + return 0; +} + +static void save(long long now) +{ + g_saved_at = now; + g_saved_once = 1; + if (file_write() == 0) + g_dirty = 0; +} + +int history_init(const char *file, const char *klass, int save_s) +{ + history_finish(); + g_started = 1; + g_class = strdup(klass); + g_save_s = save_s; + if (!g_class || !*path_of(file)) + { + if (!g_class) + g_started = 0; + else + fprintf(stderr, "plugin-hostd: no history file: neither history_file, $XDG_STATE_HOME nor $HOME is set; no " + "history is kept\n"), g_blocked = 1; + return 0; + } + file_read(); + return 0; +} + +void history_tick(long long now) +{ + if (g_started && !g_blocked && g_dirty && (!g_saved_once || now - g_saved_at >= g_save_s)) + save(now); +} + +void history_finish(void) +{ + int n; + + if (g_started && !g_blocked && g_dirty) + file_write(); + for (n = 0; n < g_n; n++) + entry_free(&g_entry[n]); + free(g_entry); + g_entry = NULL; + g_n = 0; + free(g_class); + g_class = NULL; + g_dirty = g_blocked = g_started = g_saved_once = 0; + g_saved_at = 0; +} + +/* ---------------------------------------------------------------- what is counted */ + +static void touch(history_rec_t *r, long long now) +{ + if (!r->first) + r->first = now; + r->last = now; + g_dirty = 1; +} + +/* An estimate of the 95th percentile of the windows run alone that needs no list of them: each window moves the + * estimate up by 95 parts, or down by 5, of a step that is 2 percent of the estimate (a core's percent never below 1): + * it rests where one window in twenty is above it. The first window is the estimate. */ +static void p95_fold(history_rec_t *r, double cpu) +{ + double step; + + if (r->windows_solo == 0) + { + r->cpu_p95_solo = cpu; + return; + } + step = 0.02 * (r->cpu_p95_solo > 1.0 ? r->cpu_p95_solo : 1.0); + r->cpu_p95_solo += cpu > r->cpu_p95_solo ? 0.95 * step : -0.05 * step; + if (r->cpu_p95_solo < 0) + r->cpu_p95_solo = 0; +} + +void history_window(const char *uri, const char *digest, int grouped, double cpu_pct, double pss_mb, long xruns, + long long now) +{ + history_rec_t *r = rec_get(uri, digest); + + if (!r) + return; + touch(r, now); + if (grouped) + { + r->windows_grouped++; + r->xrun_windows_grouped += xruns > 0; + return; + } + if (!(cpu_pct >= 0)) + cpu_pct = 0; + p95_fold(r, cpu_pct); + if (r->windows_solo == 0 || cpu_pct > r->cpu_max_solo) + r->cpu_max_solo = cpu_pct; + if (pss_mb > r->pss_mb_solo) + r->pss_mb_solo = pss_mb; + r->windows_solo++; + r->xrun_windows_solo += xruns > 0; +} + +void history_crash(const char *uri, const char *digest, int grouped, long long now) +{ + history_rec_t *r = rec_get(uri, digest); + + if (!r) + return; + touch(r, now); + if (grouped) + { + r->crashes_grouped++; + r->last_crash_grouped = now; + } + else + { + r->crashes_solo++; + r->last_crash = now; + } +} + +void history_restart(const char *uri, const char *digest, int grouped, long long now) +{ + history_rec_t *r = grouped ? rec_get(uri, digest) : NULL; + + if (!r) + return; + touch(r, now); + r->restarts_grouped++; +} + +void history_quarantine(const char *uri, const char *digest, long long now) +{ + history_rec_t *r = rec_get(uri, digest); + + if (!r) + return; + touch(r, now); + r->quarantines++; +} + +/* ---------------------------------------------------------------- what is read */ + +static int crashed(const history_rec_t *r, long long now) +{ + long long last = r->last_crash > r->last_crash_grouped ? r->last_crash : r->last_crash_grouped; + + return r->crashes_solo + r->crashes_grouped > 0 && now - last <= (long long)PHD_HISTORY_KEEP_DAYS * SECONDS_PER_DAY; +} + +/* the share of xrun windows grouped exceeds the share alone by the margin: xg/wg > xs/ws + m/100, cross-multiplied */ +static int raises_xruns(const history_rec_t *r) +{ + long long wg = r->windows_grouped, xg = r->xrun_windows_grouped, ws = r->windows_solo, xs = r->xrun_windows_solo; + + if (wg < PHD_HISTORY_MIN_WINDOWS) + return 0; + if (ws == 0) + return 100 * xg > (long long)PHD_HISTORY_XRUN_MARGIN_PCT * wg; + return 100 * xg * ws > 100 * xs * wg + (long long)PHD_HISTORY_XRUN_MARGIN_PCT * wg * ws; +} + +char *history_summary(const char *uri, const char *digest, long long now) +{ + const history_rec_t *r = history_find(uri, digest); + char *buf = NULL; + size_t size = 0; + FILE *mem; + + if (!r || !(mem = open_memstream(&buf, &size))) + return NULL; + fprintf(mem, "%s %s crashed=%d raises_xruns=%d", r->digest, r->klass, crashed(r, now), raises_xruns(r)); + keys_write(mem, r); + fclose(mem); + return buf; +} + +/* ---------------------------------------------------------------- clear */ + +/* the uri a held line names: its second word, after history */ +static int raw_names(const char *raw, const char *uri) +{ + char copy[HISTORY_LINE_MAX], *save = NULL, *t; + + snprintf(copy, sizeof(copy), "%s", raw); + t = strtok_r(copy, " \t", &save); + if (!t || strcmp(t, PHD_HISTORY_RECORD)) + return 0; + t = strtok_r(NULL, " \t", &save); + return t && !strcmp(t, uri); +} + +int history_clear(const char *uri) +{ + int n, keep = 0, all = !strcmp(uri, PHD_WORD_ALL); + + if (!g_started || g_blocked) + return -1; + for (n = 0; n < g_n; n++) + { + int gone = all || (g_entry[n].raw ? raw_names(g_entry[n].raw, uri) : !strcmp(g_entry[n].rec.uri, uri)); + + if (gone) + entry_free(&g_entry[n]); + else + g_entry[keep++] = g_entry[n]; + } + if (keep != g_n) + { + g_n = keep; + g_dirty = 1; + if (file_write() == 0) + g_dirty = 0; + } + return 0; +} diff --git a/src/history.h b/src/history.h new file mode 100644 index 0000000..0eb2e26 --- /dev/null +++ b/src/history.h @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * This file is part of plugin-hostd. + * + * plugin-hostd is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * plugin-hostd is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with plugin-hostd. If not, see . + * + * Copyright (C) 2026 Pau Aliagas + */ + +#ifndef HISTORY_H +#define HISTORY_H + +#include + +#include + +/* The running history: what each plugin did on this machine, per plugin, per binary and per host class + * (docs/design/notes/2026-10-27-grouping-planner.md, The history record). One line per : history key=value ...; the keys are PLUGIN_HOSTD_HISTORY_KEYS (data/history.json), the + * first line PHD_HISTORY_HEADER. Written only by the daemon, atomically, mode 0600; local to the machine. + * + * Nothing here reads a clock or a /proc file: the caller says what the window measured and what time it is. */ + +typedef struct HISTORY_REC_T { + char *uri; + char digest[PHD_SHA256_HEX_LEN + 1]; + char *klass; + long long first, last, last_crash, last_crash_grouped; /* UTC seconds */ + long long windows_solo, windows_grouped; + long long xrun_windows_solo, xrun_windows_grouped; + long long crashes_solo, crashes_grouped, restarts_grouped, quarantines; + double cpu_p95_solo, cpu_max_solo; /* percent of one core */ + double pss_mb_solo; /* MiB */ +} history_rec_t; + +/* start: read `file` (empty is $XDG_STATE_HOME/plugin-hostd/history, else $HOME/.local/state/plugin-hostd/history), + * `klass` the host class this daemon records under, `save_s` the least seconds between two saves. A missing file is + * an empty history. A file that cannot be read, or whose first line is not PHD_HISTORY_HEADER (an unknown version + * among them), blocks reading and saving it: stderr says why, the history is unavailable, and the file is not touched. + * A line outside the grammar, or with a key this version does not know, is kept verbatim by every save and not read. + * Always 0: the daemon runs without a history. */ +int history_init(const char *file, const char *klass, int save_s); + +/* the history is readable and writable */ +int history_available(void); + +/* the record of a plugin binary on this host class; NULL when there is none or the history is unavailable */ +const history_rec_t *history_find(const char *uri, const char *digest); + +/* a closed window of the worker that ran the plugin: alone, or in a group of two or more; its CPU as percent of one + * core over the window, its Pss in MiB, the xruns JACK counted in it (the worker's report; 0 when it makes none) */ +void history_window(const char *uri, const char *digest, int grouped, double cpu_pct, double pss_mb, long xruns, + long long now); + +/* the supervisor's counts: a worker death the daemon blamed on the plugin, a death of its group that was not its own + * fault, a quarantine */ +void history_crash(const char *uri, const char *digest, int grouped, long long now); +void history_restart(const char *uri, const char *digest, int grouped, long long now); +void history_quarantine(const char *uri, const char *digest, long long now); + +/* the summary the planner reads: crashed when a crash is within PHD_HISTORY_KEEP_DAYS of `now`, raises_xruns by the + * margin of protocol.h; malloc'd " crashed=<0|1> raises_xruns=<0|1> key=value ...", NULL for no record */ +char *history_summary(const char *uri, const char *digest, long long now); + +/* the idle tick: write the history when it changed and `save_s` have passed since the last write (`now` seconds, + * any steady origin) */ +void history_tick(long long now); + +/* history_clear: the lines of one uri (every binary, class, and a held line that names it), or all of them, at once and + * saved. 0, or -1 when the history is unavailable */ +int history_clear(const char *uri); + +/* write the history now if it changed, and free it */ +void history_finish(void); + +/* the line of a record, "history key=value ...\n"; for the tests of the grammar. NULL on + * a failure of memory; malloc'd */ +char *history_line(const history_rec_t *rec); + +#endif diff --git a/tests/history_test.c b/tests/history_test.c new file mode 100644 index 0000000..e5d6ada --- /dev/null +++ b/tests/history_test.c @@ -0,0 +1,660 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * This file is part of plugin-hostd. + * + * plugin-hostd is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * plugin-hostd is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with plugin-hostd. If not, see . + * + * Copyright (C) 2026 Pau Aliagas + */ + +/* src/history.c against records and files written out by hand: the line a record becomes, what a load gives back, the + * lines and files it must keep and refuse, what the counts and the summary come to. Nothing here reads a clock: every + * time is a number the test passes in. Each group prints "ok " when none of its checks failed. */ + +#include +#include +#include +#include +#include + +#include +#include + +#include "history.h" + +#define CLASS "x86_64/5-8/2.5-3.5GHz" +#define URI "clap:/p/a.clap#org.test.a" +#define DIGEST "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" +#define OTHER "cb7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" +#define DAY 86400LL + +static int failures, group_failures; +static char dir[256], file[300]; + +static void check(int ok, const char *what) +{ + if (!ok) + { + group_failures++; + printf("FAIL history: %s\n", what); + } +} + +static void check_str(const char *got, const char *want, const char *what) +{ + if (!got || strcmp(got, want)) + { + group_failures++; + printf("FAIL history: %s\n got %s\n want %s\n", what, got ? got : "(null)", want); + } +} + +static void group(const char *name) +{ + if (group_failures) + failures += group_failures; + else if (name) + printf("ok %s\n", name); + group_failures = 0; +} + +static char *slurp(const char *path) +{ + char *buf; + FILE *f = fopen(path, "r"); + size_t n; + + if (!f) + return NULL; + buf = calloc(1, 65536); + n = fread(buf, 1, 65535, f); + buf[n] = '\0'; + fclose(f); + return buf; +} + +static void put(const char *path, const char *text) +{ + FILE *f = fopen(path, "w"); + + if (!f) + { + printf("FAIL history: test can't write %s\n", path); + exit(2); + } + fputs(text, f); + fclose(f); +} + +/* the file as text, "" when there is none */ +static char *file_text(void) +{ + char *t = slurp(file); + + return t ? t : strdup(""); +} + +static void fresh(void) +{ + char cmd[400]; + + snprintf(cmd, sizeof(cmd), "rm -rf '%s'/*", dir); + if (system(cmd) != 0) + exit(2); +} + +static void start(void) +{ + history_init(file, CLASS, 300); +} + +#define HEADER PHD_HISTORY_HEADER "\n" + +static void count_everything(void) +{ + history_window(URI, DIGEST, 0, 10.0, 12.3, 0, 1000); + history_window(URI, DIGEST, 0, 20.0, 8.0, 1, 1060); + history_window(URI, DIGEST, 1, 99.0, 99.0, 2, 1120); + history_crash(URI, DIGEST, 0, 1180); + history_crash(URI, DIGEST, 1, 1240); + history_restart(URI, DIGEST, 1, 1300); + history_restart(URI, DIGEST, 0, 1310); /* alone: not a restart of a group, not counted */ + history_quarantine(URI, DIGEST, 1360); +} + +static void test_the_keys_are_the_contracts_words(void) +{ + history_rec_t r = { .uri = URI, .klass = CLASS }; + const char *const words[] = OMX_PLUGIN_HOSTD_HISTORY_KEYS_INIT; + char *line, *p; + unsigned n; + + memcpy(r.digest, DIGEST, sizeof(r.digest)); + line = history_line(&r); + p = strchr(strchr(strchr(strchr(line, ' ') + 1, ' ') + 1, ' ') + 1, ' '); + for (n = 0; n < OMX_PLUGIN_HOSTD_HISTORY_KEYS_COUNT; n++) + { + char want[64]; + + snprintf(want, sizeof(want), " %s=", words[n]); + check(p && !strncmp(p, want, strlen(want)), words[n]); + p = p ? strchr(p + 1, ' ') : NULL; + } + check(p == NULL || *p == '\n', "no key of the line is outside the words"); + free(line); + group("the_line_holds_the_contracts_keys_in_the_contracts_order"); +} + +static void test_round_trip(void) +{ + char *text, *again; + const history_rec_t *r; + char *summary; + + fresh(); + start(); + count_everything(); + history_finish(); + text = file_text(); + check_str(text, HEADER "history " URI " " DIGEST " " CLASS + " first=1000 last=1360 windows_solo=2 windows_grouped=1 cpu_p95_solo=10.19 cpu_max_solo=20.00 pss_mb_solo=12.3" + " xrun_windows_solo=1 xrun_windows_grouped=1 crashes_solo=1 crashes_grouped=1 restarts_grouped=1" + " quarantines=1 last_crash=1180 last_crash_grouped=1240\n", "the file a record is written as"); + start(); + r = history_find(URI, DIGEST); + check(r != NULL, "the record is found after a restart"); + if (r) + { + check(r->first == 1000 && r->last == 1360 && r->windows_solo == 2 && r->windows_grouped == 1, "windows and times"); + check(r->cpu_p95_solo > 10.189 && r->cpu_p95_solo < 10.191 && r->cpu_max_solo == 20.0, "cpu"); + check(r->pss_mb_solo > 12.29 && r->pss_mb_solo < 12.31, "pss"); + check(r->xrun_windows_solo == 1 && r->xrun_windows_grouped == 1, "xrun windows"); + check(r->crashes_solo == 1 && r->crashes_grouped == 1 && r->restarts_grouped == 1 && r->quarantines == 1, + "crashes, restarts and quarantines"); + check(r->last_crash == 1180 && r->last_crash_grouped == 1240, "last crashes"); + } + summary = history_summary(URI, DIGEST, 1400); + check_str(summary, DIGEST " " CLASS " crashed=1 raises_xruns=0 first=1000 last=1360 windows_solo=2 windows_grouped=1 " + "cpu_p95_solo=10.19 cpu_max_solo=20.00 pss_mb_solo=12.3 xrun_windows_solo=1 xrun_windows_grouped=1 " + "crashes_solo=1 crashes_grouped=1 restarts_grouped=1 quarantines=1 last_crash=1180 last_crash_grouped=1240", + "the summary history_get answers"); + free(summary); + history_finish(); + again = file_text(); + check_str(again, text, "a load and a save of an untouched history is byte for byte the same file"); + free(text); + free(again); + group("a_record_round_trips_through_its_file_value_for_value"); +} + +static void test_windows(void) +{ + const history_rec_t *r; + + fresh(); + start(); + history_window(URI, DIGEST, 0, 10.0, 5.0, 0, 100); + history_window(URI, DIGEST, 0, 10.0, 7.5, 0, 160); + r = history_find(URI, DIGEST); + check(r->cpu_p95_solo > 9.9899 && r->cpu_p95_solo < 9.9901, "a window that is not above the estimate moves it down 5 parts of 0.2: 9.99"); + history_window(URI, DIGEST, 0, 20.0, 6.0, 0, 220); /* above: 9.99 + 0.95 * 0.02 * 9.99 = 10.17981 */ + check(r->cpu_p95_solo > 10.1797 && r->cpu_p95_solo < 10.1799, "above the estimate moves it up 95 parts of a step of 2 percent of it"); + history_window(URI, DIGEST, 0, 5.0, 6.0, 0, 280); /* below: 10.17981 - 0.05 * 0.02 * 10.17981 = 10.16963 */ + check(r->cpu_p95_solo > 10.1695 && r->cpu_p95_solo < 10.1697, "below it moves it down 5 parts"); + check(r->cpu_max_solo == 20.0, "cpu max is the largest solo window"); + check(r->pss_mb_solo == 7.5, "pss is the largest solo window"); + check(r->windows_solo == 4 && r->windows_grouped == 0, "four solo windows"); + check(r->first == 100 && r->last == 280, "first and last window"); + history_window(URI, DIGEST, 1, 90.0, 900.0, 0, 340); + check(r->windows_grouped == 1 && r->windows_solo == 4, "a group's window is counted apart"); + check(r->cpu_max_solo == 20.0 && r->pss_mb_solo == 7.5 && r->cpu_p95_solo < 10.2, + "a group's worker time is never taken for the plugin's own: cpu and pss stay those of the solo windows"); + check(r->xrun_windows_solo == 0 && r->xrun_windows_grouped == 0, "no xrun reported, none counted"); + history_window(URI, DIGEST, 0, 1.0, 1.0, 3, 400); + history_window(URI, DIGEST, 1, 1.0, 1.0, 1, 460); + history_window(URI, DIGEST, 1, 1.0, 1.0, 5, 520); + check(r->xrun_windows_solo == 1 && r->xrun_windows_grouped == 2, "a window counts once however many xruns it saw"); + history_finish(); + group("windows_fold_alone_and_in_a_group_into_their_own_fields"); +} + +static void test_no_record_for_what_is_not_a_plugin(void) +{ + fresh(); + start(); + history_window("", DIGEST, 0, 1, 1, 0, 1); + history_window("clap:/with space#x", DIGEST, 0, 1, 1, 0, 1); + history_window(URI, "nothex", 0, 1, 1, 0, 1); + history_window(URI, "BA7816BF8F01CFEA414140DE5DAE2223B00361A396177A9CB410FF61F20015AD", 0, 1, 1, 0, 1); + history_crash(NULL, DIGEST, 0, 1); + history_finish(); + check_str(file_text(), "", "nothing is written for a uri that is empty or holds a space, or a digest that is not 64 lowercase hex"); + group("a_key_that_is_not_a_plugin_binary_makes_no_record"); +} + +static void test_summary(void) +{ + char *s; + + /* crashed: within 30 days of the last crash, the later of the two */ + fresh(); + start(); + history_crash(URI, DIGEST, 0, 1000); + s = history_summary(URI, DIGEST, 1000 + 30 * DAY); + check(s && strstr(s, "crashed=1"), "a crash exactly 30 days ago still counts"); + free(s); + s = history_summary(URI, DIGEST, 1000 + 30 * DAY + 1); + check(s && strstr(s, "crashed=0"), "a crash 30 days and a second ago does not"); + free(s); + history_crash(URI, DIGEST, 1, 1000 + 20 * DAY); + s = history_summary(URI, DIGEST, 1000 + 30 * DAY + 1); + check(s && strstr(s, "crashed=1"), "a recent grouped crash counts though the solo one is old"); + free(s); + history_finish(); + + /* raises_xruns, with no solo windows: more than 5 percent of the grouped windows, from 30 windows */ + fresh(); + start(); + for (int w = 0; w < 29; w++) + history_window(URI, DIGEST, 1, 1, 1, w < 29 ? 1 : 0, 100 + w); + s = history_summary(URI, DIGEST, 200); + check(s && strstr(s, "raises_xruns=0"), "29 grouped windows, all with xruns, are too few to say"); + free(s); + history_window(URI, DIGEST, 1, 1, 1, 1, 200); + s = history_summary(URI, DIGEST, 200); + check(s && strstr(s, "raises_xruns=1"), "30 grouped windows, all with xruns, are enough"); + free(s); + history_finish(); + + fresh(); + start(); + for (int w = 0; w < 30; w++) + history_window(URI, DIGEST, 1, 1, 1, w < 1 ? 1 : 0, 100 + w); + s = history_summary(URI, DIGEST, 200); + check(s && strstr(s, "raises_xruns=0"), "1 of 30 grouped windows is 3.3 percent: under the 5 percent"); + free(s); + history_finish(); + fresh(); + start(); + for (int w = 0; w < 30; w++) + history_window(URI, DIGEST, 1, 1, 1, w < 2 ? 1 : 0, 100 + w); + s = history_summary(URI, DIGEST, 200); + check(s && strstr(s, "raises_xruns=1"), "2 of 30 grouped windows is 6.7 percent: over it"); + free(s); + history_finish(); + fresh(); + start(); + for (int w = 0; w < 40; w++) + history_window(URI, DIGEST, 1, 1, 1, w < 2 ? 1 : 0, 100 + w); + s = history_summary(URI, DIGEST, 200); + check(s && strstr(s, "raises_xruns=0"), "2 of 40 grouped windows is exactly 5 percent: not over it"); + free(s); + history_finish(); + + /* with solo windows: the grouped share must exceed the solo share by 5 points, strictly */ + fresh(); + start(); + for (int w = 0; w < 100; w++) + history_window(URI, DIGEST, 0, 1, 1, w < 10 ? 1 : 0, 100 + w); /* solo: 10 of 100 = 10 % */ + for (int w = 0; w < 40; w++) + history_window(URI, DIGEST, 1, 1, 1, w < 6 ? 1 : 0, 300 + w); /* grouped: 6 of 40 = 15 %: exactly +5 */ + s = history_summary(URI, DIGEST, 400); + check(s && strstr(s, "raises_xruns=0"), "exactly 5 points over the solo share does not exceed it"); + free(s); + history_window(URI, DIGEST, 1, 1, 1, 1, 500); /* 7 of 41 = 17.07 % */ + s = history_summary(URI, DIGEST, 600); + check(s && strstr(s, "raises_xruns=1"), "more than 5 points over the solo share is a plugin that raises xruns"); + free(s); + s = history_summary(URI, OTHER, 600); + check(s == NULL, "no record of another binary"); + history_finish(); + group("the_summary_marks_crashes_and_raised_xruns_at_the_declared_margins"); +} + +static void test_another_host_class_is_kept_and_not_mixed(void) +{ + char *text; + + fresh(); + put(file, HEADER "history " URI " " DIGEST " aarch64/5-8/<1.5GHz first=5 last=6 windows_solo=7 windows_grouped=0 " + "cpu_p95_solo=1.00 cpu_max_solo=2.00 pss_mb_solo=3.0 xrun_windows_solo=0 xrun_windows_grouped=0 " + "crashes_solo=9 crashes_grouped=0 restarts_grouped=0 quarantines=0 last_crash=6 last_crash_grouped=0\n"); + start(); + check(history_find(URI, DIGEST) == NULL, "the record of another host class is not this class's"); + history_window(URI, DIGEST, 0, 4.0, 4.0, 0, 50); + history_finish(); + text = file_text(); + check(strstr(text, "aarch64/5-8/<1.5GHz first=5 last=6 windows_solo=7") != NULL, "the other class's line is kept as it was"); + check(strstr(text, CLASS " first=50 last=50 windows_solo=1") != NULL, "this class has a line of its own"); + free(text); + group("a_record_of_another_host_class_is_kept_and_never_read_as_this_ones"); +} + +static void test_lines_this_version_cannot_read(void) +{ + const char *bad[] = { + "history clap:/bad/1 " DIGEST " " CLASS " first=1 last=2 future_key=5", /* a key it does not know */ + "history clap:/bad/2 " DIGEST " " CLASS " first=-1", /* negative */ + "history clap:/bad/3 " DIGEST " " CLASS " cpu_max_solo=nan", /* not a number */ + "history clap:/bad/4 " DIGEST " " CLASS " cpu_max_solo=inf", + "history clap:/bad/5 " DIGEST " " CLASS " windows_solo=1.5", /* a count that is not whole */ + "history clap:/bad/6 " DIGEST " " CLASS " windows_solo=3 windows_solo=4", /* one key twice */ + "history clap:/bad/7 nothex " CLASS " first=1", /* a digest that is not one */ + "history clap:/bad/8 " DIGEST, /* too few words */ + "plugin clap:/bad/9 " DIGEST " " CLASS " first=1", /* not a history line */ + "history clap:/bad/10 " DIGEST " " CLASS " first", /* no value */ + "garbage", + }; + char all[8192] = HEADER, *text; + unsigned n; + const history_rec_t *r; + + fresh(); + strcat(all, "history " URI " " DIGEST " " CLASS " first=10 last=20 windows_solo=3\n"); + for (n = 0; n < sizeof(bad) / sizeof(bad[0]); n++) + { + strcat(all, bad[n]); + strcat(all, "\n"); + } + put(file, all); + start(); + check(history_available(), "bad lines do not block the file"); + r = history_find(URI, DIGEST); + check(r && r->windows_solo == 3 && r->first == 10, "the good line is read"); + history_window(URI, DIGEST, 0, 1.0, 1.0, 0, 30); + history_finish(); + text = file_text(); + for (n = 0; n < sizeof(bad) / sizeof(bad[0]); n++) + { + char want[400]; + + snprintf(want, sizeof(want), "\n%s\n", bad[n]); + check(strstr(text, want) != NULL, bad[n]); + } + check(strstr(text, "windows_solo=4") != NULL, "the good line is updated beside them"); + free(text); + group("a_line_outside_the_grammar_is_kept_verbatim_and_not_read"); +} + +static void test_a_second_line_of_one_binary(void) +{ + char *text; + const history_rec_t *r; + + fresh(); + put(file, HEADER "history " URI " " DIGEST " " CLASS " windows_solo=1\n" + "history " URI " " DIGEST " " CLASS " windows_solo=9\n"); + start(); + r = history_find(URI, DIGEST); + check(r && r->windows_solo == 1, "the first line holds"); + history_window(URI, DIGEST, 0, 1.0, 1.0, 0, 3); + history_finish(); + text = file_text(); + check(strstr(text, "windows_solo=9\n") != NULL, "the second is kept as it was"); + free(text); + group("a_second_line_of_one_binary_and_class_is_kept_and_not_read"); +} + +static void test_comments_and_blanks(void) +{ + char *text; + + fresh(); + put(file, HEADER "# kept\n\nhistory " URI " " DIGEST " " CLASS " windows_solo=1\n"); + start(); + history_window(URI, DIGEST, 0, 1.0, 1.0, 0, 3); + history_finish(); + text = file_text(); + check(strstr(text, HEADER "# kept\nhistory " URI) != NULL, "a comment stays where it was, a blank line goes"); + free(text); + group("a_comment_line_is_kept_in_its_place"); +} + +static void test_blocked_files(void) +{ + struct { const char *name, *content; } cases[] = { + { "an unknown version", "# plugin-hostd history/2\nhistory " URI " " DIGEST " " CLASS " windows_solo=1\n" }, + { "not a history file", "something else entirely\nhistory " URI " " DIGEST " " CLASS " windows_solo=1\n" }, + { "a header with a suffix", "# plugin-hostd history/1 extra\nhistory " URI " " DIGEST " " CLASS " windows_solo=1\n" }, + }; + unsigned n; + + for (n = 0; n < sizeof(cases) / sizeof(cases[0]); n++) + { + char *text; + + fresh(); + put(file, cases[n].content); + start(); + check(!history_available(), cases[n].name); + check(history_find(URI, DIGEST) == NULL, "a blocked history reads nothing"); + history_window(URI, DIGEST, 0, 1.0, 1.0, 0, 3); + history_crash(URI, DIGEST, 0, 4); + check(history_clear("all") == -1, "a blocked history is not cleared"); + history_tick(100000); + history_finish(); + text = file_text(); + check_str(text, cases[n].content, "a blocked history is never overwritten"); + free(text); + } + /* a file that cannot be read: a directory where it should be */ + fresh(); + mkdir(file, 0700); + start(); + check(!history_available(), "a history that is a directory can't be read"); + history_window(URI, DIGEST, 0, 1.0, 1.0, 0, 3); + history_finish(); + check(access(file, F_OK) == 0, "and is left where it was"); + group("a_file_with_an_unknown_version_or_that_cannot_be_read_blocks_the_history_and_is_never_touched"); +} + +static void test_missing_and_empty_files(void) +{ + fresh(); + start(); + check(history_available(), "a missing file is an empty history"); + history_finish(); + check_str(file_text(), "", "and nothing is written for it"); + put(file, ""); + start(); + check(history_available(), "an empty file is an empty history"); + history_window(URI, DIGEST, 0, 1.0, 1.0, 0, 3); + history_finish(); + check(strncmp(file_text(), HEADER, strlen(HEADER)) == 0, "which gets its header"); + group("a_missing_or_empty_file_is_an_empty_history"); +} + +static void test_the_file(void) +{ + struct stat st; + char sub[400], listing[600], *l; + FILE *p; + + fresh(); + snprintf(sub, sizeof(sub), "%s/a/b/history", dir); + history_init(sub, CLASS, 300); + history_window(URI, DIGEST, 0, 1.0, 1.0, 0, 3); + history_finish(); + check(stat(sub, &st) == 0 && (st.st_mode & 0777) == 0600, "the file is mode 0600"); + snprintf(sub, sizeof(sub), "%s/a/b", dir); + check(stat(sub, &st) == 0 && (st.st_mode & 0777) == 0700, "its missing directories are made mode 0700"); + snprintf(listing, sizeof(listing), "ls -A '%s/a/b'", dir); + p = popen(listing, "r"); + l = calloc(1, 256); + if (p) + { + size_t n = fread(l, 1, 255, p); + + l[n] = '\0'; + pclose(p); + } + check_str(l, "history\n", "no temporary file is left beside it"); + free(l); + group("the_history_file_is_mode_0600_in_a_private_directory_with_no_temporary_beside_it"); +} + +static void test_a_failed_save(void) +{ + char *before, *after; + + if (geteuid() == 0) + { + printf("skip history: a failed save needs a directory root can't write to\n"); + printf("ok a_save_that_fails_leaves_the_old_file_whole_and_is_tried_again\n"); + return; + } + fresh(); + start(); + history_window(URI, DIGEST, 0, 1.0, 1.0, 0, 3); + history_tick(0); + before = file_text(); + check(strlen(before) > strlen(HEADER), "the first save wrote the record"); + history_window(URI, DIGEST, 0, 2.0, 2.0, 0, 70); + chmod(dir, 0500); + history_tick(1000); + chmod(dir, 0700); + after = file_text(); + check_str(after, before, "a save that can't write leaves the old file as it was"); + free(after); + history_tick(1299); + after = file_text(); + check_str(after, before, "a failed save is tried again after the interval, not at every tick"); + free(after); + history_tick(1300); + after = file_text(); + check(strstr(after, "windows_solo=2") != NULL, "the change was not forgotten: the next tick writes it"); + free(before); + free(after); + history_finish(); + group("a_save_that_fails_leaves_the_old_file_whole_and_is_tried_again"); +} + +static void test_save_interval(void) +{ + char *t; + + fresh(); + history_init(file, CLASS, 300); + history_tick(5); + check_str(file_text(), "", "nothing changed, nothing written"); + history_window(URI, DIGEST, 0, 1.0, 1.0, 0, 3); + history_tick(10); + t = file_text(); + check(strstr(t, "windows_solo=1") != NULL, "the first change is written at the first tick"); + free(t); + history_window(URI, DIGEST, 0, 1.0, 1.0, 0, 70); + history_tick(309); + t = file_text(); + check(strstr(t, "windows_solo=1") != NULL, "299 s after the save it is not written"); + free(t); + history_tick(310); + t = file_text(); + check(strstr(t, "windows_solo=2") != NULL, "300 s after it is"); + free(t); + history_window(URI, DIGEST, 0, 1.0, 1.0, 0, 130); + history_finish(); + t = file_text(); + check(strstr(t, "windows_solo=3") != NULL, "and quitting writes what the interval had not"); + free(t); + group("the_history_is_saved_at_most_once_per_interval_and_at_quit"); +} + +static void test_clear(void) +{ + char *text; + + fresh(); + put(file, HEADER + "history " URI " " DIGEST " " CLASS " windows_solo=1\n" + "history " URI " " OTHER " " CLASS " windows_solo=2\n" + "history " URI " " DIGEST " aarch64/1-2/<1.5GHz windows_solo=3\n" + "history " URI " " DIGEST " " CLASS " windows_solo=4 future=1\n" + "history clap:/p/b.clap#org.test.b " DIGEST " " CLASS " windows_solo=5\n"); + start(); + check(history_clear(URI) == 0, "clear of a uri answers"); + text = file_text(); + check_str(text, HEADER "history clap:/p/b.clap#org.test.b " DIGEST " " CLASS " first=0 last=0 windows_solo=5 " + "windows_grouped=0 cpu_p95_solo=0.00 cpu_max_solo=0.00 pss_mb_solo=0.0 xrun_windows_solo=0 " + "xrun_windows_grouped=0 crashes_solo=0 crashes_grouped=0 restarts_grouped=0 quarantines=0 last_crash=0 " + "last_crash_grouped=0\n", + "every binary, class and unreadable line of the uri is gone at once, the other plugin stays"); + free(text); + check(history_find(URI, DIGEST) == NULL, "and the memory too"); + history_window(URI, DIGEST, 0, 1.0, 1.0, 0, 3); + check(history_clear("all") == 0, "clear all"); + text = file_text(); + check_str(text, HEADER, "empties the file but for its header"); + free(text); + history_finish(); + group("clear_removes_a_plugins_lines_of_every_binary_and_class_or_all_of_them_at_once"); +} + +static void test_the_path(void) +{ + char want[400], *text; + + fresh(); + snprintf(want, sizeof(want), "%s/state", dir); + setenv("XDG_STATE_HOME", want, 1); + history_init("", CLASS, 300); + history_window(URI, DIGEST, 0, 1.0, 1.0, 0, 3); + history_finish(); + snprintf(want, sizeof(want), "%s/state/plugin-hostd/history", dir); + check(access(want, F_OK) == 0, "$XDG_STATE_HOME/plugin-hostd/history"); + unsetenv("XDG_STATE_HOME"); + snprintf(want, sizeof(want), "%s/home", dir); + setenv("HOME", want, 1); + history_init("", CLASS, 300); + history_window(URI, DIGEST, 0, 1.0, 1.0, 0, 3); + history_finish(); + snprintf(want, sizeof(want), "%s/home/.local/state/plugin-hostd/history", dir); + check(access(want, F_OK) == 0, "$HOME/.local/state/plugin-hostd/history"); + snprintf(want, sizeof(want), "%s/elsewhere", dir); + history_init(want, CLASS, 300); + history_window(URI, DIGEST, 0, 1.0, 1.0, 0, 3); + history_finish(); + text = slurp(want); + check(text != NULL, "history_file names another"); + free(text); + group("the_default_path_is_the_xdg_state_home_then_home_and_history_file_overrides_both"); +} + +int main(void) +{ + snprintf(dir, sizeof(dir), "/tmp/plugin-hostd-history-test.XXXXXX"); + if (!mkdtemp(dir)) + return 2; + snprintf(file, sizeof(file), "%s/history", dir); + test_the_keys_are_the_contracts_words(); + test_round_trip(); + test_windows(); + test_no_record_for_what_is_not_a_plugin(); + test_summary(); + test_another_host_class_is_kept_and_not_mixed(); + test_lines_this_version_cannot_read(); + test_a_second_line_of_one_binary(); + test_comments_and_blanks(); + test_blocked_files(); + test_missing_and_empty_files(); + test_the_file(); + test_a_failed_save(); + test_save_interval(); + test_clear(); + test_the_path(); + group(NULL); + fresh(); + rmdir(dir); + printf(failures ? "history: %d failed\n" : "history: ok\n", failures); + return failures ? 1 : 0; +} diff --git a/tests/sabotage.py b/tests/sabotage.py index 71ff56b..3f4a98f 100644 --- a/tests/sabotage.py +++ b/tests/sabotage.py @@ -13,7 +13,7 @@ # an LV2 bundle (MOD_HOST, LV2_DIR, LV2_URI, LV2_BUNDLE, LV2_PARAM) and is skipped without them; "pin" breaks # include/plugin-hostd/pin.h and runs tests/pin_test.c; "readme" breaks README.md or BUILDING.md and runs # tests/readme_build.py, with no daemon to build; "probeschema" breaks protocol/probe-output.schema.json and runs -# tests/probe_schema.py; "relay" breaks src/relay.c and runs tests/relay_test.c; "alloc" breaks src/relay.c or src/lv2.c and runs make test-alloc; "contract" breaks the probe runner or the generator and runs tests/contract_derived.py; "consumer" breaks the header and compiles tests/consumer.c against it; "probe" breaks src/probe.c and runs tests/probe_test.py; "nopins" breaks the packaging or a workflow and runs tests/no_pins.py; "ciguards" breaks a workflow and runs tests/ci_guards.py; +# tests/probe_schema.py; "relay" breaks src/relay.c and runs tests/relay_test.c; "history" breaks src/history.c or src/fsutil.c and runs tests/history_test.c; "alloc" breaks src/relay.c or src/lv2.c and runs make test-alloc; "contract" breaks the probe runner or the generator and runs tests/contract_derived.py; "consumer" breaks the header and compiles tests/consumer.c against it; "probe" breaks src/probe.c and runs tests/probe_test.py; "nopins" breaks the packaging or a workflow and runs tests/no_pins.py; "ciguards" breaks a workflow and runs tests/ci_guards.py; # "vocabulary" breaks the probe schema, its template or a copy of omx-contract's omx-vocabulary.schema.json and runs # tests/vocabulary_derived.py; "verbtable" breaks the header and runs tests/verb_table.py; "presetschema" breaks # protocol/preset.schema.json and runs tests/preset_format.py @@ -752,6 +752,114 @@ " if (!strcmp(verb, OMX_VERB_WORKER_LIST))\n", "verbs_declared", "verbs"), ("a declared verb is typed by hand in the daemon", "src/verbs.c", "if (!strcmp(verb, OMX_VERB_CHAIN_GET))", 'if (!strcmp(verb, "chain_get"))', "verbs_contract", "verbs"), + ('the history file is left readable by others', 'src/history.c', + ' ok = fflush(file) == 0 && !ferror(file) && fsync(fileno(file)) == 0;', + ' fchmod(fileno(file), 0644);\n ok = fflush(file) == 0 && !ferror(file) && fsync(fileno(file)) == 0;', + 'the_history_file_is_mode_0600_in_a_private_directory_with_no_temporary_beside_it', "history"), + ("the history's directories are made world readable", 'src/fsutil.c', + 'mkdir(path, 0700)', + 'mkdir(path, 0755)', + 'the_history_file_is_mode_0600_in_a_private_directory_with_no_temporary_beside_it', "history"), + ('a line this version cannot read is dropped by the next save', 'src/history.c', + ' held_add(line, n);\n }\n else if (rec_at(', + ' }\n else if (rec_at(', + 'a_line_outside_the_grammar_is_kept_verbatim_and_not_read', "history"), + ('a second line of one binary is dropped by the next save', 'src/history.c', + ' rec_free(&rec);\n held_add(line, n);', + ' rec_free(&rec);', + 'a_second_line_of_one_binary_and_class_is_kept_and_not_read', "history"), + ('a negative count is read', 'src/history.c', + "if (!*text || *text == '-' || ", + "if (!*text || ", + 'a_line_outside_the_grammar_is_kept_verbatim_and_not_read', "history"), + ('a decimal that is not a number is read', 'src/history.c', + 'if (*end || errno || !isfinite(v))', + 'if (*end || errno)', + 'a_line_outside_the_grammar_is_kept_verbatim_and_not_read', "history"), + ('a key twice in one line is read', 'src/history.c', + '(seen & (1u << key)) || ', + '(seen & 0) || ', + 'a_line_outside_the_grammar_is_kept_verbatim_and_not_read', "history"), + ('a digest that is not a digest is read', 'src/history.c', + '!phd_pin_hex_valid(tok[2]))', + '0)', + 'a_line_outside_the_grammar_is_kept_verbatim_and_not_read', "history"), + ('a file of another version is read', 'src/history.c', + 'if (n != strlen(PHD_HISTORY_HEADER) || strncmp(line, PHD_HISTORY_HEADER, n))', + 'if (0)', + 'a_file_with_an_unknown_version_or_that_cannot_be_read_blocks_the_history_and_is_never_touched', "history"), + ('a blocked history is cleared', 'src/history.c', + ' if (!g_started || g_blocked)\n return -1;', + ' if (!g_started)\n return -1;', + 'a_file_with_an_unknown_version_or_that_cannot_be_read_blocks_the_history_and_is_never_touched', "history"), + ("the file's header is not written", 'src/history.c', + ' fprintf(file, "%s\\n", PHD_HISTORY_HEADER);\n', + '', + 'a_record_round_trips_through_its_file_value_for_value', "history"), + ('the pss is written to two decimals', 'src/history.c', + '" %s=%.1f"', + '" %s=%.2f"', + 'a_record_round_trips_through_its_file_value_for_value', "history"), + ('a restart of a plugin alone is counted', 'src/history.c', + 'history_rec_t *r = grouped ? rec_get(uri, digest) : NULL;', + 'history_rec_t *r = ((void)grouped, rec_get(uri, digest));', + 'a_record_round_trips_through_its_file_value_for_value', "history"), + ('the cpu estimate moves up and down by one half', 'src/history.c', + '0.95 * step : -0.05 * step', + '0.5 * step : -0.5 * step', + 'windows_fold_alone_and_in_a_group_into_their_own_fields', "history"), + ("a group's window is folded into the plugin's own cpu", 'src/history.c', + ' if (grouped)\n {\n r->windows_grouped++;', + ' if (!grouped)\n {\n r->windows_grouped++;', + 'windows_fold_alone_and_in_a_group_into_their_own_fields', "history"), + ('a space in a uri makes a record', 'src/history.c', + ' || strpbrk(uri, " \\t\\r\\n")', + '', + 'a_key_that_is_not_a_plugin_binary_makes_no_record', "history"), + ('a digest that is not one makes a record', 'src/history.c', + '|| !phd_pin_hex_valid(digest) ', + '', + 'a_key_that_is_not_a_plugin_binary_makes_no_record', "history"), + ('a crash 30 days old no longer counts', 'src/history.c', + 'now - last <= (long long)PHD_HISTORY_KEEP_DAYS', + 'now - last < (long long)PHD_HISTORY_KEEP_DAYS', + 'the_summary_marks_crashes_and_raised_xruns_at_the_declared_margins', "history"), + ('the xrun margin is not strict', 'src/history.c', + 'return 100 * xg * ws > 100 * xs * wg', + 'return 100 * xg * ws >= 100 * xs * wg', + 'the_summary_marks_crashes_and_raised_xruns_at_the_declared_margins', "history"), + ('thirty windows are too few', 'src/history.c', + 'if (wg < PHD_HISTORY_MIN_WINDOWS)', + 'if (wg <= PHD_HISTORY_MIN_WINDOWS)', + 'the_summary_marks_crashes_and_raised_xruns_at_the_declared_margins', "history"), + ('the xrun floor of a plugin that never ran alone is not strict', 'src/history.c', + 'return 100 * xg > (long long)PHD_HISTORY_XRUN_MARGIN_PCT * wg;', + 'return 100 * xg >= (long long)PHD_HISTORY_XRUN_MARGIN_PCT * wg;', + 'the_summary_marks_crashes_and_raised_xruns_at_the_declared_margins', "history"), + ("another host class's record is read as this one's", 'src/history.c', + ' !strcmp(g_entry[n].rec.klass, klass))', + ' klass != NULL)', + 'a_record_of_another_host_class_is_kept_and_never_read_as_this_ones', "history"), + ('a clear leaves the lines it could not read', 'src/history.c', + 'g_entry[n].raw ? raw_names(g_entry[n].raw, uri) :', + 'g_entry[n].raw ? raw_names(g_entry[n].raw, "") :', + 'clear_removes_a_plugins_lines_of_every_binary_and_class_or_all_of_them_at_once', "history"), + ('the history is saved at every tick', 'src/history.c', + '(!g_saved_once || now - g_saved_at >= g_save_s)', + '1', + 'the_history_is_saved_at_most_once_per_interval_and_at_quit', "history"), + ('a failed save forgets the change', 'src/history.c', + ' g_saved_once = 1;\n if (file_write() == 0)\n g_dirty = 0;', + ' g_saved_once = 1;\n file_write();\n g_dirty = 0;', + 'a_save_that_fails_leaves_the_old_file_whole_and_is_tried_again', "history"), + ('the state home is not looked at', 'src/history.c', + ' else if (state && *state)', + ' else if (0 && state && *state)', + 'the_default_path_is_the_xdg_state_home_then_home_and_history_file_overrides_both', "history"), + ('history_file does not override', 'src/history.c', + ' if (*configured)\n snprintf(g_path, sizeof(g_path), "%s", configured);\n else if', + ' if (0)\n snprintf(g_path, sizeof(g_path), "%s", configured);\n else if', + 'the_default_path_is_the_xdg_state_home_then_home_and_history_file_overrides_both', "history"), ] @@ -811,6 +919,16 @@ def run_consumer_test(tree): p = subprocess.run([exe], capture_output=True, text=True, timeout=60) return p.returncode, p.stdout + ("ok consumer\n" if p.returncode == 0 else "") +def run_history_test(tree): + """tests/history_test.c against the tree's src/history.c; it prints "ok " for each group that passed""" + os.makedirs(os.path.join(tree, "tests"), exist_ok=True) + shutil.copy(os.path.join(ROOT, "tests", "history_test.c"), os.path.join(tree, "tests")) + subprocess.run(["make", "-s", "-B", "MOD_HOST_DIR=" + MOD_HOST_DIR, "tests/history-test"], cwd=tree, check=True, + stdout=subprocess.DEVNULL) + p = subprocess.run([os.path.join(tree, "tests", "history-test")], capture_output=True, text=True, timeout=120) + return p.returncode, p.stdout + + def run_alloc_test(tree): """make test-alloc: the tables of src/relay.c and src/lv2.c when realloc fails; prints "ok alloc_test" when no check failed""" for f in ("tests/gesture-alloc-test", "tests/lv2-alloc-test", "tests/lv2-alloc-lv2.o"): @@ -846,6 +964,8 @@ def run_test(tree, name, jack=False): return run_format_test(tree) if jack == "relay": return run_relay_test(tree) + if jack == "history": + return run_history_test(tree) if jack == "contract": return run_contract_test(tree) if jack == "pin": @@ -898,6 +1018,9 @@ def run_test(tree, name, jack=False): if os.environ.get("SABOTAGE_ONLY") and os.environ["SABOTAGE_ONLY"] not in name: continue jack = entry[5] if len(entry) > 5 else False + # SABOTAGE_KIND=: only the arms of that kind ("history", "pin", ...) + if os.environ.get("SABOTAGE_KIND") and os.environ["SABOTAGE_KIND"] != jack: + continue if os.geteuid() == 0 and name in ROOT_WRITES_READ_ONLY: print("skip sabotage (root writes a 0500 directory, so the test skips itself): %s" % what) continue @@ -919,6 +1042,10 @@ def run_test(tree, name, jack=False): os.makedirs(tree) for f in ("README.md", "BUILDING.md"): shutil.copy(os.path.join(ROOT, f), tree) + elif jack == "history": + for d in ("src", "include"): + shutil.copytree(os.path.join(ROOT, d), os.path.join(tree, d), ignore=shutil.ignore_patterns("*.o")) + shutil.copy(os.path.join(ROOT, "Makefile"), tree) elif jack == "alloc": shutil.copytree(os.path.join(ROOT, "src"), os.path.join(tree, "src")) shutil.copytree(os.path.join(ROOT, "include"), os.path.join(tree, "include")) @@ -990,7 +1117,7 @@ def run_test(tree, name, jack=False): stdout=subprocess.DEVNULL) if jack == "probe": build_probe(tree) - elif jack not in ("readme", "nopins", "ciguards", "format", "relay", "contract", "consumer", "probeschema", "presetschema", "vocabulary", "alloc", + elif jack not in ("readme", "nopins", "ciguards", "format", "relay", "history", "contract", "consumer", "probeschema", "presetschema", "vocabulary", "alloc", "verbtable"): for f in os.listdir(os.path.join(tree, "src")): if f.endswith(".o"): From baf7f3ad56c33b689ed2d1f32d4c6e38935b0a80 Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Tue, 27 Oct 2026 07:01:14 +0100 Subject: [PATCH 05/16] Read a worker's CPU time and Pss from /proc proc_parse_stat takes utime and stime from the fields after the last ')' of a stat line, since the command name before it may hold spaces and parentheses; proc_parse_pss takes the Pss line of smaps_rollup and not Pss_Anon or Pss_File. tests/proc_test.c reads lines written out by hand and this process's own files, and five sabotage arms break each reader (#184). --- .gitignore | 1 + Makefile | 12 +++++-- src/proc.c | 88 +++++++++++++++++++++++++++++++++++++++++++++ src/proc.h | 12 +++++++ tests/proc_test.c | 92 +++++++++++++++++++++++++++++++++++++++++++++++ tests/sabotage.py | 38 ++++++++++++++++++-- 6 files changed, 238 insertions(+), 5 deletions(-) create mode 100644 tests/proc_test.c diff --git a/.gitignore b/.gitignore index 3018b3c..717af94 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ tools/protocol-gen *.new tests/pin-test tests/history-test +tests/proc-test tests/relay-test tests/relay-test-relay.o tests/format-test diff --git a/Makefile b/Makefile index 7038e9a..c1cda5b 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,7 @@ FAKE = tests/fake-host FORMAT_TEST = tests/format-test PIN_TEST = tests/pin-test HISTORY_TEST = tests/history-test +PROC_TEST = tests/proc-test RELAY_TEST = tests/relay-test GESTURE_ALLOC_TEST = tests/gesture-alloc-test LV2_ALLOC_TEST = tests/lv2-alloc-test @@ -255,12 +256,12 @@ fuzz-corpus-check: # clean rule clean: - @rm -f src/*.o $(PROG) $(PROBE) tests/probe-child tests/lv2-probe-child tests/rt_interpose.so $(FAKE) $(PIN_TEST) $(HISTORY_TEST) $(RELAY_TEST) tests/relay-test-relay.o $(FORMAT_TEST) $(GESTURE_ALLOC_TEST) $(LV2_ALLOC_TEST) tests/lv2-alloc-lv2.o $(TRANSFORMS_TEST) $(GEN) tests/stress.clap tests/jack_levels tests/host_scenarios $(LV2_FIXTURE) + @rm -f src/*.o $(PROG) $(PROBE) tests/probe-child tests/lv2-probe-child tests/rt_interpose.so $(FAKE) $(PIN_TEST) $(HISTORY_TEST) $(PROC_TEST) $(RELAY_TEST) tests/relay-test-relay.o $(FORMAT_TEST) $(GESTURE_ALLOC_TEST) $(LV2_ALLOC_TEST) tests/lv2-alloc-lv2.o $(TRANSFORMS_TEST) $(GEN) tests/stress.clap tests/jack_levels tests/host_scenarios $(LV2_FIXTURE) @rm -rf build # the daemon against workers that are not plugin hosts at all (tests/fake-host speaks the same protocol and can be # made to die on cue): placement, forwarding, ledger, replay, attribution, the storm bound; no jack, no plugin -test: test-protocol-lib test-pin test-history test-transforms test-format-coverage test-describe test-relay test-alloc test-format test-daemon test-verbs-declared check-rendered check-generated check-schema test-probe-schema test-vocabulary test-probe test-contract test-consumer test-perturbation check-readme check-own-host-page check-no-pins check-ci-guards test-mod-host-json test-socket check-verb-table check-preset-schema test-install-schemas +test: test-protocol-lib test-pin test-history test-proc test-transforms test-format-coverage test-describe test-relay test-alloc test-format test-daemon test-verbs-declared check-rendered check-generated check-schema test-probe-schema test-vocabulary test-probe test-contract test-consumer test-perturbation check-readme check-own-host-page check-no-pins check-ci-guards test-mod-host-json test-socket check-verb-table check-preset-schema test-install-schemas MOD_HOST_DIR=$(MOD_HOST_DIR) python3 tests/verbs_contract.py # the protocol clients plugin-hostd-tests ships (tests/lib/), against a peer that frames by NUL, and their sabotage @@ -319,6 +320,13 @@ test-history: $(HISTORY_TEST) $(HISTORY_TEST): tests/history_test.c src/history.c src/history.h src/fsutil.c src/fsutil.h include/plugin-hostd/protocol.h $(RENDERED) $(CC) -Iinclude -Isrc $(CFLAGS) -Werror -o $@ tests/history_test.c src/history.c src/fsutil.c -lm +# src/proc.c's readers of /proc: the text the kernel writes, and this process's own files +test-proc: $(PROC_TEST) + ./$(PROC_TEST) + +$(PROC_TEST): tests/proc_test.c src/proc.c src/proc.h src/conf.h include/plugin-hostd/protocol.h $(RENDERED) + $(CC) $(INCS) -Isrc $(CFLAGS) -Werror -o $@ tests/proc_test.c src/proc.c -pthread -lm + # the transforms the format tables name (src/transforms.c, docs/formats.md): every one and its inverse over its vectors, # and a lookup that is not one to one refused test-transforms: $(TRANSFORMS_TEST) diff --git a/src/proc.c b/src/proc.c index e763008..be55a43 100644 --- a/src/proc.c +++ b/src/proc.c @@ -439,3 +439,91 @@ char *proc_find_lv2_bundle(const char *lv2_path, const char *uri) free(copy); return result; } + +int proc_parse_stat(const char *text, long long *utime, long long *stime) +{ + const char *p = strrchr(text, ')'); + char copy[1024], *save = NULL, *t, *end; + int field = 3; + + if (!p || strlen(p + 1) >= sizeof(copy)) + return -1; + snprintf(copy, sizeof(copy), "%s", p + 1); + /* the words after the name: state is field 3, utime field 14, stime field 15 */ + for (t = strtok_r(copy, " \n", &save); t; t = strtok_r(NULL, " \n", &save), field++) + { + if (field != 14 && field != 15) + continue; + long long v = strtoll(t, &end, 10); + + if (end == t || *end || v < 0) + return -1; + if (field == 14) + *utime = v; + else + { + *stime = v; + return 0; + } + } + return -1; +} + +int proc_parse_pss(const char *text, long long *kb) +{ + const char *p = text; + + /* "Pss:" at the start of a line: not Pss_Anon, Pss_File or Pss_Shmem */ + while (p && *p) + { + if (!strncmp(p, "Pss:", 4)) + { + char *end; + long long v = strtoll(p + 4, &end, 10); + + if (end == p + 4 || v < 0 || strncmp(end, " kB", 3)) + return -1; + *kb = v; + return 0; + } + p = strchr(p, '\n'); + if (p) + p++; + } + return -1; +} + +static int read_proc(pid_t pid, const char *name, char *buf, size_t size) +{ + char path[64]; + int fd, n; + + snprintf(path, sizeof(path), "/proc/%d/%s", (int)pid, name); + if ((fd = open(path, O_RDONLY | O_CLOEXEC)) < 0) + return -1; + n = read(fd, buf, size - 1); + close(fd); + if (n <= 0) + return -1; + buf[n] = '\0'; + return 0; +} + +int proc_cpu_seconds(pid_t pid, double *seconds) +{ + char text[1024]; + long long u = 0, s = 0; + long hz = sysconf(_SC_CLK_TCK); + + if (hz <= 0 || read_proc(pid, "stat", text, sizeof(text)) != 0 || proc_parse_stat(text, &u, &s) != 0) + return -1; + *seconds = (double)(u + s) / (double)hz; + return 0; +} + +int proc_pss_kb(pid_t pid, long long *kb) +{ + char text[4096]; + + return read_proc(pid, "smaps_rollup", text, sizeof(text)) == 0 ? proc_parse_pss(text, kb) : -1; +} diff --git a/src/proc.h b/src/proc.h index 1bfa343..6026de2 100644 --- a/src/proc.h +++ b/src/proc.h @@ -75,4 +75,16 @@ int proc_have_binary(const char *bin); int64_t proc_now_ms(void); +/* What the kernel says of a worker: the CPU time it has used and its memory, for the running history. The first two + * read the text of /proc//stat and /proc//smaps_rollup. 0, or -1 when the text is not what the kernel writes. */ + +/* utime and stime of a stat line, in clock ticks: the fields after the last ')', since the command name before it may + * hold spaces and parentheses */ +int proc_parse_stat(const char *text, long long *utime, long long *stime); +/* the Pss line of smaps_rollup, in kB */ +int proc_parse_pss(const char *text, long long *kb); +/* seconds of CPU (utime + stime) the process has used, and its Pss in kB; -1 when the process or the file is gone */ +int proc_cpu_seconds(pid_t pid, double *seconds); +int proc_pss_kb(pid_t pid, long long *kb); + #endif diff --git a/tests/proc_test.c b/tests/proc_test.c new file mode 100644 index 0000000..efc451f --- /dev/null +++ b/tests/proc_test.c @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * This file is part of plugin-hostd. + * + * plugin-hostd is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * plugin-hostd is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with plugin-hostd. If not, see . + * + * Copyright (C) 2026 Pau Aliagas + */ + +/* src/proc.c's readers of /proc against text written out by hand, in the shape the kernel writes it: a stat line whose + * command name holds spaces and parentheses, an smaps_rollup with its Pss_ lines, and then the two files of this very + * process, whose CPU time is spent here and counted by the CPU it used, not by the clock. */ + +#include +#include +#include +#include + +#include "proc.h" + +static int failures; + +static void check(int ok, const char *what) +{ + if (!ok) + { + failures++; + printf("FAIL proc: %s\n", what); + } +} + +int main(void) +{ + long long u = -1, s = -1, kb = -1; + double cpu = 0, later; + volatile unsigned long spin = 0; + unsigned long spun; + + check(proc_parse_stat("1234 (plain) S 1 1234 1234 0 -1 4194560 100 0 0 0 7 3 0 0 20 0 1 0 100 1000000 50\n", &u, &s) == 0 && + u == 7 && s == 3, "utime 7 and stime 3 of a plain stat line"); + u = s = -1; + check(proc_parse_stat("42 (my (odd) name) R 1 42 42 0 -1 4194560 9 0 1 0 123456 654321 0 0 20 0 1 0 100 1000000 50\n", &u, &s) == 0 && + u == 123456 && s == 654321, "a name with spaces and parentheses does not move the fields"); + check(proc_parse_stat("42 (x) R 1 42", &u, &s) == -1, "a line that ends before stime is refused"); + check(proc_parse_stat("42 (x) R 1 42 42 0 -1 4194560 9 0 1 0 abc 5 0\n", &u, &s) == -1, "a utime that is not a number is refused"); + check(proc_parse_stat("no name here", &u, &s) == -1, "a line with no name is refused"); + + check(proc_parse_pss("Rss: 900 kB\nPss_Anon: 99 kB\nPss_File: 7 kB\nPss: 1234 kB\n", &kb) == 0 && kb == 1234, + "Pss, not Pss_Anon or Pss_File"); + kb = -1; + check(proc_parse_pss("Pss_Anon: 99 kB\nPss_File: 7 kB\n", &kb) == -1, "no Pss line: refused"); + check(proc_parse_pss("Pss: kB\n", &kb) == -1, "a Pss that is not a number is refused"); + + check(proc_cpu_seconds(getpid(), &cpu) == 0, "this process's CPU time is read"); + /* spin in batches, reading the CPU time between them; a reader that never sees it grow ends the loop at the bound */ + for (int batch = 0; batch < 3000 && proc_cpu_seconds(getpid(), &later) == 0 && later < cpu + 0.03; batch++) + for (int n = 0; n < 1000000; n++) + spin++; + spun = spin; + check(spun > 0, "the loop ran"); + check(later >= cpu + 0.03, "the CPU time of a process that spins grows by what it spun"); + { + struct rusage ru; + double own; + + getrusage(RUSAGE_SELF, &ru); + own = ru.ru_utime.tv_sec + ru.ru_utime.tv_usec / 1e6 + ru.ru_stime.tv_sec + ru.ru_stime.tv_usec / 1e6; + proc_cpu_seconds(getpid(), &later); + check(later > own - 0.05 && later < own + 0.05, "the CPU time is in seconds, and is what getrusage says of this process"); + } + if (proc_pss_kb(getpid(), &kb) == 0) + check(kb > 100 && kb < 100000000, "this process's Pss is a believable number of kB"); + else + printf("skip proc: no smaps_rollup for this process\n"); + check(proc_cpu_seconds(2147483000, &cpu) == -1, "a process that is not there answers -1"); + if (failures) + printf("proc: %d failed\n", failures); + else + printf("ok proc_test\n"); + return failures ? 1 : 0; +} diff --git a/tests/sabotage.py b/tests/sabotage.py index 3f4a98f..c95c9e7 100644 --- a/tests/sabotage.py +++ b/tests/sabotage.py @@ -13,7 +13,7 @@ # an LV2 bundle (MOD_HOST, LV2_DIR, LV2_URI, LV2_BUNDLE, LV2_PARAM) and is skipped without them; "pin" breaks # include/plugin-hostd/pin.h and runs tests/pin_test.c; "readme" breaks README.md or BUILDING.md and runs # tests/readme_build.py, with no daemon to build; "probeschema" breaks protocol/probe-output.schema.json and runs -# tests/probe_schema.py; "relay" breaks src/relay.c and runs tests/relay_test.c; "history" breaks src/history.c or src/fsutil.c and runs tests/history_test.c; "alloc" breaks src/relay.c or src/lv2.c and runs make test-alloc; "contract" breaks the probe runner or the generator and runs tests/contract_derived.py; "consumer" breaks the header and compiles tests/consumer.c against it; "probe" breaks src/probe.c and runs tests/probe_test.py; "nopins" breaks the packaging or a workflow and runs tests/no_pins.py; "ciguards" breaks a workflow and runs tests/ci_guards.py; +# tests/probe_schema.py; "relay" breaks src/relay.c and runs tests/relay_test.c; "history" breaks src/history.c or src/fsutil.c and runs tests/history_test.c; "proc" breaks src/proc.c and runs tests/proc_test.c; "alloc" breaks src/relay.c or src/lv2.c and runs make test-alloc; "contract" breaks the probe runner or the generator and runs tests/contract_derived.py; "consumer" breaks the header and compiles tests/consumer.c against it; "probe" breaks src/probe.c and runs tests/probe_test.py; "nopins" breaks the packaging or a workflow and runs tests/no_pins.py; "ciguards" breaks a workflow and runs tests/ci_guards.py; # "vocabulary" breaks the probe schema, its template or a copy of omx-contract's omx-vocabulary.schema.json and runs # tests/vocabulary_derived.py; "verbtable" breaks the header and runs tests/verb_table.py; "presetschema" breaks # protocol/preset.schema.json and runs tests/preset_format.py @@ -860,6 +860,26 @@ ' if (*configured)\n snprintf(g_path, sizeof(g_path), "%s", configured);\n else if', ' if (0)\n snprintf(g_path, sizeof(g_path), "%s", configured);\n else if', 'the_default_path_is_the_xdg_state_home_then_home_and_history_file_overrides_both', "history"), + ("the stat line's fields are counted from the first parenthesis", 'src/proc.c', + "const char *p = strrchr(text, ')');", + "const char *p = strchr(text, ')');", + "proc_test", "proc"), + ('utime is read from the field before it', 'src/proc.c', + 'if (field != 14 && field != 15)', + 'if (field != 13 && field != 14)', + "proc_test", "proc"), + ('Pss_Anon is taken for Pss', 'src/proc.c', + 'if (!strncmp(p, "Pss:", 4))', + 'if (!strncmp(p, "Pss", 3))', + "proc_test", "proc"), + ('a Pss that is not a number is taken for 0', 'src/proc.c', + 'if (end == p + 4 || v < 0 || strncmp(end, " kB", 3))', + 'if (v < 0 || strncmp(end, " kB", 3))', + "proc_test", "proc"), + ('CPU time is counted in ticks, not seconds', 'src/proc.c', + '*seconds = (double)(u + s) / (double)hz;', + '*seconds = (double)(u + s);', + "proc_test", "proc"), ] @@ -929,6 +949,16 @@ def run_history_test(tree): return p.returncode, p.stdout +def run_proc_test(tree): + """tests/proc_test.c against the tree's src/proc.c; it prints "ok proc_test" when no check failed""" + os.makedirs(os.path.join(tree, "tests"), exist_ok=True) + shutil.copy(os.path.join(ROOT, "tests", "proc_test.c"), os.path.join(tree, "tests")) + subprocess.run(["make", "-s", "-B", "MOD_HOST_DIR=" + MOD_HOST_DIR, "tests/proc-test"], cwd=tree, check=True, + stdout=subprocess.DEVNULL) + p = subprocess.run([os.path.join(tree, "tests", "proc-test")], capture_output=True, text=True, timeout=120) + return p.returncode, p.stdout + + def run_alloc_test(tree): """make test-alloc: the tables of src/relay.c and src/lv2.c when realloc fails; prints "ok alloc_test" when no check failed""" for f in ("tests/gesture-alloc-test", "tests/lv2-alloc-test", "tests/lv2-alloc-lv2.o"): @@ -966,6 +996,8 @@ def run_test(tree, name, jack=False): return run_relay_test(tree) if jack == "history": return run_history_test(tree) + if jack == "proc": + return run_proc_test(tree) if jack == "contract": return run_contract_test(tree) if jack == "pin": @@ -1042,7 +1074,7 @@ def run_test(tree, name, jack=False): os.makedirs(tree) for f in ("README.md", "BUILDING.md"): shutil.copy(os.path.join(ROOT, f), tree) - elif jack == "history": + elif jack in ("history", "proc"): for d in ("src", "include"): shutil.copytree(os.path.join(ROOT, d), os.path.join(tree, d), ignore=shutil.ignore_patterns("*.o")) shutil.copy(os.path.join(ROOT, "Makefile"), tree) @@ -1117,7 +1149,7 @@ def run_test(tree, name, jack=False): stdout=subprocess.DEVNULL) if jack == "probe": build_probe(tree) - elif jack not in ("readme", "nopins", "ciguards", "format", "relay", "history", "contract", "consumer", "probeschema", "presetschema", "vocabulary", "alloc", + elif jack not in ("readme", "nopins", "ciguards", "format", "relay", "history", "proc", "contract", "consumer", "probeschema", "presetschema", "vocabulary", "alloc", "verbtable"): for f in os.listdir(os.path.join(tree, "src")): if f.endswith(".o"): From 7ed3ce20499a42f405bc90eb0a58e9c616f6b628 Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Tue, 27 Oct 2026 07:14:23 +0100 Subject: [PATCH 06/16] Give the history summary as one word, and a save a full interval from the first tick history_get has to answer a shape the verb declares word for word, so the summary is now digest=...,class=...,crashed=...,raises_xruns=...,key=value joined by commas. The interval between two saves begins at the first tick, so the first save waits history_save_s too and the file is written at most once per interval and at quit. tests/history_test.c also holds the contract's band cuts and the shape of this machine's class, with four more sabotage arms (#184). --- Makefile | 6 +-- src/history.c | 30 +++++++------- src/history.h | 7 ++-- tests/history_test.c | 94 ++++++++++++++++++++++++++++++++++---------- tests/sabotage.py | 32 ++++++++++++--- 5 files changed, 122 insertions(+), 47 deletions(-) diff --git a/Makefile b/Makefile index c1cda5b..2987db0 100644 --- a/Makefile +++ b/Makefile @@ -313,12 +313,12 @@ $(PIN_TEST): tests/pin_test.c include/plugin-hostd/pin.h $(CC) -Iinclude $(CFLAGS) -Werror -o $@ tests/pin_test.c # the running history (src/history.c, docs/design/notes/2026-10-27-grouping-planner.md): its line grammar, its file, what its -# counts and its summary come to, against records written out by hand +# counts and its summary come to, against records written out by hand; and the host class it is kept under (src/machine.c) test-history: $(HISTORY_TEST) ./$(HISTORY_TEST) -$(HISTORY_TEST): tests/history_test.c src/history.c src/history.h src/fsutil.c src/fsutil.h include/plugin-hostd/protocol.h $(RENDERED) - $(CC) -Iinclude -Isrc $(CFLAGS) -Werror -o $@ tests/history_test.c src/history.c src/fsutil.c -lm +$(HISTORY_TEST): tests/history_test.c src/history.c src/history.h src/fsutil.c src/fsutil.h src/machine.c src/machine.h include/plugin-hostd/protocol.h $(RENDERED) + $(CC) -Iinclude -Isrc $(CONTRACT_CFLAGS) $(CFLAGS) -Werror -o $@ tests/history_test.c src/history.c src/fsutil.c src/machine.c -lm # src/proc.c's readers of /proc: the text the kernel writes, and this process's own files test-proc: $(PROC_TEST) diff --git a/src/history.c b/src/history.c index 96c81bf..c608041 100644 --- a/src/history.c +++ b/src/history.c @@ -78,8 +78,7 @@ static char g_path[PATH_MAX]; static char *g_class; static int g_save_s; static int g_dirty, g_blocked, g_started; -static long long g_saved_at; -static int g_saved_once; +static long long g_saved_at = -1; /* when the interval between two saves began; -1 until the first tick */ /* ---------------------------------------------------------------- the line grammar */ @@ -174,8 +173,8 @@ static int rec_parse(char *line, history_rec_t *r) return 0; } -/* " key=value" for every key, in the order of the words */ -static void keys_write(FILE *out, const history_rec_t *r) +/* "key=value" for every key, in the order of the words */ +static void keys_write(FILE *out, const history_rec_t *r, char sep) { int n; @@ -186,13 +185,13 @@ static void keys_write(FILE *out, const history_rec_t *r) switch (g_keys[n].kind) { case K_PERCENT: - fprintf(out, " %s=%.2f", g_words[n], *(const double *)v); + fprintf(out, "%c%s=%.2f", sep, g_words[n], *(const double *)v); break; case K_MIB: - fprintf(out, " %s=%.1f", g_words[n], *(const double *)v); + fprintf(out, "%c%s=%.1f", sep, g_words[n], *(const double *)v); break; default: - fprintf(out, " %s=%lld", g_words[n], *(const long long *)v); + fprintf(out, "%c%s=%lld", sep, g_words[n], *(const long long *)v); } } } @@ -206,7 +205,7 @@ char *history_line(const history_rec_t *r) if (!mem) return NULL; fprintf(mem, "%s %s %s %s", PHD_HISTORY_RECORD, r->uri, r->digest, r->klass); - keys_write(mem, r); + keys_write(mem, r, ' '); fputc('\n', mem); fclose(mem); return buf; @@ -443,7 +442,6 @@ static int file_write(void) static void save(long long now) { g_saved_at = now; - g_saved_once = 1; if (file_write() == 0) g_dirty = 0; } @@ -469,7 +467,11 @@ int history_init(const char *file, const char *klass, int save_s) void history_tick(long long now) { - if (g_started && !g_blocked && g_dirty && (!g_saved_once || now - g_saved_at >= g_save_s)) + if (!g_started || g_blocked) + return; + if (g_saved_at < 0) + g_saved_at = now; + if (g_dirty && now - g_saved_at >= g_save_s) save(now); } @@ -486,8 +488,8 @@ void history_finish(void) g_n = 0; free(g_class); g_class = NULL; - g_dirty = g_blocked = g_started = g_saved_once = 0; - g_saved_at = 0; + g_dirty = g_blocked = g_started = 0; + g_saved_at = -1; } /* ---------------------------------------------------------------- what is counted */ @@ -612,8 +614,8 @@ char *history_summary(const char *uri, const char *digest, long long now) if (!r || !(mem = open_memstream(&buf, &size))) return NULL; - fprintf(mem, "%s %s crashed=%d raises_xruns=%d", r->digest, r->klass, crashed(r, now), raises_xruns(r)); - keys_write(mem, r); + fprintf(mem, "digest=%s,class=%s,crashed=%d,raises_xruns=%d", r->digest, r->klass, crashed(r, now), raises_xruns(r)); + keys_write(mem, r, ','); fclose(mem); return buf; } diff --git a/src/history.h b/src/history.h index 0eb2e26..78be21d 100644 --- a/src/history.h +++ b/src/history.h @@ -70,11 +70,12 @@ void history_restart(const char *uri, const char *digest, int grouped, long long void history_quarantine(const char *uri, const char *digest, long long now); /* the summary the planner reads: crashed when a crash is within PHD_HISTORY_KEEP_DAYS of `now`, raises_xruns by the - * margin of protocol.h; malloc'd " crashed=<0|1> raises_xruns=<0|1> key=value ...", NULL for no record */ + * margin of protocol.h; malloc'd, one word: "digest=,class=,crashed=<0|1>,raises_xruns=<0|1>,key=value,..." with every key + * of the record, NULL for no record */ char *history_summary(const char *uri, const char *digest, long long now); -/* the idle tick: write the history when it changed and `save_s` have passed since the last write (`now` seconds, - * any steady origin) */ +/* the idle tick: write the history when it changed and `save_s` have passed since the last write, or since the first + * tick for the first one (`now` seconds, any steady origin) */ void history_tick(long long now); /* history_clear: the lines of one uri (every binary, class, and a held line that names it), or all of them, at once and diff --git a/tests/history_test.c b/tests/history_test.c index e5d6ada..4146bf2 100644 --- a/tests/history_test.c +++ b/tests/history_test.c @@ -26,12 +26,14 @@ #include #include #include +#include #include #include #include #include "history.h" +#include "machine.h" #define CLASS "x86_64/5-8/2.5-3.5GHz" #define URI "clap:/p/a.clap#org.test.a" @@ -185,9 +187,10 @@ static void test_round_trip(void) check(r->last_crash == 1180 && r->last_crash_grouped == 1240, "last crashes"); } summary = history_summary(URI, DIGEST, 1400); - check_str(summary, DIGEST " " CLASS " crashed=1 raises_xruns=0 first=1000 last=1360 windows_solo=2 windows_grouped=1 " - "cpu_p95_solo=10.19 cpu_max_solo=20.00 pss_mb_solo=12.3 xrun_windows_solo=1 xrun_windows_grouped=1 " - "crashes_solo=1 crashes_grouped=1 restarts_grouped=1 quarantines=1 last_crash=1180 last_crash_grouped=1240", + check_str(summary, "digest=" DIGEST ",class=" CLASS ",crashed=1,raises_xruns=0,first=1000,last=1360,windows_solo=2," + "windows_grouped=1,cpu_p95_solo=10.19,cpu_max_solo=20.00,pss_mb_solo=12.3,xrun_windows_solo=1," + "xrun_windows_grouped=1,crashes_solo=1,crashes_grouped=1,restarts_grouped=1,quarantines=1,last_crash=1180," + "last_crash_grouped=1240", "the summary history_get answers"); free(summary); history_finish(); @@ -252,14 +255,14 @@ static void test_summary(void) start(); history_crash(URI, DIGEST, 0, 1000); s = history_summary(URI, DIGEST, 1000 + 30 * DAY); - check(s && strstr(s, "crashed=1"), "a crash exactly 30 days ago still counts"); + check(s && strstr(s, ",crashed=1,"), "a crash exactly 30 days ago still counts"); free(s); s = history_summary(URI, DIGEST, 1000 + 30 * DAY + 1); - check(s && strstr(s, "crashed=0"), "a crash 30 days and a second ago does not"); + check(s && strstr(s, ",crashed=0,"), "a crash 30 days and a second ago does not"); free(s); history_crash(URI, DIGEST, 1, 1000 + 20 * DAY); s = history_summary(URI, DIGEST, 1000 + 30 * DAY + 1); - check(s && strstr(s, "crashed=1"), "a recent grouped crash counts though the solo one is old"); + check(s && strstr(s, ",crashed=1,"), "a recent grouped crash counts though the solo one is old"); free(s); history_finish(); @@ -269,11 +272,11 @@ static void test_summary(void) for (int w = 0; w < 29; w++) history_window(URI, DIGEST, 1, 1, 1, w < 29 ? 1 : 0, 100 + w); s = history_summary(URI, DIGEST, 200); - check(s && strstr(s, "raises_xruns=0"), "29 grouped windows, all with xruns, are too few to say"); + check(s && strstr(s, ",raises_xruns=0,"), "29 grouped windows, all with xruns, are too few to say"); free(s); history_window(URI, DIGEST, 1, 1, 1, 1, 200); s = history_summary(URI, DIGEST, 200); - check(s && strstr(s, "raises_xruns=1"), "30 grouped windows, all with xruns, are enough"); + check(s && strstr(s, ",raises_xruns=1,"), "30 grouped windows, all with xruns, are enough"); free(s); history_finish(); @@ -282,7 +285,7 @@ static void test_summary(void) for (int w = 0; w < 30; w++) history_window(URI, DIGEST, 1, 1, 1, w < 1 ? 1 : 0, 100 + w); s = history_summary(URI, DIGEST, 200); - check(s && strstr(s, "raises_xruns=0"), "1 of 30 grouped windows is 3.3 percent: under the 5 percent"); + check(s && strstr(s, ",raises_xruns=0,"), "1 of 30 grouped windows is 3.3 percent: under the 5 percent"); free(s); history_finish(); fresh(); @@ -290,7 +293,7 @@ static void test_summary(void) for (int w = 0; w < 30; w++) history_window(URI, DIGEST, 1, 1, 1, w < 2 ? 1 : 0, 100 + w); s = history_summary(URI, DIGEST, 200); - check(s && strstr(s, "raises_xruns=1"), "2 of 30 grouped windows is 6.7 percent: over it"); + check(s && strstr(s, ",raises_xruns=1,"), "2 of 30 grouped windows is 6.7 percent: over it"); free(s); history_finish(); fresh(); @@ -298,7 +301,7 @@ static void test_summary(void) for (int w = 0; w < 40; w++) history_window(URI, DIGEST, 1, 1, 1, w < 2 ? 1 : 0, 100 + w); s = history_summary(URI, DIGEST, 200); - check(s && strstr(s, "raises_xruns=0"), "2 of 40 grouped windows is exactly 5 percent: not over it"); + check(s && strstr(s, ",raises_xruns=0,"), "2 of 40 grouped windows is exactly 5 percent: not over it"); free(s); history_finish(); @@ -310,11 +313,11 @@ static void test_summary(void) for (int w = 0; w < 40; w++) history_window(URI, DIGEST, 1, 1, 1, w < 6 ? 1 : 0, 300 + w); /* grouped: 6 of 40 = 15 %: exactly +5 */ s = history_summary(URI, DIGEST, 400); - check(s && strstr(s, "raises_xruns=0"), "exactly 5 points over the solo share does not exceed it"); + check(s && strstr(s, ",raises_xruns=0,"), "exactly 5 points over the solo share does not exceed it"); free(s); history_window(URI, DIGEST, 1, 1, 1, 1, 500); /* 7 of 41 = 17.07 % */ s = history_summary(URI, DIGEST, 600); - check(s && strstr(s, "raises_xruns=1"), "more than 5 points over the solo share is a plugin that raises xruns"); + check(s && strstr(s, ",raises_xruns=1,"), "more than 5 points over the solo share is a plugin that raises xruns"); free(s); s = history_summary(URI, OTHER, 600); check(s == NULL, "no record of another binary"); @@ -443,6 +446,7 @@ static void test_blocked_files(void) history_crash(URI, DIGEST, 0, 4); check(history_clear("all") == -1, "a blocked history is not cleared"); history_tick(100000); + history_tick(200000); history_finish(); text = file_text(); check_str(text, cases[n].content, "a blocked history is never overwritten"); @@ -516,8 +520,9 @@ static void test_a_failed_save(void) } fresh(); start(); - history_window(URI, DIGEST, 0, 1.0, 1.0, 0, 3); history_tick(0); + history_window(URI, DIGEST, 0, 1.0, 1.0, 0, 3); + history_tick(300); before = file_text(); check(strlen(before) > strlen(HEADER), "the first save wrote the record"); history_window(URI, DIGEST, 0, 2.0, 2.0, 0, 70); @@ -546,22 +551,25 @@ static void test_save_interval(void) fresh(); history_init(file, CLASS, 300); - history_tick(5); - check_str(file_text(), "", "nothing changed, nothing written"); history_window(URI, DIGEST, 0, 1.0, 1.0, 0, 3); - history_tick(10); + history_tick(1000); /* the first tick opens the first interval */ + check_str(file_text(), "", "the first interval has not passed: nothing written"); + history_tick(1299); + check_str(file_text(), "", "299 s into it: still nothing"); + history_tick(1300); t = file_text(); - check(strstr(t, "windows_solo=1") != NULL, "the first change is written at the first tick"); + check(strstr(t, "windows_solo=1") != NULL, "300 s into it the change is written"); free(t); history_window(URI, DIGEST, 0, 1.0, 1.0, 0, 70); - history_tick(309); + history_tick(1599); t = file_text(); - check(strstr(t, "windows_solo=1") != NULL, "299 s after the save it is not written"); + check(strstr(t, "windows_solo=1") != NULL, "299 s after a save the next change is not written"); free(t); - history_tick(310); + history_tick(1600); t = file_text(); check(strstr(t, "windows_solo=2") != NULL, "300 s after it is"); free(t); + history_tick(5000); history_window(URI, DIGEST, 0, 1.0, 1.0, 0, 130); history_finish(); t = file_text(); @@ -630,6 +638,49 @@ static void test_the_path(void) group("the_default_path_is_the_xdg_state_home_then_home_and_history_file_overrides_both"); } +static void test_host_class(void) +{ + static const struct { long ncpu; const char *want; } cores[] = { + { 1, "1-2" }, { 2, "1-2" }, { 3, "3-4" }, { 4, "3-4" }, { 5, "5-8" }, { 8, "5-8" }, { 9, "9-16" }, { 16, "9-16" }, + { 17, "17-32" }, { 32, "17-32" }, { 33, "33+" }, { 256, "33+" }, + }; + static const struct { long khz; const char *want; } clocks[] = { + { 0, "<1.5GHz" }, { 1499999, "<1.5GHz" }, { 1500000, "1.5-2.5GHz" }, { 2499999, "1.5-2.5GHz" }, + { 2500000, "2.5-3.5GHz" }, { 3499999, "2.5-3.5GHz" }, { 3500000, "3.5-4.5GHz" }, { 4499999, "3.5-4.5GHz" }, + { 4500000, "4.5GHz+" }, { 5800000, "4.5GHz+" }, + }; + const char *core, *clock; + char klass[64], want[96]; + struct utsname u; + unsigned n; + + for (n = 0; n < sizeof(cores) / sizeof(cores[0]); n++) + { + char what[48]; + + machine_bands(cores[n].ncpu, 3000000, &core, &clock); + snprintf(what, sizeof(what), "%ld cores", cores[n].ncpu); + check_str(core, cores[n].want, what); + } + for (n = 0; n < sizeof(clocks) / sizeof(clocks[0]); n++) + { + char what[48]; + + machine_bands(4, clocks[n].khz, &core, &clock); + snprintf(what, sizeof(what), "%ld kHz", clocks[n].khz); + check_str(clock, clocks[n].want, what); + } + check(machine_class(klass, sizeof(klass)) == 0, "this machine has a class"); + uname(&u); + snprintf(want, sizeof(want), "%s/", u.machine); + check(!strncmp(klass, want, strlen(want)), "it starts with uname's machine"); + check(strchr(klass, '/') && strchr(strchr(klass, '/') + 1, '/') && !strchr(strchr(strchr(klass, '/') + 1, '/') + 1, '/'), + "arch/cores/clock: two slashes"); + check(!strpbrk(klass, " \t\n"), "and no white space"); + check(machine_class(klass, 5) == -1, "a buffer that is too small is refused, not cut"); + group("the_host_class_is_arch_cores_clock_with_the_contracts_bands_cut_where_the_contract_cuts_them"); +} + int main(void) { snprintf(dir, sizeof(dir), "/tmp/plugin-hostd-history-test.XXXXXX"); @@ -652,6 +703,7 @@ int main(void) test_save_interval(); test_clear(); test_the_path(); + test_host_class(); group(NULL); fresh(); rmdir(dir); diff --git a/tests/sabotage.py b/tests/sabotage.py index c95c9e7..3f13004 100644 --- a/tests/sabotage.py +++ b/tests/sabotage.py @@ -797,8 +797,8 @@ '', 'a_record_round_trips_through_its_file_value_for_value', "history"), ('the pss is written to two decimals', 'src/history.c', - '" %s=%.1f"', - '" %s=%.2f"', + '"%c%s=%.1f"', + '"%c%s=%.2f"', 'a_record_round_trips_through_its_file_value_for_value', "history"), ('a restart of a plugin alone is counted', 'src/history.c', 'history_rec_t *r = grouped ? rec_get(uri, digest) : NULL;', @@ -845,12 +845,12 @@ 'g_entry[n].raw ? raw_names(g_entry[n].raw, "") :', 'clear_removes_a_plugins_lines_of_every_binary_and_class_or_all_of_them_at_once', "history"), ('the history is saved at every tick', 'src/history.c', - '(!g_saved_once || now - g_saved_at >= g_save_s)', - '1', + 'if (g_dirty && now - g_saved_at >= g_save_s)', + 'if (g_dirty)', 'the_history_is_saved_at_most_once_per_interval_and_at_quit', "history"), ('a failed save forgets the change', 'src/history.c', - ' g_saved_once = 1;\n if (file_write() == 0)\n g_dirty = 0;', - ' g_saved_once = 1;\n file_write();\n g_dirty = 0;', + ' g_saved_at = now;\n if (file_write() == 0)\n g_dirty = 0;', + ' g_saved_at = now;\n file_write();\n g_dirty = 0;', 'a_save_that_fails_leaves_the_old_file_whole_and_is_tried_again', "history"), ('the state home is not looked at', 'src/history.c', ' else if (state && *state)', @@ -880,6 +880,26 @@ '*seconds = (double)(u + s) / (double)hz;', '*seconds = (double)(u + s);', "proc_test", "proc"), + ('a machine of 8 cores is in the band of 9 to 16', 'src/machine.c', + 'ncpu <= 8 ?', + 'ncpu < 8 ?', + "the_host_class_is_arch_cores_clock_with_the_contracts_bands_cut_where_the_contract_cuts_them", "history"), + ('a clock of 2.5 GHz is in the band under it', 'src/machine.c', + 'khz < 2500000 ?', + 'khz <= 2500000 ?', + "the_host_class_is_arch_cores_clock_with_the_contracts_bands_cut_where_the_contract_cuts_them", "history"), + ('the class is cores/arch/clock', 'src/machine.c', + 'n = snprintf(out, size, "%s/%s/%s", u.machine, core, clock);', + 'n = snprintf(out, size, "%s/%s/%s", core, u.machine, clock);', + "the_host_class_is_arch_cores_clock_with_the_contracts_bands_cut_where_the_contract_cuts_them", "history"), + ('a class that does not fit is cut', 'src/machine.c', + 'return n > 0 && (size_t)n < size ? 0 : -1;', + 'return n > 0 ? 0 : -1;', + "the_host_class_is_arch_cores_clock_with_the_contracts_bands_cut_where_the_contract_cuts_them", "history"), + ('the first interval is over before it began', 'src/history.c', + 'if (g_saved_at < 0)\n g_saved_at = now;', + 'if (g_saved_at < 0)\n g_saved_at = now - g_save_s;', + 'the_history_is_saved_at_most_once_per_interval_and_at_quit', "history"), ] From 5d43143d1624891dffe031bb00307535d40bc27b Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Tue, 27 Oct 2026 07:17:41 +0100 Subject: [PATCH 07/16] Count each worker's windows and deaths into the history; history_get and history_clear The supervisor keeps one window per worker: its CPU from /proc//stat and its Pss from smaps_rollup after history_window_s, folded into every member that has a pin, alone or in a group. A window in which the members changed is dropped; a worker that dies is counted if it was up for half the window. A crash the daemon names a culprit for, a member that went down with a group and a quarantine are counted where they happen. The records are written by the idle tick at most once per history_save_s and at quit. history_get answers the summary of the binary the pin holds now, as one word, - for none and unavailable while the file is blocked or history_window_s is 0; history_clear deletes it at once. Both are declared in data/verbs.json and tell the fuzz recorder. The test harness gives every daemon a history file of its own. Eight daemon tests and nineteen sabotage arms cover the counts, the windows, the verbs, the quit save and the blocked file (#184). --- README.md | 40 +++ data/verbs.json | 28 ++ debian/control | 5 +- .../corpus/command/plugin-hostd-history_clear | Bin 0 -> 32 bytes .../command/plugin-hostd-history_clear-extra | Bin 0 -> 38 bytes .../plugin-hostd-history_clear-refused | Bin 0 -> 32 bytes .../command/plugin-hostd-history_clear-tabs | Bin 0 -> 34 bytes fuzz/corpus/command/plugin-hostd-history_get | Bin 0 -> 30 bytes .../command/plugin-hostd-history_get-extra | Bin 0 -> 36 bytes .../command/plugin-hostd-history_get-refused | Bin 0 -> 30 bytes .../command/plugin-hostd-history_get-tabs | Bin 0 -> 32 bytes fuzz/fuzz_command.c | 14 +- fuzz/recorder.c | 16 + fuzz/recorder.h | 1 + .../plugin-hostd/plugin_hostd_host_verbs.h | 18 +- include/plugin-hostd/protocol.h | 16 +- packaging/plugin-hostd.spec | 3 +- protocol/plugin-hostd.json | 8 +- src/conf.c | 10 + src/main.c | 21 ++ src/pins.c | 10 + src/pins.h | 5 + src/supervisor.c | 177 ++++++++++- src/supervisor.h | 13 + src/verbs.c | 4 + tests/daemon_test.py | 297 ++++++++++++++++++ tests/harness.py | 2 + tests/sabotage.py | 80 +++++ tests/verbs_declared.py | 2 + 29 files changed, 755 insertions(+), 15 deletions(-) create mode 100644 fuzz/corpus/command/plugin-hostd-history_clear create mode 100644 fuzz/corpus/command/plugin-hostd-history_clear-extra create mode 100644 fuzz/corpus/command/plugin-hostd-history_clear-refused create mode 100644 fuzz/corpus/command/plugin-hostd-history_clear-tabs create mode 100644 fuzz/corpus/command/plugin-hostd-history_get create mode 100644 fuzz/corpus/command/plugin-hostd-history_get-extra create mode 100644 fuzz/corpus/command/plugin-hostd-history_get-refused create mode 100644 fuzz/corpus/command/plugin-hostd-history_get-tabs diff --git a/README.md b/README.md index 235333d..100c7ec 100644 --- a/README.md +++ b/README.md @@ -185,6 +185,8 @@ Verbs the daemon adds | `chain_get` | `` | `resp 0 ,... ,...` | read a chain back: its worker (- when it has none), its members in the declared order, and the members its worker runs now in their order (- for none); a quarantined member is declared and not running; -522 for a chain no chain declared, -902 for a chain that is not a number | | `descriptor_get` | `` | `resp 0 ""` | the scanned descriptor of the plugin, answered from descriptor_dir: the file named for its binary as it is now (@-.json), written by the scan the plugin's add ran. -529 when there is none, -530 while its scan runs, -531 when its last scan wrote nothing, -528 when the directory could not be written | | `panel_get` | `` | `resp 0 ""` | the panel the plugin publishes (org.openmixer.panel-description/1), answered from descriptor_dir: the file panels/@-.json, named as its descriptor, which the same scan writes once the descriptor is in place, {"plugin": {"format", "id"}, "panel": }; an LV2 plugin's panel is null. The codes are descriptor_get's: -529 when there is none, -530 while its scan runs, -531 when its last scan wrote nothing, -528 when the directory could not be written | +| `history_get` | `` | `resp 0 ` | what the daemon kept of the plugin on this host class, for the binary its pin holds now (the running history: the windows it ran alone and in a group, its crashes, restarts and quarantines): one word of comma-joined digest=, class=, crashed=<0\|1>, raises_xruns=<0\|1> and then every key=value of the record (data/history.json); - when the plugin has no pin or no record, unavailable while the history file is blocked or history_window_s is 0, -902 for a line that is not one word | +| `history_clear` | ` \| all` | `resp 0` | delete the history of one plugin (every binary and host class of it), or all of it, from memory and from the file at once; -902 while the history is blocked or off, or for a line that is not one word | A `` is `up` (accepting commands), `starting` (spawned, not yet accepting), `backoff` (dead, waiting to be respawned), `given-up` (the storm bound is spent for its placement). @@ -325,6 +327,8 @@ mechanism, and they stay optional. The preset verbs and the preset file are [doc | `chain_get` | daemon | required | required | required | required | | `descriptor_get` | daemon | required | required | required | required | | `panel_get` | daemon | required | required | required | required | +| `history_get` | daemon | required | required | required | required | +| `history_clear` | daemon | required | required | required | required | The profiles: `lv2-worker`, an LV2 plugin's worker: mod-host, or omx-clap-host's LV2 adapter; `clap-worker`, a CLAP plugin's worker, omx-clap-host; `console`, the openmixer console as a controller of the daemon; `zynthian`, Zynthian's mod-ui and its plugind engine as a controller of the daemon. `required` is answered as its grammar says; `optional` is answered, or refused with -902, and never misread; `absent` is refused on purpose, and no client of the profile may need it. @@ -551,6 +555,42 @@ when it refuses the replayed `pin_expect` or `add`), writes `PHD_ERR_REPLAY`, th the worker does not hold that instance until a later replay passes. The hashing and the layout serialisation are `include/plugin-hostd/pin.h`, installed beside the protocol header for the hosts that check a layout. +The running history +------------------- + +The daemon keeps what each plugin did on this machine, per plugin, per binary and per host class: the file named by +`history_file`, else `$XDG_STATE_HOME/plugin-hostd/history`, else `$HOME/.local/state/plugin-hostd/history`. It is +local: it holds URIs, digests and counters, no audio, no path, no user, session or preset name, the daemon sends it +nowhere and no verb writes it from outside; `history_clear ` deletes it. Only the daemon writes it, mode +0600 in a directory of mode 0700, to a temporary file that is renamed over it, at most once per `history_save_s` and +at quit. `history_window_s` 0 keeps none. + +The first line is `# plugin-hostd history/1`, then one line per key, `history +key=value ...`. The binary digest is the SHA-256 of the first file the plugin's pin holds, so a plugin with no pin has +no history and a changed plugin starts a record of its own, the old one kept and not read. The host class is +`arch/cores/clock` as the probe writes it (`x86_64/5-8/2.5-3.5GHz`), or the `host_class` setting. The keys are the +words of `PLUGIN_HOSTD_HISTORY_KEYS` in [data/history.json](data/history.json): `first`, `last`, `windows_solo`, +`windows_grouped`, `cpu_p95_solo`, `cpu_max_solo`, `pss_mb_solo`, `xrun_windows_solo`, `xrun_windows_grouped`, +`crashes_solo`, `crashes_grouped`, `restarts_grouped`, `quarantines`, `last_crash`, `last_crash_grouped`. + +A window is `history_window_s` seconds of one worker: its utime plus stime from `/proc//stat` as percent of one +core, and the `Pss` of its `smaps_rollup`. A worker of its own gives a window to the plugin it holds; a worker of two +or more gives one to each member, counted in `windows_grouped` and nowhere else, because a group's CPU cannot be split +among its members. `cpu_p95_solo` is an estimate that needs no list of windows: each window above it moves it up +by 0.95 of 2 percent of itself and each one not above moves it down by 0.05 of that. A window in which the members of +the worker changed is dropped, and a worker that died is counted if it had been up for half the window. A crash is +counted when the daemon names a culprit (`crashes_solo` in a worker of its own, `crashes_grouped` in a group), the +members that went down with it and were not the culprit get `restarts_grouped`, and a quarantine counts in +`quarantines`. No worker reports xruns yet, so `xrun_windows_*` stay 0. + +A line this version cannot read, or with a key it does not know, is kept as it is by every save and is not read. A file +that cannot be read, or whose first line is another version, is neither read nor overwritten: stderr says why, and +`history_get` answers `unavailable` until the operator repairs or removes it. + +`history_get ` answers the record of the binary the pin holds now, on this host class, as one word: `digest=`, +`class=`, `crashed=1` when a crash is within 30 days, `raises_xruns=1` when the share of xrun windows in a group +exceeds the share alone by 5 points (from 30 windows in a group), and then every key. `-` is no record. + What a worker must do --------------------- diff --git a/data/verbs.json b/data/verbs.json index 40160ac..c55899e 100644 --- a/data/verbs.json +++ b/data/verbs.json @@ -492,6 +492,34 @@ "mod-host": "unsupported" }, "doc": "One preset, 0 to count - 1: the key preset_load takes for it, and its name." + }, + "historyGet": { + "verb": "history_get", + "format": "history_get %s", + "reply": "resp 0 ", + "codes": [ + { + "ref": "HOST_REPLY_CODES.invalidOperation" + } + ], + "implementedBy": { + "plugin-hostd": "required" + }, + "doc": "What the daemon kept of one plugin on this host class, for the binary its pin holds now: . The history is one word, digest=,class=,crashed=<0|1>,raises_xruns=<0|1> and then every key=value of PLUGIN_HOSTD_HISTORY_KEYS, joined by commas; - when the plugin has no record or no pin, unavailable while the history file is blocked." + }, + "historyClear": { + "verb": "history_clear", + "format": "history_clear %s", + "reply": "resp 0", + "codes": [ + { + "ref": "HOST_REPLY_CODES.invalidOperation" + } + ], + "implementedBy": { + "plugin-hostd": "required" + }, + "doc": "Delete the history of one plugin, every binary and host class, or all of it: | all. Refused while the history file is blocked." } } } diff --git a/debian/control b/debian/control index 4e6f4fa..022b35f 100644 --- a/debian/control +++ b/debian/control @@ -63,6 +63,7 @@ Description: plugin-hostd's format tables and verbs, read at run time plugin-hostd's package of omx-contract's contract directory: the format tables a scan maps a CLAP or LV2 plugin through (formats/clap.json, formats/lv2.json) and the verbs the daemon adds to mod-host's protocol, with - their replies, their refusals and what each host owes them (verbs.json), - under /usr/share/omx-contract/contract.d/3/plugin-hostd. A program that reads + their replies, their refusals and what each host owes them (verbs.json), and + the keys of the running history (history.json), under + /usr/share/omx-contract/contract.d/3/plugin-hostd. A program that reads them at run time depends on this package, never on the development one. diff --git a/fuzz/corpus/command/plugin-hostd-history_clear b/fuzz/corpus/command/plugin-hostd-history_clear new file mode 100644 index 0000000000000000000000000000000000000000..76357615d33233e8e9049b6727ec48eb6839ed3d GIT binary patch literal 32 ncmZROWyr`ZF3B&dj8D!E3K-sD#$5K&&&e=wBQTf literal 0 HcmV?d00001 diff --git a/fuzz/corpus/command/plugin-hostd-history_clear-extra b/fuzz/corpus/command/plugin-hostd-history_clear-extra new file mode 100644 index 0000000000000000000000000000000000000000..b2e084aae6be853425e856aba639342c3a5ca944 GIT binary patch literal 38 tcmZQjX2{4aF3B&dj8D!E3K-sD#$5K&&*Ruttcr<1ONmz4krKr literal 0 HcmV?d00001 diff --git a/fuzz/corpus/command/plugin-hostd-history_clear-refused b/fuzz/corpus/command/plugin-hostd-history_clear-refused new file mode 100644 index 0000000000000000000000000000000000000000..e2f9147e6ff8eba04ce15989172bbe4eeb648d3e GIT binary patch literal 32 ncmZRTWyr`ZF3B&dj8D!E3K-sD#$5K&&&e=wSWua literal 0 HcmV?d00001 diff --git a/fuzz/corpus/command/plugin-hostd-history_clear-tabs b/fuzz/corpus/command/plugin-hostd-history_clear-tabs new file mode 100644 index 0000000000000000000000000000000000000000..7f1cf27f12f86533ff8062c19d3bad530d3abec5 GIT binary patch literal 34 pcmZROXUNDbF3B&dj8D!E3K-sD#$5K&&=cH0szZm3*P_$ literal 0 HcmV?d00001 diff --git a/fuzz/corpus/command/plugin-hostd-history_get b/fuzz/corpus/command/plugin-hostd-history_get new file mode 100644 index 0000000000000000000000000000000000000000..438c8e1be3e05fe18f7f689f9af0740bd24070be GIT binary patch literal 30 lcmZROX2{4aF3B&dj89K3Q7A3Svq~$is `return SUCCESS;` * input: plugin-hostd-pin_set-refused - * mutant: src/verbs.c:220 `return reply_text;` -> `return sup_resp(SUCCESS);` + * mutant: src/verbs.c:224 `return reply_text;` -> `return sup_resp(SUCCESS);` * input: mod-host-set_bpm-refused */ if (strcmp(g_rec.reply, g_rec.answer)) abort(); @@ -229,7 +229,7 @@ __attribute__((noinline)) static void check_reply(const struct words *w) int code; /* one reply, and nothing sent empty - * mutant: src/verbs.c:238 `reply(msg->sender_id, handle(line, tok, ntok));` -> `free(handle(line, tok, ntok));` + * mutant: src/verbs.c:242 `reply(msg->sender_id, handle(line, tok, ntok));` -> `free(handle(line, tok, ntok));` * input: add-default */ if (g_rec.nreplies != 1 || g_rec.reply_size == 0) abort(); @@ -252,7 +252,7 @@ __attribute__((noinline)) static void check_reply(const struct words *w) } shape = w->n ? rec_shape_of(w->w[0], strlen(w->w[0])) : REC_MOD_HOST_SHAPE; /* anything else has the shape its verb declares, or mod-host's - * mutant: src/verbs.c:209 `sup_resp(SUCCESS)` -> `sup_resp(1)` + * mutant: src/verbs.c:213 `sup_resp(SUCCESS)` -> `sup_resp(1)` * input: mod-host-quit */ if (!syntax_reply_matches(shape, g_rec.reply) && !syntax_reply_matches(REC_MOD_HOST_SHAPE, g_rec.reply)) abort(); @@ -329,6 +329,8 @@ static const char *const g_call_words[REC_FN_COUNT] = { [REC_DESCRIPTORS_GET] = "1", [REC_DESCRIPTORS_PANEL_GET] = "1", [REC_LV2_REFUSALS_FORGET] = "*", + [REC_HISTORY_GET] = "1", + [REC_HISTORY_CLEAR] = "1", }; static int is_placement(const char *word) @@ -485,17 +487,17 @@ __attribute__((noinline)) static void check_round_trip(const struct words *w, ui free(line); /* the same calls with the same arguments - * mutant: src/verbs.c:232 `strtok_r(copy, " \t", &save)` -> `strtok_r(copy, " ", &save)` + * mutant: src/verbs.c:236 `strtok_r(copy, " \t", &save)` -> `strtok_r(copy, " ", &save)` * input: mod-host-add-tabs */ if (!rec_calls_equal(first, nfirst, g_rec.calls, g_rec.ncalls)) abort(); /* the same reply - * mutant: src/verbs.c:206 `ntok != 1` -> `ntok != 1 || line[0] != ' '` + * mutant: src/verbs.c:210 `ntok != 1` -> `ntok != 1 || line[0] != ' '` * input: quit-leading-space */ if (g_rec.reply == NULL || strcmp(g_rec.reply, reply_first)) abort(); /* the same state, the same host settings, and quitting the same - * mutant: src/verbs.c:208 `g_quitting = 1;` -> `g_quitting = line[0] == ' ';` + * mutant: src/verbs.c:212 `g_quitting = 1;` -> `g_quitting = line[0] == ' ';` * input: quit-leading-space */ if (g_quitting != quit_first || g_rec.state != state_first || g_rec.host_settings != host_first) abort(); diff --git a/fuzz/recorder.c b/fuzz/recorder.c index bd14e04..5c6b35b 100644 --- a/fuzz/recorder.c +++ b/fuzz/recorder.c @@ -494,6 +494,22 @@ char *sup_chain_get(int id) return answer(c, daemon_shape(OMX_VERB_CHAIN_GET), 1); } +char *sup_history_get(const char *uri) +{ + struct rec_call *c = record(REC_HISTORY_GET); + + arg_s(c, uri); + return answer(c, daemon_shape(OMX_VERB_HISTORY_GET), 0); +} + +char *sup_history_clear(const char *uri) +{ + struct rec_call *c = record(REC_HISTORY_CLEAR); + + arg_s(c, uri); + return answer(c, daemon_shape(OMX_VERB_HISTORY_CLEAR), 1); +} + /* a checkpoint of what a preset or patch changed: it writes down state, it changes none */ void sup_after_reply(int id, const char *verb) { diff --git a/fuzz/recorder.h b/fuzz/recorder.h index f4367d0..f2d3616 100644 --- a/fuzz/recorder.h +++ b/fuzz/recorder.h @@ -34,6 +34,7 @@ enum { REC_WORKER_LIST, REC_INSTANCE_INFO, REC_RESET, REC_QUARANTINE_CLEAR, REC_POLICY_SET, REC_PIN_ACCEPT, REC_POOL_CONFIG, REC_LV2_WORKER_SET, REC_WORKER_ENV, REC_CHAIN, REC_CHAIN_CLEAR, REC_CHAIN_GET, REC_AFTER_REPLY, REC_PINS_SET, REC_PINS_CLEAR, REC_DESCRIPTORS_ADD, REC_DESCRIPTORS_GET, REC_DESCRIPTORS_PANEL_GET, REC_LV2_REFUSALS_FORGET, + REC_HISTORY_GET, REC_HISTORY_CLEAR, REC_FN_COUNT }; diff --git a/include/plugin-hostd/plugin_hostd_host_verbs.h b/include/plugin-hostd/plugin_hostd_host_verbs.h index 99a017f..c9afe6b 100644 --- a/include/plugin-hostd/plugin_hostd_host_verbs.h +++ b/include/plugin-hostd/plugin_hostd_host_verbs.h @@ -166,7 +166,21 @@ struct omx_host_verb { #define OMX_VERB_PRESET_GET_CODES_COUNT 3u #define OMX_VERB_PRESET_GET_CODES_INIT { -515, -3, -902 } /* reply.plugin-hostd.no-preset-list, reply.host.instance-non-exists, reply.host.invalid-operation */ -#define PLUGIN_HOSTD_HOST_VERB_COUNT 19u +/* PLUGIN_HOSTD_VERBS.historyGet (data/verbs.json): What the daemon kept of one plugin on this host class, for the binary its pin holds now: . The history is one word, digest=,class=,crashed=<0|1>,raises_xruns=<0|1> and then every key=value of PLUGIN_HOSTD_HISTORY_KEYS, joined by commas; - when the plugin has no record or no pin, unavailable while the history file is blocked. */ +#define OMX_VERB_HISTORY_GET "history_get" +#define OMX_VERB_HISTORY_GET_FMT "history_get %s" +#define OMX_VERB_HISTORY_GET_REPLY "resp 0 " +#define OMX_VERB_HISTORY_GET_CODES_COUNT 1u +#define OMX_VERB_HISTORY_GET_CODES_INIT { -902 } /* reply.host.invalid-operation */ + +/* PLUGIN_HOSTD_VERBS.historyClear (data/verbs.json): Delete the history of one plugin, every binary and host class, or all of it: | all. Refused while the history file is blocked. */ +#define OMX_VERB_HISTORY_CLEAR "history_clear" +#define OMX_VERB_HISTORY_CLEAR_FMT "history_clear %s" +#define OMX_VERB_HISTORY_CLEAR_REPLY "resp 0" +#define OMX_VERB_HISTORY_CLEAR_CODES_COUNT 1u +#define OMX_VERB_HISTORY_CLEAR_CODES_INIT { -902 } /* reply.host.invalid-operation */ + +#define PLUGIN_HOSTD_HOST_VERB_COUNT 21u /* Every verb, in the order of its items, with what each host owes it. */ static inline const struct omx_host_verb *plugin_hostd_host_verbs(void) { @@ -190,6 +204,8 @@ static inline const struct omx_host_verb *plugin_hostd_host_verbs(void) { { OMX_VERB_PANEL_GET, OMX_VERB_PANEL_GET_FMT, OMX_VERB_PANEL_GET_REPLY, { OMX_HOST_VERB_REQUIRED, OMX_HOST_VERB_UNSUPPORTED, OMX_HOST_VERB_UNSUPPORTED } }, { OMX_VERB_PRESET_LIST, OMX_VERB_PRESET_LIST_FMT, OMX_VERB_PRESET_LIST_REPLY, { OMX_HOST_VERB_OPTIONAL, OMX_HOST_VERB_OPTIONAL, OMX_HOST_VERB_UNSUPPORTED } }, { OMX_VERB_PRESET_GET, OMX_VERB_PRESET_GET_FMT, OMX_VERB_PRESET_GET_REPLY, { OMX_HOST_VERB_OPTIONAL, OMX_HOST_VERB_OPTIONAL, OMX_HOST_VERB_UNSUPPORTED } }, + { OMX_VERB_HISTORY_GET, OMX_VERB_HISTORY_GET_FMT, OMX_VERB_HISTORY_GET_REPLY, { OMX_HOST_VERB_REQUIRED, OMX_HOST_VERB_UNSUPPORTED, OMX_HOST_VERB_UNSUPPORTED } }, + { OMX_VERB_HISTORY_CLEAR, OMX_VERB_HISTORY_CLEAR_FMT, OMX_VERB_HISTORY_CLEAR_REPLY, { OMX_HOST_VERB_REQUIRED, OMX_HOST_VERB_UNSUPPORTED, OMX_HOST_VERB_UNSUPPORTED } }, }; return table; } diff --git a/include/plugin-hostd/protocol.h b/include/plugin-hostd/protocol.h index 1d8f390..5b4ed3c 100644 --- a/include/plugin-hostd/protocol.h +++ b/include/plugin-hostd/protocol.h @@ -97,6 +97,7 @@ #define PHD_FORMAT_ARGUMENT "" #define PHD_WORD_ALL "all" #define PHD_WORD_NONE "-" +#define PHD_WORD_UNAVAILABLE "unavailable" /* the built-in formats, X(id, name) in the order of the daemon's table (src/formats.c): the rest of what a format is * (its prefix, its worker, the files its pin holds, whether its pin holds in a shared world, who answers its instance @@ -499,7 +500,16 @@ static inline int phd_pool_name_valid(const char *name) "{\"plugin\": {\"format\", \"id\"}, \"panel\": }; an LV2 plugin's panel is null. The codes are " OMX_VERB_DESCRIPTOR_GET "'s: -529 " \ "when there is none, -530 while its scan runs, -531 when its last scan wrote nothing, -528 when the directory " \ - "could not be written") + "could not be written") \ + X(HISTORY_GET, OMX_VERB_HISTORY_GET, "", OMX_VERB_HISTORY_GET_REPLY, \ + "what the daemon kept of the plugin on this host class, for the binary its pin holds now (the running history: " \ + "the windows it ran alone and in a group, its crashes, restarts and quarantines): one word of comma-joined " \ + "digest=, class=, crashed=<0|1>, raises_xruns=<0|1> and then every key=value of the record " \ + "(data/history.json); " PHD_WORD_NONE " when the plugin has no pin or no record, " PHD_WORD_UNAVAILABLE " while the " \ + "history file is blocked or history_window_s is 0, -902 for a line that is not one word") \ + X(HISTORY_CLEAR, OMX_VERB_HISTORY_CLEAR, " | " PHD_WORD_ALL, OMX_VERB_HISTORY_CLEAR_REPLY, \ + "delete the history of one plugin (every binary and host class of it), or all of it, from memory and from the " \ + "file at once; -902 while the history is blocked or off, or for a line that is not one word") /* ---------------------------------------------------------------- instance verbs */ @@ -827,7 +837,9 @@ static inline int phd_is_note_verb(const char *verb) X(OMX_VERB_CHAIN_CLEAR, DAEMON, REQUIRED, REQUIRED, REQUIRED, REQUIRED) \ X(OMX_VERB_CHAIN_GET, DAEMON, REQUIRED, REQUIRED, REQUIRED, REQUIRED) \ X(OMX_VERB_DESCRIPTOR_GET, DAEMON, REQUIRED, REQUIRED, REQUIRED, REQUIRED) \ - X(OMX_VERB_PANEL_GET, DAEMON, REQUIRED, REQUIRED, REQUIRED, REQUIRED) + X(OMX_VERB_PANEL_GET, DAEMON, REQUIRED, REQUIRED, REQUIRED, REQUIRED) \ + X(OMX_VERB_HISTORY_GET, DAEMON, REQUIRED, REQUIRED, REQUIRED, REQUIRED) \ + X(OMX_VERB_HISTORY_CLEAR, DAEMON, REQUIRED, REQUIRED, REQUIRED, REQUIRED) /* ---------------------------------------------------------------- host settings */ diff --git a/packaging/plugin-hostd.spec b/packaging/plugin-hostd.spec index cc6b26a..3545b9a 100644 --- a/packaging/plugin-hostd.spec +++ b/packaging/plugin-hostd.spec @@ -73,7 +73,8 @@ Requires: omx-clap-core-contract >= 0.22.0 plugin-hostd's package of omx-contract's contract directory: the format tables a scan maps a CLAP or LV2 plugin through (formats/clap.json, formats/lv2.json) and the verbs the daemon adds to mod-host's protocol, with their replies, -their refusals and what each host owes them (verbs.json), under +their refusals and what each host owes them (verbs.json), and the keys of the +running history (history.json), under share/omx-contract/contract.d/3/plugin-hostd. A program that reads them at run time requires this package, never the development one. diff --git a/protocol/plugin-hostd.json b/protocol/plugin-hostd.json index 06ca8a3..09f5f27 100644 --- a/protocol/plugin-hostd.json +++ b/protocol/plugin-hostd.json @@ -77,7 +77,9 @@ {"name": "chain_clear", "arguments": "", "reply": "resp 0", "meaning": "forget the chain: each member goes to a worker of its own; -522 for a chain no chain declared"}, {"name": "chain_get", "arguments": "", "reply": "resp 0 ,... ,...", "meaning": "read a chain back: its worker (- when it has none), its members in the declared order, and the members its worker runs now in their order (- for none); a quarantined member is declared and not running; -522 for a chain no chain declared, -902 for a chain that is not a number"}, {"name": "descriptor_get", "arguments": "", "reply": "resp 0 \"\"", "meaning": "the scanned descriptor of the plugin, answered from descriptor_dir: the file named for its binary as it is now (@-.json), written by the scan the plugin's add ran. -529 when there is none, -530 while its scan runs, -531 when its last scan wrote nothing, -528 when the directory could not be written"}, - {"name": "panel_get", "arguments": "", "reply": "resp 0 \"\"", "meaning": "the panel the plugin publishes (org.openmixer.panel-description/1), answered from descriptor_dir: the file panels/@-.json, named as its descriptor, which the same scan writes once the descriptor is in place, {\"plugin\": {\"format\", \"id\"}, \"panel\": }; an LV2 plugin's panel is null. The codes are descriptor_get's: -529 when there is none, -530 while its scan runs, -531 when its last scan wrote nothing, -528 when the directory could not be written"} + {"name": "panel_get", "arguments": "", "reply": "resp 0 \"\"", "meaning": "the panel the plugin publishes (org.openmixer.panel-description/1), answered from descriptor_dir: the file panels/@-.json, named as its descriptor, which the same scan writes once the descriptor is in place, {\"plugin\": {\"format\", \"id\"}, \"panel\": }; an LV2 plugin's panel is null. The codes are descriptor_get's: -529 when there is none, -530 while its scan runs, -531 when its last scan wrote nothing, -528 when the directory could not be written"}, + {"name": "history_get", "arguments": "", "reply": "resp 0 ", "meaning": "what the daemon kept of the plugin on this host class, for the binary its pin holds now (the running history: the windows it ran alone and in a group, its crashes, restarts and quarantines): one word of comma-joined digest=, class=, crashed=<0|1>, raises_xruns=<0|1> and then every key=value of the record (data/history.json); - when the plugin has no pin or no record, unavailable while the history file is blocked or history_window_s is 0, -902 for a line that is not one word"}, + {"name": "history_clear", "arguments": " | all", "reply": "resp 0", "meaning": "delete the history of one plugin (every binary and host class of it), or all of it, from memory and from the file at once; -902 while the history is blocked or off, or for a line that is not one word"} ], "instance_verbs": [ {"name": "track_info", "arguments": " <#RRGGBB | -> [bus | return | master]", "reply": "resp 0", "ledger": "replace", "meaning": "the strip's name (\"\" for none), colour and kind, absent for an input channel; a worker whose plugin format has a track-info extension passes them on; any other word is -902"}, @@ -169,7 +171,9 @@ {"name": "chain_clear", "source": "daemon", "levels": {"lv2-worker": "required", "clap-worker": "required", "console": "required", "zynthian": "required"}}, {"name": "chain_get", "source": "daemon", "levels": {"lv2-worker": "required", "clap-worker": "required", "console": "required", "zynthian": "required"}}, {"name": "descriptor_get", "source": "daemon", "levels": {"lv2-worker": "required", "clap-worker": "required", "console": "required", "zynthian": "required"}}, - {"name": "panel_get", "source": "daemon", "levels": {"lv2-worker": "required", "clap-worker": "required", "console": "required", "zynthian": "required"}} + {"name": "panel_get", "source": "daemon", "levels": {"lv2-worker": "required", "clap-worker": "required", "console": "required", "zynthian": "required"}}, + {"name": "history_get", "source": "daemon", "levels": {"lv2-worker": "required", "clap-worker": "required", "console": "required", "zynthian": "required"}}, + {"name": "history_clear", "source": "daemon", "levels": {"lv2-worker": "required", "clap-worker": "required", "console": "required", "zynthian": "required"}} ], "track_kinds": ["bus", "return", "master"], "param_scales": [ diff --git a/src/conf.c b/src/conf.c index d664303..6d34255 100644 --- a/src/conf.c +++ b/src/conf.c @@ -136,6 +136,16 @@ int conf_check(const conf_t *conf) fprintf(stderr, "\n"); return -1; } + if (strpbrk(conf->host_class, " \t")) + { + fprintf(stderr, "host_class '%s' holds white space\n", conf->host_class); + return -1; + } + if (conf->history_window_s < 0 || conf->history_save_s < 0) + { + fprintf(stderr, "history_window_s and history_save_s are not negative\n"); + return -1; + } return 0; } diff --git a/src/main.c b/src/main.c index 6bcb53d..89358f5 100644 --- a/src/main.c +++ b/src/main.c @@ -29,6 +29,8 @@ #include "conf.h" #include "descriptors.h" +#include "history.h" +#include "machine.h" #include "pins.h" #include "relay.h" #include "supervisor.h" @@ -44,6 +46,23 @@ static volatile int running; static conf_t g_conf; +/* the running history, unless history_window_s is 0: under the class of this machine, or the one host_class names */ +static void history_start(const conf_t *conf) +{ + char klass[64]; + + if (conf->history_window_s <= 0) + return; + if (*conf->host_class) + snprintf(klass, sizeof(klass), "%s", conf->host_class); + else if (machine_class(klass, sizeof(klass)) != 0) + { + fprintf(stderr, "plugin-hostd: the host class of this machine does not fit; no history is kept\n"); + return; + } + history_init(conf->history_file, klass, conf->history_save_s); +} + static void idle_cb(void) { sup_tick(); @@ -146,6 +165,7 @@ int main(int argc, char **argv) if (g_conf.require_pins && pins_init(g_conf.pin_store, g_conf.pin_file, g_conf.pins_dir) != 0) return EXIT_FAILURE; + history_start(&g_conf); signal(SIGPIPE, SIG_IGN); sup_init(&g_conf); /* the feedback port is the relay's: the daemon's events and the workers' lines go out on it through one writer */ @@ -175,6 +195,7 @@ int main(int argc, char **argv) socket_finish(); descriptors_finish(); sup_finish(); + history_finish(); relay_finish(); return 0; } diff --git a/src/pins.c b/src/pins.c index f4ec48b..d842676 100644 --- a/src/pins.c +++ b/src/pins.c @@ -584,6 +584,16 @@ int pins_known(const char *uri) return find(bare(uri)) != NULL; } +int pins_digest(const char *uri, char hex[PHD_SHA256_HEX_LEN + 1]) +{ + const pin_t *p = find(bare(uri)); + + if (!p || p->nfiles < 1) + return -1; + memcpy(hex, p->digest[0], PHD_SHA256_HEX_LEN + 1); + return 0; +} + int pins_in_process(const char *uri) { const pin_t *p = find(bare(uri)); diff --git a/src/pins.h b/src/pins.h index b18acba..0f573bf 100644 --- a/src/pins.h +++ b/src/pins.h @@ -21,6 +21,8 @@ #ifndef PINS_H #define PINS_H +#include + #include "formats.h" /* The pins by URI: policy, like placement, and not in the ledger. Two layers. The host's: pin_set, a pin file at start, @@ -47,6 +49,9 @@ int pins_clear(const char *uri); /* whether the uri has a pin, and whether it is one a verdict of in-process qualified */ int pins_known(const char *uri); +/* the SHA-256 of the first file the pin of the uri holds, as the pin holds it: the binary the running history is kept + * under. -1 when the uri has no pin */ +int pins_digest(const char *uri, char hex[PHD_SHA256_HEX_LEN + 1]); int pins_in_process(const char *uri); /* first use: the files the format loads for the uri, hashed, recorded with an unknown layout and written to the store. diff --git a/src/supervisor.c b/src/supervisor.c index 183e069..0642735 100644 --- a/src/supervisor.c +++ b/src/supervisor.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -37,6 +38,7 @@ /* mod-host's pseudo port of the host's own bypass, which param_info names no parameter */ +#include "history.h" #include "pins.h" #include "relay.h" #include "supervisor.h" @@ -675,6 +677,133 @@ static int chain_whole(const chain_t *c) return 1; } +/* ---------------------------------------------------------------- the running history */ + +/* The history is kept by plugin binary: the first file its pin holds. A plugin with no pin (require_pins off, or never + * pinned) has none, and nothing is kept of it. src/history.c does nothing when the history is off or unavailable. */ +static long long utc_now(void) +{ + return (long long)time(NULL); +} + +static void hist_quarantine(const char *uri) +{ + char digest[PHD_SHA256_HEX_LEN + 1]; + + if (pins_digest(uri, digest) == 0) + history_quarantine(uri, digest, utc_now()); +} + +static void hist_crash(const char *uri, int grouped) +{ + char digest[PHD_SHA256_HEX_LEN + 1]; + + if (pins_digest(uri, digest) == 0) + history_crash(uri, digest, grouped, utc_now()); +} + +static void hist_restart(const char *uri, int grouped) +{ + char digest[PHD_SHA256_HEX_LEN + 1]; + + if (pins_digest(uri, digest) == 0) + history_restart(uri, digest, grouped, utc_now()); +} + +/* the window [hist_start_ms, at_ms] of a worker, its CPU over it as percent of one core, folded into every member */ +static void hist_fold(worker_t *w, int64_t at_ms, double cpu1, long long pss_kb) +{ + double seconds = (double)(at_ms - w->hist_start_ms) / 1000.0, pct = 0; + int n, grouped = w->ninst >= 2; + + if (seconds > 0 && cpu1 > w->hist_cpu0) + pct = (cpu1 - w->hist_cpu0) / seconds * 100.0; + for (n = 0; n < w->ninst; n++) + { + char digest[PHD_SHA256_HEX_LEN + 1]; + + if (pins_digest(w->inst[n]->uri, digest) == 0) + history_window(w->inst[n]->uri, digest, grouped, pct, (double)pss_kb / 1024.0, 0, utc_now()); + } +} + +static void hist_begin(worker_t *w, int64_t now) +{ + w->hist_pid = 0; + if (proc_cpu_seconds(w->pid, &w->hist_cpu0) != 0) + return; + w->hist_pid = w->pid; + w->hist_start_ms = now; + w->hist_ninst = w->ninst; + w->hist_half_ms = 0; +} + +/* the idle tick of one worker: a window opens when it is up, takes its mid-window sample, and closes after + * history_window_s; members that came or went open it again, uncounted */ +static void hist_tick(worker_t *w, int64_t now) +{ + int64_t window_ms = (int64_t)g_conf.history_window_s * 1000; + double cpu; + long long pss = 0; + + if (g_conf.history_window_s <= 0) + return; + if (w->state != W_UP || w->pid <= 0) + { + w->hist_pid = 0; + return; + } + if (w->hist_pid != w->pid || w->hist_ninst != w->ninst) + { + hist_begin(w, now); + return; + } + if (!w->hist_half_ms && now - w->hist_start_ms >= window_ms / 2) + { + if (proc_cpu_seconds(w->pid, &w->hist_half_cpu) != 0) + { + w->hist_pid = 0; + return; + } + proc_pss_kb(w->pid, &w->hist_half_pss_kb); + w->hist_half_ms = now; + } + if (now - w->hist_start_ms < window_ms) + return; + if (proc_cpu_seconds(w->pid, &cpu) != 0) + { + w->hist_pid = 0; + return; + } + proc_pss_kb(w->pid, &pss); + if (w->hist_half_pss_kb > pss) + pss = w->hist_half_pss_kb; + hist_fold(w, now, cpu, pss); + w->hist_half_pss_kb = 0; + hist_begin(w, now); +} + +/* a worker that died: its window counts if it was up for half of it. The dead process is not yet reaped, so its CPU is + * still readable; when it is not, the mid-window sample stands for the window */ +static void hist_death(worker_t *w, int64_t now) +{ + double cpu; + long long pss = w->hist_half_pss_kb; + + if (g_conf.history_window_s > 0 && w->hist_pid && w->hist_pid == w->pid) + { + int64_t window_ms = (int64_t)g_conf.history_window_s * 1000; + + if (now - w->hist_start_ms >= window_ms / 2 && proc_cpu_seconds(w->pid, &cpu) == 0) + hist_fold(w, now, cpu, pss); + else if (w->hist_half_ms) + hist_fold(w, w->hist_half_ms, w->hist_half_cpu, pss); + } + w->hist_pid = 0; + w->hist_half_ms = 0; + w->hist_half_pss_kb = 0; +} + static void quarantine(instance_t *i, worker_t *w) { meta_t *m = meta_of(i->id); @@ -683,6 +812,7 @@ static void quarantine(instance_t *i, worker_t *w) { m->quarantined = 1; event(PHD_EVENT_INSTANCE_QUARANTINED_FMT, i->id, w->k); + hist_quarantine(i->uri); } /* the culprit of a chain that was split: the others are gathered again, without it, by the tick */ if (i->chain && !chain_whole(i->chain)) @@ -743,10 +873,15 @@ static void clear_suspects(worker_t *w) static void on_death(worker_t *w, int in_flight) { - int status = 0, n, done = 0; + int status = 0, n, done = 0, grouped = w->ninst >= 2, nmembers = w->ninst; char ids[1024], how[32]; int64_t now = proc_now_ms(); + instance_t **members = NULL, *blamed = NULL; + /* the history: the window it was in, and who was in the worker; before the process is reaped */ + hist_death(w, now); + if (grouped && (members = malloc(nmembers * sizeof(instance_t *)))) + memcpy(members, w->inst, nmembers * sizeof(instance_t *)); for (n = 0; n < 50 && !(done = proc_exited(w->pid, &status)); n++) usleep(10000); if (!done) @@ -782,6 +917,7 @@ static void on_death(worker_t *w, int in_flight) if (w->ninst == 1) { meta_of(w->inst[0]->id)->crashes++; + hist_crash(w->inst[0]->uri, 0); if (w->from_split) quarantine(w->inst[0], w); } @@ -798,6 +934,8 @@ static void on_death(worker_t *w, int in_flight) if (culprit) { meta_of(culprit->id)->crashes++; + hist_crash(culprit->uri, w->ninst >= 2); + blamed = culprit; quarantine(culprit, w); move_to_own(culprit, now, 0); } @@ -808,6 +946,11 @@ static void on_death(worker_t *w, int in_flight) } } } + /* the members of a group that went down with it and were not the culprit */ + for (n = 0; members && n < nmembers; n++) + if (members[n] != blamed) + hist_restart(members[n]->uri, 1); + free(members); if (w->ninst == 0) { worker_free(w); @@ -1156,6 +1299,9 @@ void sup_tick(void) if (due) checkpoint(w); } + for (n = 0; n < g_nworkers; n++) + hist_tick(g_workers[n], now); + history_tick(now / 1000); for (n = 0; g_nchains && n < MAX_INSTANCE; n++) if (g_chain[n] && g_chain[n]->rejoin) { @@ -1448,10 +1594,13 @@ static char *add(const char *uri, int id, const char *placement, const char *cli event(PHD_EVENT_VERB_DROPPED_FMT, id, line); m->crashes++; note_death(m->deaths, &m->ndeaths, now); + /* the plugin it was adding is not in the worker yet: it joins the members that are */ + hist_crash(fwd, w->ninst >= 1); if (w->pool) { m->quarantined = 1; event(PHD_EVENT_INSTANCE_QUARANTINED_FMT, id, w->k); + hist_quarantine(fwd); } on_death(w, ATTRIBUTED); return sup_resp(code); @@ -1900,6 +2049,32 @@ char *sup_instance_info(int id) return strdup(buffer); } +char *sup_history_get(const char *uri) +{ + char digest[PHD_SHA256_HEX_LEN + 1], *summary, *text; + const char *bare_uri; + + format_of_uri(uri, &bare_uri); + if (!history_available()) + return strdup("resp 0 " PHD_WORD_UNAVAILABLE); + if (pins_digest(bare_uri, digest) != 0 || !(summary = history_summary(bare_uri, digest, utc_now()))) + return strdup("resp 0 " PHD_WORD_NONE); + text = malloc(strlen(summary) + 8); + if (text) + sprintf(text, "resp 0 %s", summary); + free(summary); + return text ? text : sup_resp(ERR_MEMORY_ALLOCATION); +} + +char *sup_history_clear(const char *uri) +{ + const char *bare_uri = uri; + + if (strcmp(uri, PHD_WORD_ALL)) + format_of_uri(uri, &bare_uri); + return sup_resp(history_clear(bare_uri) == 0 ? SUCCESS : ERR_INVALID_OPERATION); +} + char *sup_reset(const char *which) { int n, all = !which || !*which || !strcmp(which, PHD_WORD_ALL), found = 0, id; diff --git a/src/supervisor.h b/src/supervisor.h index 4ef297d..c7a1d23 100644 --- a/src/supervisor.h +++ b/src/supervisor.h @@ -95,6 +95,14 @@ typedef struct WORKER_T { int npend; int64_t pend_until_ms; struct CHAIN_T *chain; /* the chain this worker runs, its members in w->inst in the chain's order */ + /* the running history's window of this worker (src/history.c): what it has used since hist_start_ms */ + pid_t hist_pid; /* the process the window belongs to, 0 for none */ + int64_t hist_start_ms; + double hist_cpu0; /* its CPU seconds when the window began */ + int hist_ninst; /* its members then: a window in which they changed is not counted */ + int64_t hist_half_ms; /* the mid-window sample, 0 before it: what a window that ends in a death is counted from */ + double hist_half_cpu; + long long hist_half_pss_kb; } worker_t; /* the instances a chain verb declared, in order; the ones not quarantined run in its worker */ @@ -147,6 +155,11 @@ char *sup_chain_clear(int id); /* chain_get: the chain's worker, its members as declared and the ones its worker runs, each in order */ char *sup_chain_get(int id); +/* history_get: the summary of the binary the uri's pin holds, on this host class (src/history.c); history_clear: the + * lines of one uri or all of them. Both answer as the verbs do */ +char *sup_history_get(const char *uri); +char *sup_history_clear(const char *uri); + /* the idle tick: reap the dead, respawn the due, checkpoint the quiet */ void sup_tick(void); diff --git a/src/verbs.c b/src/verbs.c index 4dc630a..c8c06ee 100644 --- a/src/verbs.c +++ b/src/verbs.c @@ -201,6 +201,10 @@ static char *handle(char *line, char **tok, int ntok) lv2_refusals_forget(NULL); return sup_resp(rc); } + if (!strcmp(verb, OMX_VERB_HISTORY_GET)) + return ntok == 2 ? sup_history_get(tok[1]) : sup_resp(ERR_INVALID_OPERATION); + if (!strcmp(verb, OMX_VERB_HISTORY_CLEAR)) + return ntok == 2 ? sup_history_clear(tok[1]) : sup_resp(ERR_INVALID_OPERATION); if (verb_is(verb, QUIT)) { if (ntok != 1) diff --git a/tests/daemon_test.py b/tests/daemon_test.py index 815b159..7602cfe 100644 --- a/tests/daemon_test.py +++ b/tests/daemon_test.py @@ -3238,4 +3238,301 @@ def a_missing_key_falls_to_the_documented_default_and_the_key_holds_over_the_var s.close() +# ---------------------------------------------------------------- the running history (src/history.c) + +HISTORY_CLASS = "testarch/1-2/<1.5GHz" + + +def history_conf(p, **kw): + """a daemon that keeps its history in the test's own file, under a class of the test's own, a window of one second""" + c = {"pin_mode": "strict", "history_file": os.path.join(p.dir, "history"), "history_window_s": 1, + "history_save_s": 1, "host_class": HISTORY_CLASS} + c.update(kw) + return c + + +def pin_in_process(p, d, uri=None, clap=None): + """the pin of the CLAP (or of another) with the verdict that lets it share a worker""" + d.expect("pin_set %s verdict=in-process" % pin_line(p, uri=uri, clap=clap), "resp 0") + + +def history_of(d, uri): + """history_get's one word as a dict, or the word itself ('-', 'unavailable')""" + r = d.send("history_get %s" % uri) + check(r.startswith("resp 0 "), "history_get answers resp 0: %r" % r) + word = r[len("resp 0 "):] + if word in ("-", "unavailable"): + return word + return dict(kv.split("=", 1) for kv in word.split(",")) + + +def history_lines(path): + """(uri, digest, class, {key: value}) of every history line of the file""" + if not os.path.exists(path): + return [] + out = [] + with open(path) as f: + for l in f: + w = l.split() + if w and w[0] == "history": + out.append((w[1], w[2], w[3], dict(kv.split("=", 1) for kv in w[4:]))) + return out + + +@test +def a_pinned_plugin_leaves_a_record_of_its_windows_alone_and_in_a_group(): + t0 = int(time.time()) + p = Plugins() + d = p.daemon(conf=history_conf(p)) + try: + pin_in_process(p, d) + d.expect("add %s 0 own" % p.clap_uri, "resp 0") + h = wait_for(lambda: (lambda r: r if isinstance(r, dict) and int(r["windows_solo"]) >= 1 else None)(history_of(d, p.clap_uri)), + "a window of the plugin alone") + check(h["digest"] == sha(p.clap), "the record is of the binary its pin holds: %s" % h["digest"]) + check(h["class"] == HISTORY_CLASS, "under the class the setting names: %s" % h["class"]) + check(h["windows_grouped"] == "0" and h["crashed"] == "0" and h["raises_xruns"] == "0", "%s" % h) + check(0 < float(h["pss_mb_solo"]) < 100, "the worker's Pss was read, in MiB: %s" % h["pss_mb_solo"]) + check(0 <= float(h["cpu_p95_solo"]) <= float(h["cpu_max_solo"]), "cpu p95 %s, max %s" % (h["cpu_p95_solo"], h["cpu_max_solo"])) + check(t0 <= int(h["first"]) <= int(h["last"]), "first %s, last %s, since %d" % (h["first"], h["last"], t0)) + check(history_of(d, "clap:/nowhere.clap#x") == "-", "a plugin with no pin has no history") + # the file, written within history_save_s of a change + lines = wait_for(lambda: history_lines(os.path.join(p.dir, "history")) or None, "the history file") + check(len(lines) == 1 and lines[0][0] == p.clap_uri and lines[0][1] == sha(p.clap) and lines[0][2] == HISTORY_CLASS, + "the file holds the record under its key: %s" % lines) + st = os.stat(os.path.join(p.dir, "history")) + check(st.st_mode & 0o777 == 0o600, "mode %o" % (st.st_mode & 0o777)) + # a group: the same plugin twice in a pool; the windows of the group are counted apart + d.expect("remove 0", "resp 0") + solo = int(history_of(d, p.clap_uri)["windows_solo"]) + d.expect("add %s 1 pool:g" % p.clap_uri, "resp 1") + d.expect("add %s 2 pool:g" % p.clap_uri, "resp 2") + h = wait_for(lambda: (lambda r: r if int(r["windows_grouped"]) >= 1 else None)(history_of(d, p.clap_uri)), "a window of the group") + check(int(h["windows_solo"]) == solo, "the group's windows did not move the solo ones: %d then %s" % (solo, h["windows_solo"])) + finally: + d.close() + p.close() + + +@test +def crashes_restarts_and_quarantines_are_counted_for_the_plugin_that_earned_them(): + p = Plugins() + other, other_uri = other_clap(p) + d = p.daemon(conf=history_conf(p, history_window_s=3600)) + try: + pin_in_process(p, d) + pin_in_process(p, d, other_uri, other) + d.expect("add %s 0 pool:g" % p.clap_uri, "resp 0") + d.expect("add %s 1 pool:g" % other_uri, "resp 1") + a, b = history_of(d, p.clap_uri), history_of(d, other_uri) + check(a == "-" and b == "-", "nothing has happened yet, so there is no record: %s %s" % (a, b)) + mark = d.mark() + # the second dies just after a reply, in a pool of two: it is the culprit, the first only went down with it + d.expect("param_set 1 crash 1", "resp 0") + d.wait_event("instance_quarantined 1 ", since=mark) + d.wait_up(0) + a, b = history_of(d, p.clap_uri), history_of(d, other_uri) + check((a["crashes_grouped"], a["restarts_grouped"], a["quarantines"], a["crashes_solo"]) == ("0", "1", "0", "0"), + "the member that was not the culprit went down with the group: %s" % a) + check((b["crashes_grouped"], b["restarts_grouped"], b["quarantines"], b["crashes_solo"]) == ("1", "0", "1", "0"), + "the culprit crashed in a group and was quarantined: %s" % b) + check(int(b["last_crash_grouped"]) > 1600000000 and b["last_crash"] == "0", "when: %s" % b) + check(b["crashed"] == "1" and a["crashed"] == "0", "only the culprit is marked crashed: %s %s" % (a["crashed"], b["crashed"])) + # the culprit is alone now: its next crash is a crash alone + mark = d.mark() + d.wait_up(1) + d.expect("param_set 1 crash 1", "resp 0") + d.wait_event("instance_restored 1 ", since=mark) + b = history_of(d, other_uri) + check((b["crashes_solo"], b["crashes_grouped"], b["restarts_grouped"]) == ("1", "1", "0"), "a crash alone: %s" % b) + a = history_of(d, p.clap_uri) + check(a["restarts_grouped"] == "1" and a["crashes_solo"] == "0", "the other plugin is not touched by it: %s" % a) + finally: + d.close() + p.close() + + +@test +def a_worker_that_dies_after_half_a_window_still_counts_it_and_before_half_does_not(): + p = Plugins() + d = p.daemon(conf=history_conf(p, history_window_s=4, history_save_s=3600)) + try: + pin_in_process(p, d) + d.expect("add %s 0 own" % p.clap_uri, "resp 0") + mark = d.mark() + d.expect("param_set 0 crash 1", "resp 0") + d.wait_event("instance_restored 0 ", since=mark) + h = history_of(d, p.clap_uri) + check(h["crashes_solo"] == "1" and h["windows_solo"] == "0", "up for less than half a window: not counted: %s" % h) + # half of four seconds is two: the new worker is up, and a window of it is open; let it pass half + time.sleep(2.6) + mark = d.mark() + d.expect("param_set 0 crash 1", "resp 0") + d.wait_event("instance_restored 0 ", since=mark) + h = history_of(d, p.clap_uri) + check(h["crashes_solo"] == "2" and h["windows_solo"] == "1", "up for more than half a window: counted: %s" % h) + finally: + d.close() + p.close() + + +@test +def history_clear_deletes_one_plugin_or_all_of_it_and_the_file_follows_at_once(): + p = Plugins() + other, other_uri = other_clap(p) + d = p.daemon(conf=history_conf(p, history_save_s=3600)) + path = os.path.join(p.dir, "history") + try: + for uri, clap in ((p.clap_uri, None), (other_uri, other)): + pin_in_process(p, d, uri if clap else None, clap) + d.expect("add %s %d own" % (uri, 0 if not clap else 1), "resp %d" % (0 if not clap else 1)) + wait_for(lambda: all(isinstance(history_of(d, u), dict) for u in (p.clap_uri, other_uri)), "both have a window") + d.expect("history_clear %s" % p.clap_uri, "resp 0") + check(history_of(d, p.clap_uri) == "-", "the plugin has no history") + check(isinstance(history_of(d, other_uri), dict), "and the other one is untouched") + check([l[0] for l in history_lines(path)] == [other_uri], "the file lost the line at once: %s" % history_lines(path)) + d.expect("history_clear all", "resp 0") + check(history_lines(path) == [], "all of it: %s" % history_lines(path)) + d.expect("history_clear", "resp -902") + d.expect("history_clear a b", "resp -902") + d.expect("history_get", "resp -902") + finally: + d.close() + p.close() + + +@test +def a_history_file_of_another_version_is_neither_read_nor_overwritten_and_the_daemon_runs_on(): + p = Plugins() + path = os.path.join(p.dir, "history") + body = "# plugin-hostd history/2\nhistory %s %s %s windows_solo=7\n" % (p.clap_uri, sha(p.clap), HISTORY_CLASS) + write(path, body.encode()) + d = p.daemon(conf=history_conf(p)) + try: + pin_in_process(p, d) + d.expect("add %s 0 own" % p.clap_uri, "resp 0") + check(history_of(d, p.clap_uri) == "unavailable", "the history is unavailable") + d.expect("history_clear all", "resp -902") + check("has a version this daemon does not know" in d.log(), "stderr says why: %s" % d.log()) + check(d.holder(0)[1]["state"] == "up", "the plugin runs") + finally: + d.close() + with open(path) as f: + check(f.read() == body, "the file is as it was") + p.close() + + +@test +def with_history_window_s_zero_nothing_is_kept_and_a_plugin_runs_as_before(): + p = Plugins() + d = p.daemon(conf=history_conf(p, history_window_s=0)) + try: + pin_in_process(p, d) + d.expect("add %s 0 own" % p.clap_uri, "resp 0") + check(history_of(d, p.clap_uri) == "unavailable", "no history") + d.expect("param_set 0 crash 1", "resp 0") + d.wait_up(0) + check(not os.path.exists(os.path.join(p.dir, "history")), "no file") + finally: + d.close() + p.close() + + +@test +def the_history_is_written_when_the_daemon_quits_and_found_again_by_the_next_one(): + p = Plugins() + path = os.path.join(p.dir, "history") + d = p.daemon(conf=history_conf(p, history_save_s=3600)) + try: + pin_in_process(p, d) + d.expect("add %s 0 own" % p.clap_uri, "resp 0") + wait_for(lambda: isinstance(history_of(d, p.clap_uri), dict) or None, "a window") + check(not os.path.exists(path), "an hour between saves: nothing written yet") + d.expect("quit", "resp 0") + wait_for(lambda: d.proc.poll() is not None, "the daemon to exit") + finally: + d.close() + lines = history_lines(path) + check(len(lines) == 1 and int(lines[0][3]["windows_solo"]) >= 1, "the quit wrote the record: %s" % lines) + first = lines[0][3]["first"] + d = p.daemon(conf=history_conf(p, history_save_s=3600)) + try: + pin_in_process(p, d) + h = history_of(d, p.clap_uri) + check(isinstance(h, dict) and h["first"] == first and h["windows_solo"] == lines[0][3]["windows_solo"], + "the next daemon reads what the last wrote: %s" % h) + finally: + d.close() + p.close() + + +@test +def the_history_goes_to_the_xdg_state_home_by_default_and_to_the_machines_class_unless_host_class_says_otherwise(): + p = Plugins() + state = os.path.join(p.dir, "xdg") + d = p.daemon(conf=history_conf(p, history_file=None, host_class=None, history_save_s=3600), env={"XDG_STATE_HOME": state}) + try: + pin_in_process(p, d) + d.expect("add %s 0 own" % p.clap_uri, "resp 0") + h = wait_for(lambda: (lambda r: r if isinstance(r, dict) else None)(history_of(d, p.clap_uri)), "a window") + arch = subprocess.check_output(["uname", "-m"], text=True).strip() + cls = h["class"].split("/") + check(cls[0] == arch and len(cls) == 3, "arch/cores/clock of this machine: %s" % h["class"]) + d.expect("quit", "resp 0") + wait_for(lambda: d.proc.poll() is not None, "the daemon to exit") + finally: + d.close() + lines = history_lines(os.path.join(state, "plugin-hostd", "history")) + check(len(lines) == 1 and lines[0][2] == h["class"], "$XDG_STATE_HOME/plugin-hostd/history: %s" % lines) + p.close() + bad = Plugins() + try: + d = bad.daemon(conf=history_conf(bad, host_class="two words")) + except Fail: + d = None + check(d is None, "a host class with white space stops the daemon") + bad.close() + + +@test +def a_window_in_which_the_members_of_a_worker_changed_is_not_counted(): + p = Plugins() + other, other_uri = other_clap(p) + d = p.daemon(conf=history_conf(p, history_window_s=4, history_save_s=3600)) + try: + pin_in_process(p, d) + pin_in_process(p, d, other_uri, other) + d.expect("add %s 0 pool:g" % p.clap_uri, "resp 0") + t0 = time.monotonic() + time.sleep(2.4) + d.expect("add %s 1 pool:g" % other_uri, "resp 1") + # the window that began with one member would have closed at four seconds with two: it was dropped, and a new one + # began when the second came, to close at six + time.sleep(max(0, 4.8 - (time.monotonic() - t0))) + check(history_of(d, p.clap_uri) == "-" and history_of(d, other_uri) == "-", + "no window closed on a pool whose members changed: %s %s" % (history_of(d, p.clap_uri), history_of(d, other_uri))) + h = wait_for(lambda: (lambda r: r if isinstance(r, dict) else None)(history_of(d, p.clap_uri)), "the window of the two") + check(h["windows_grouped"] == "1" and h["windows_solo"] == "0", "it is a window of a group: %s" % h) + finally: + d.close() + p.close() + + +@test +def an_lv2_plugins_record_is_under_the_first_file_its_pin_holds(): + p = Plugins() + d = p.daemon(conf=history_conf(p, lv2_worker="mod-host")) + try: + d.expect(p.lv2_pin(), "resp 0") + d.expect("add lv2:%s 0 own" % LV2_URI, "resp 0") + h = wait_for(lambda: (lambda r: r if isinstance(r, dict) else None)(history_of(d, "lv2:" + LV2_URI)), "a window") + check(h["digest"] == sha(os.path.join(p.bundle, "manifest.ttl")), "the first file of the pin, manifest.ttl: %s" % h["digest"]) + check(history_of(d, LV2_URI)["digest"] == h["digest"], "with or without the lv2: prefix") + d.expect("history_clear lv2:%s" % LV2_URI, "resp 0") + check(history_of(d, LV2_URI) == "-", "and cleared by either") + finally: + d.close() + p.close() + + sys.exit(run_tests(TESTS)) diff --git a/tests/harness.py b/tests/harness.py index dd0e853..8319af0 100644 --- a/tests/harness.py +++ b/tests/harness.py @@ -91,6 +91,8 @@ def __init__(self, exe, worker, clap_worker=None, conf=None, env=None, feedback= "descriptor_dir": os.path.join(self.tmp, "descriptors"), "qualification_dir": os.path.join(self.tmp, "qualification"), "probe": os.path.join(self.tmp, "no-probe"), + # the running history is the test's own, never the user's + "history_file": os.path.join(self.tmp, "history"), } settings.update(conf or {}) self.conf = os.path.join(self.tmp, "plugin-hostd.conf") diff --git a/tests/sabotage.py b/tests/sabotage.py index 3f13004..152197e 100644 --- a/tests/sabotage.py +++ b/tests/sabotage.py @@ -900,6 +900,86 @@ 'if (g_saved_at < 0)\n g_saved_at = now;', 'if (g_saved_at < 0)\n g_saved_at = now - g_save_s;', 'the_history_is_saved_at_most_once_per_interval_and_at_quit', "history"), + ('a plugin alone in a worker is counted as a group', 'src/supervisor.c', + 'int n, grouped = w->ninst >= 2;', + 'int n, grouped = w->ninst >= 1;', + 'a_pinned_plugin_leaves_a_record_of_its_windows_alone_and_in_a_group'), + ('a window never closes', 'src/supervisor.c', + ' if (now - w->hist_start_ms < window_ms)\n return;', + ' if (1)\n return;', + 'a_pinned_plugin_leaves_a_record_of_its_windows_alone_and_in_a_group'), + ('the Pss is taken for MiB when it is kB', 'src/supervisor.c', + '(double)pss_kb / 1024.0', + '(double)pss_kb', + 'a_pinned_plugin_leaves_a_record_of_its_windows_alone_and_in_a_group'), + ('a file is written at no save', 'src/supervisor.c', + ' history_tick(now / 1000);\n', + '', + 'a_pinned_plugin_leaves_a_record_of_its_windows_alone_and_in_a_group'), + ('the history is not written at quit', 'src/main.c', + ' sup_finish();\n history_finish();', + ' sup_finish();', + 'the_history_is_written_when_the_daemon_quits_and_found_again_by_the_next_one'), + ('a crash of a worker of its own is counted as a crash of a group', 'src/supervisor.c', + 'hist_crash(w->inst[0]->uri, 0);', + 'hist_crash(w->inst[0]->uri, 1);', + 'crashes_restarts_and_quarantines_are_counted_for_the_plugin_that_earned_them'), + ('the culprit of a pool is counted as a crash alone', 'src/supervisor.c', + 'hist_crash(culprit->uri, w->ninst >= 2);', + 'hist_crash(culprit->uri, 0);', + 'crashes_restarts_and_quarantines_are_counted_for_the_plugin_that_earned_them'), + ('the culprit is also counted as a restart', 'src/supervisor.c', + ' if (members[n] != blamed)', + ' if (1)', + 'crashes_restarts_and_quarantines_are_counted_for_the_plugin_that_earned_them'), + ('the members of a group that went down are not counted', 'src/supervisor.c', + ' hist_restart(members[n]->uri, 1);', + ' ;', + 'crashes_restarts_and_quarantines_are_counted_for_the_plugin_that_earned_them'), + ('a quarantine is not counted', 'src/supervisor.c', + ' hist_quarantine(i->uri);\n }\n /* the culprit of a chain', + ' }\n /* the culprit of a chain', + 'crashes_restarts_and_quarantines_are_counted_for_the_plugin_that_earned_them'), + ('a window of less than half counts at a death', 'src/supervisor.c', + 'if (now - w->hist_start_ms >= window_ms / 2 && proc_cpu_seconds(w->pid, &cpu) == 0)', + 'if (proc_cpu_seconds(w->pid, &cpu) == 0)', + 'a_worker_that_dies_after_half_a_window_still_counts_it_and_before_half_does_not'), + ('a death does not close the window it ended', 'src/supervisor.c', + ' hist_death(w, now);\n if (grouped &&', + ' if (grouped &&', + 'a_worker_that_dies_after_half_a_window_still_counts_it_and_before_half_does_not'), + ('a window in which the members changed is counted', 'src/supervisor.c', + 'if (w->hist_pid != w->pid || w->hist_ninst != w->ninst)', + 'if (w->hist_pid != w->pid)', + 'a_window_in_which_the_members_of_a_worker_changed_is_not_counted'), + ('history_window_s 0 keeps a history', 'src/main.c', + ' if (conf->history_window_s <= 0)\n return;\n if (*conf->host_class)', + ' if (*conf->host_class)', + 'with_history_window_s_zero_nothing_is_kept_and_a_plugin_runs_as_before'), + ('a blocked history answers history_clear', 'src/supervisor.c', + 'history_clear(bare_uri) == 0 ? SUCCESS : ERR_INVALID_OPERATION', + 'SUCCESS', + 'a_history_file_of_another_version_is_neither_read_nor_overwritten_and_the_daemon_runs_on'), + ('a blocked history does not say so', 'src/supervisor.c', + ' if (!history_available())\n return strdup("resp 0 " PHD_WORD_UNAVAILABLE);\n', + '', + 'a_history_file_of_another_version_is_neither_read_nor_overwritten_and_the_daemon_runs_on'), + ('the history is kept under the last file of the pin', 'src/pins.c', + 'memcpy(hex, p->digest[0], PHD_SHA256_HEX_LEN + 1);', + 'memcpy(hex, p->digest[p->nfiles - 1], PHD_SHA256_HEX_LEN + 1);', + 'an_lv2_plugins_record_is_under_the_first_file_its_pin_holds'), + ('the lv2 prefix is part of the uri history_get is asked for', 'src/supervisor.c', + ' format_of_uri(uri, &bare_uri);\n if (!history_available())', + ' bare_uri = uri;\n if (!history_available())', + 'an_lv2_plugins_record_is_under_the_first_file_its_pin_holds'), + ('a host class with white space is taken', 'src/conf.c', + 'if (strpbrk(conf->host_class, " \\t"))', + 'if (0)', + 'the_history_goes_to_the_xdg_state_home_by_default_and_to_the_machines_class_unless_host_class_says_otherwise'), + ('history_clear of a uri does not strip the lv2 prefix', 'src/supervisor.c', + ' if (strcmp(uri, PHD_WORD_ALL))\n format_of_uri(uri, &bare_uri);\n', + '', + 'an_lv2_plugins_record_is_under_the_first_file_its_pin_holds'), ] diff --git a/tests/verbs_declared.py b/tests/verbs_declared.py index 644452c..387b791 100644 --- a/tests/verbs_declared.py +++ b/tests/verbs_declared.py @@ -202,6 +202,8 @@ def code(reply): "chain": ("7 1 2", "7", ("8 99", "8 1 1")), "chain_get": ("7", "x", ("9",)), "chain_clear": ("7", "x", ("7",)), + "history_get": ("{uri}", "", ()), + "history_clear": ("clap:/nowhere.clap#x", "", ()), "pin_clear": ("all", "", ()), } ORDER = list(LINES) From 026360d5b7dea4014ab2761328888f0704666db6 Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Tue, 27 Oct 2026 09:35:29 +0100 Subject: [PATCH 08/16] Expect history.json in the contract package's file list, and cite the design note by name The contract package now carries history.json, so the RPM and DEB checks that list its files must name it. The private-refs check refuses a path into docs/design, so the four citations of the grouping-planner note say its name instead of its path. --- .github/workflows/ci.yml | 3 ++- Makefile | 2 +- data/history.json | 2 +- include/plugin-hostd/protocol.h | 2 +- src/history.h | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e46c640..52d56ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -235,6 +235,7 @@ jobs: printf '%s\n' /usr/share/omx-contract/contract.d/3/plugin-hostd/MANIFEST.json \ /usr/share/omx-contract/contract.d/3/plugin-hostd/formats/clap.json \ /usr/share/omx-contract/contract.d/3/plugin-hostd/formats/lv2.json \ + /usr/share/omx-contract/contract.d/3/plugin-hostd/history.json \ /usr/share/omx-contract/contract.d/3/plugin-hostd/verbs.json \ /usr/share/licenses/plugin-hostd-contract/COPYING | sort > expected.txt diff -u expected.txt files.txt @@ -407,7 +408,7 @@ jobs: # the contract directory's files are the noarch contract package's, which the daemon and the headers depend on test "$(dpkg-deb -f debs/plugin-hostd-contract_*.deb Architecture)" = all dpkg -L plugin-hostd-contract | tee contract.txt - for f in MANIFEST.json formats/clap.json formats/lv2.json verbs.json; do + for f in MANIFEST.json formats/clap.json formats/lv2.json history.json verbs.json; do grep -qx "/usr/share/omx-contract/contract.d/3/plugin-hostd/$f" contract.txt done dpkg-deb -f debs/plugin-hostd-contract_*.deb Breaks Replaces | tee breaks.txt diff --git a/Makefile b/Makefile index 2987db0..7767192 100644 --- a/Makefile +++ b/Makefile @@ -312,7 +312,7 @@ test-pin: $(PIN_TEST) $(PIN_TEST): tests/pin_test.c include/plugin-hostd/pin.h $(CC) -Iinclude $(CFLAGS) -Werror -o $@ tests/pin_test.c -# the running history (src/history.c, docs/design/notes/2026-10-27-grouping-planner.md): its line grammar, its file, what its +# the running history (src/history.c, the grouping-planner design note): its line grammar, its file, what its # counts and its summary come to, against records written out by hand; and the host class it is kept under (src/machine.c) test-history: $(HISTORY_TEST) ./$(HISTORY_TEST) diff --git a/data/history.json b/data/history.json index e6514dc..33a0775 100644 --- a/data/history.json +++ b/data/history.json @@ -6,7 +6,7 @@ "PLUGIN_HOSTD_HISTORY_KEYS": { "kind": "words", "since": "0.13.0", - "doc": "The keys of a line of the running-history file, in the order the daemon writes them (docs/design/notes/2026-10-27-grouping-planner.md, The history record): history key=value ..., after the header line '# plugin-hostd history/1'. The value of every key is a whole number or a decimal, never negative. A line with a key outside these words is kept as it is by every save and never read.", + "doc": "The keys of a line of the running-history file, in the order the daemon writes them (the grouping-planner design note, The history record): history key=value ..., after the header line '# plugin-hostd history/1'. The value of every key is a whole number or a decimal, never negative. A line with a key outside these words is kept as it is by every save and never read.", "words": { "first": { "word": "first", diff --git a/include/plugin-hostd/protocol.h b/include/plugin-hostd/protocol.h index 5b4ed3c..ce46cd2 100644 --- a/include/plugin-hostd/protocol.h +++ b/include/plugin-hostd/protocol.h @@ -987,7 +987,7 @@ static inline int phd_is_note_verb(const char *verb) #define PHD_DEFAULT_HISTORY_WINDOW_S 60 #define PHD_DEFAULT_HISTORY_SAVE_S 300 -/* The running history (docs/design/notes/2026-10-27-grouping-planner.md, The history record): what each plugin did on +/* The running history (the grouping-planner design note, The history record): what each plugin did on * this machine, one line per in a file of the daemon's own, whose keys are the words * of PLUGIN_HOSTD_HISTORY_KEYS (data/history.json). The first line is PHD_HISTORY_HEADER; the file is local, mode 0600, * and only the daemon writes it. The summary history_get answers reads crashes within PHD_HISTORY_KEEP_DAYS of diff --git a/src/history.h b/src/history.h index 78be21d..33967a5 100644 --- a/src/history.h +++ b/src/history.h @@ -26,7 +26,7 @@ #include /* The running history: what each plugin did on this machine, per plugin, per binary and per host class - * (docs/design/notes/2026-10-27-grouping-planner.md, The history record). One line per : history key=value ...; the keys are PLUGIN_HOSTD_HISTORY_KEYS (data/history.json), the * first line PHD_HISTORY_HEADER. Written only by the daemon, atomically, mode 0600; local to the machine. * From f6996e4978f60f9c4e77aa3a0a7d327c40f01ecf Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Tue, 27 Oct 2026 10:11:29 +0100 Subject: [PATCH 09/16] README: grouped workers against one process per plugin, measured The numbers of the page at openmixer.org/docs/grouped-vs-processes, with its caveats and what was not measured. Refs #182 --- README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.md b/README.md index 94d3162..5c0436a 100644 --- a/README.md +++ b/README.md @@ -156,6 +156,32 @@ A token after the instance that is not a placement is the fork's optional jack client name, passed on. An instance that crashed in a pool is placed `own` whatever the hint until `quarantine_clear`. +Grouped workers against one process per plugin +---------------------------------------------- + +A chain runs its plugins in one worker; the other way is one process per plugin. One measurement of the two, with +16 plugins, with the numbers of the page at : + +| | grouped | one process per plugin | | +|---|---|---|---| +| resident memory (RSS), 48 kHz | 125.4 MB | 1764.6 MB | 14x smaller grouped | +| proportional memory (PSS), 48 kHz | 42.9 MB | 118.2 MB | 2.8x smaller grouped | + +- RSS counts a shared library once for every process that maps it, so a design with many processes looks larger than + it is. PSS divides shared pages between the processes that use them, so it is the honest figure. Do not read the RSS + ratio as the saving in real memory. +- CPU: 1.2 to 1.9x less when grouped, across 3, 8 and 16 plugins at 48 and 96 kHz. +- Context switches: up to about 14x fewer when grouped, 13.9x at 48 kHz and 14.8x at 96 kHz with 16 plugins. +- Xruns over the 12 runs of each mode: 10 grouped against 41 with one process per plugin. Each run is one window of + 10 seconds and the xruns in it are few and unevenly spread, so the 10 against 41 is suggestive and not proof. + +How: a Raspberry Pi 5 with 4 cores, CPU governor on performance during the runs, plugin-hostd 0.12.0 with +omx-clap-host 0.22.1 as the worker, N omx plugins in series, 256 frames, on a private JACK server. + +Not measured: only 256 frames; only plugin-hostd, with omx plugins; the grouping planner is not part of the result and +no default has been changed because of the numbers; DSP load and xruns are those of the JACK dummy driver, so no claim +about real audio hardware is made. + Verbs the daemon adds --------------------- From 658dd8a5e32117646f045d75a778468f3753699f Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Tue, 27 Oct 2026 10:35:50 +0100 Subject: [PATCH 10/16] Declare plan, plan_get and plan_clear, the grouping reasons and the grouping settings The three verbs, their reply codes (-532 for a chain no plan planned) and the reason words a plan gives a member it leaves alone are declared in data/ and rendered, as the grouping-planner design note says. The settings grouping (off by default), group_max, group_cost_max, group_rate and group_quantum are in protocol.h, and the place of a segment's worker, chain:., is part of the placement grammar. data/grouping.json joins the contract package's file lists in CI, the spec and debian/control. --- .github/workflows/ci.yml | 3 +- README.md | 15 ++++- data/grouping.json | 65 +++++++++++++++++++ data/verbs.json | 65 +++++++++++++++++++ debian/control | 1 + doc/plugin-hostd.1 | 2 +- .../plugin-hostd/plugin_hostd_host_verbs.h | 26 +++++++- .../plugin-hostd/plugin_hostd_vocabulary.h | 19 ++++++ include/plugin-hostd/protocol.h | 60 ++++++++++++++++- packaging/plugin-hostd.spec | 3 +- protocol/plugin-hostd.json | 17 ++++- 11 files changed, 266 insertions(+), 10 deletions(-) create mode 100644 data/grouping.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e46c640..ae60640 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -235,6 +235,7 @@ jobs: printf '%s\n' /usr/share/omx-contract/contract.d/3/plugin-hostd/MANIFEST.json \ /usr/share/omx-contract/contract.d/3/plugin-hostd/formats/clap.json \ /usr/share/omx-contract/contract.d/3/plugin-hostd/formats/lv2.json \ + /usr/share/omx-contract/contract.d/3/plugin-hostd/grouping.json \ /usr/share/omx-contract/contract.d/3/plugin-hostd/verbs.json \ /usr/share/licenses/plugin-hostd-contract/COPYING | sort > expected.txt diff -u expected.txt files.txt @@ -407,7 +408,7 @@ jobs: # the contract directory's files are the noarch contract package's, which the daemon and the headers depend on test "$(dpkg-deb -f debs/plugin-hostd-contract_*.deb Architecture)" = all dpkg -L plugin-hostd-contract | tee contract.txt - for f in MANIFEST.json formats/clap.json formats/lv2.json verbs.json; do + for f in MANIFEST.json formats/clap.json formats/lv2.json grouping.json verbs.json; do grep -qx "/usr/share/omx-contract/contract.d/3/plugin-hostd/$f" contract.txt done dpkg-deb -f debs/plugin-hostd-contract_*.deb Breaks Replaces | tee breaks.txt diff --git a/README.md b/README.md index 94d3162..bd28d3c 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,7 @@ The settings live in a file, `-c`, `$PLUGIN_HOSTD_CONF` or | `lv2_path` | `$LV2_PATH`, else `/usr/lib64/lv2:/usr/lib/lv2:/usr/local/lib/lv2` | where bundles are searched | | `state_root` | `$XDG_RUNTIME_DIR`, else `/tmp` | the daemon's checkpoints and worker logs | | `pin_mode` | `first-use` | strict or first-use, while require_pins is 1: first-use records the pin of a plugin on its first add, strict admits only a plugin a controller, a pin_file or pins_dir pinned | +| `grouping` | `off` | off or on: whether plan applies its plan (a run of consecutive members that may share a worker is declared as a chain), or only answers it; the default stays off until the benchmark has published a win | | `pin_store` | `` | the file first-use pins are kept in; empty is $XDG_STATE_HOME/plugin-hostd/pins, else $HOME/.local/state/plugin-hostd/pins | | `pin_file` | `` | a file of pin lines loaded at start, as plugin-qualify --all --pins writes it; empty is none | | `pins_dir` | `/usr/share/plugin-hostd/pins.d` | the read-only pins of packages: every file in it whose name ends in .pins, read at start in name order, under the store, the pin_file and pin_set, except that a pin of verdict=refused there always holds; empty is none | @@ -126,6 +127,10 @@ The settings live in a file, `-c`, `$PLUGIN_HOSTD_CONF` or | `checkpoint_ms` | 5000 ms | a quiet interval with a changed ledger writes a checkpoint | | `idle_ms` | 25 ms | the period of the idle tick: reap, respawn, checkpoint | | `pool_max` | 8 instances | instances in a pool before a sibling opens | +| `group_max` | 8 instances | most members of one segment of a plan; 0 sets no limit | +| `group_cost_max` | 60 percent of a core | most the costs of the members of one segment of a plan add to, each a member's coreFractionP95 of its qualification record at group_rate and group_quantum; a member over it alone stays alone; 0 sets no limit | +| `group_rate` | 48000 Hz | the sample rate the cost of a member is read at | +| `group_quantum` | 128 frames | the quantum the cost of a member is read at | | `require_pins` | 1 | 1: add admits only a plugin that has a pin (see pin_mode), its files hashed before any worker sees it, and a plugin of a format whose pin holds only in a world of its own gets a worker of its own, its world the plugin's files (a default that resolves to a pool is placed own); 0: no pin is checked | @@ -179,6 +184,9 @@ Verbs the daemon adds | `chain` | ` [ ...]` | `resp 0` | the instances, in this order, are one chain, run in one worker of place chain:: each member's state is saved, it is removed from the worker it was in and replayed in the chain's. A chain already declared takes the new order, and a member left out of it goes back to a worker of its own; while its worker is up that worker is kept: a new member joins it (its add, the order, its state, its ledger, its connections) and no other member is restarted, unless the new member's format narrows the worker's world, which forms the chain again. Members of clap and lv2-clap form one chain. A worker that runs chains in one client (clap, lv2-clap) is sent this line, with the members it holds, after the adds of its replay; one that does not (mod-host) never is, and a refusal of it is written to stderr and is no failure. Every instance verb still names one member by its own number. A member that crashes the worker is quarantined to a worker of its own and the chain goes on without it; quarantine_clear puts it back. -518 for an instance no add made, -519 for one named twice, -520 for one in another chain, -521 for members no one worker program runs together, -501 for a plugin that may not share a worker while require_pins is on (a CLAP plugin whose first-use pin is not in-process; an LV2 member whose bundle is not found, since a chain's worker sees a world narrowed to the union of its members' bundles), -902 for a chain or an instance that is not a number, or fewer than chain_members_min (1) instances | | `chain_clear` | `` | `resp 0` | forget the chain: each member goes to a worker of its own; -522 for a chain no chain declared | | `chain_get` | `` | `resp 0 ,... ,...` | read a chain back: its worker (- when it has none), its members in the declared order, and the members its worker runs now in their order (- for none); a quarantined member is declared and not running; -522 for a chain no chain declared, -902 for a chain that is not a number | +| `plan` | ` [ ...]` | `resp 0 ,...;,...;...` | the instances, in signal order, are planned (the grouping-planner design note): the daemon decides, from the pin and the cost of each, which runs of consecutive members share a worker. A run of two or more is declared as a chain of place chain:., k counting segments from 0; a member that shares with no neighbour is placed own. The reply names the segments in order, the members of one joined by commas and the segments by semicolons. A chain planned again keeps the running worker of every segment that did not change. The plan numbers are its own, apart from chain's. While the setting grouping is off the plan is answered and kept for plan_get and nothing moves. A segment that cannot be formed refuses the whole plan with its own code, moves nothing and is told by plan_get as plan-refused:. -518 for an instance no add made, -519 for one named twice, -520 for one in a chain or in the plan of another chain, -501 for a segment no world can be built for, -902 for a chain or an instance that is not a number, or fewer than chain_members_min (1) instances | +| `plan_get` | `` | `resp 0 ,...;,...;... :,...` | read the last plan of a chain back: its segments as plan answered them, then the members that have a reason, instance:reason joined by commas (- for none); the reasons are the words of PLUGIN_HOSTD_GROUPING_REASONS (data/grouping.json); -532 for a chain no plan planned, -902 for a chain that is not a number | +| `plan_clear` | `` | `resp 0` | forget the plan: each member goes to a worker of its own; -532 for a chain no plan planned | | `descriptor_get` | `` | `resp 0 ""` | the scanned descriptor of the plugin, answered from descriptor_dir: the file named for its binary as it is now (@-.json), written by the scan the plugin's add ran. -529 when there is none, -530 while its scan runs, -531 when its last scan wrote nothing, -528 when the directory could not be written | | `panel_get` | `` | `resp 0 ""` | the panel the plugin publishes (org.openmixer.panel-description/1), answered from descriptor_dir: the file panels/@-.json, named as its descriptor, which the same scan writes once the descriptor is in place, {"plugin": {"format", "id"}, "panel": }; an LV2 plugin's panel is null. The codes are descriptor_get's: -529 when there is none, -530 while its scan runs, -531 when its last scan wrote nothing, -528 when the directory could not be written | @@ -319,6 +327,9 @@ mechanism, and they stay optional. The preset verbs and the preset file are [doc | `chain` | daemon | required | required | required | required | | `chain_clear` | daemon | required | required | required | required | | `chain_get` | daemon | required | required | required | required | +| `plan` | daemon | required | required | required | required | +| `plan_get` | daemon | required | required | required | required | +| `plan_clear` | daemon | required | required | required | required | | `descriptor_get` | daemon | required | required | required | required | | `panel_get` | daemon | required | required | required | required | @@ -361,6 +372,7 @@ Error codes the daemon adds to mod-host's: | `-529` | `PHD_ERR_DESCRIPTOR_ABSENT` | descriptor_get (panel_get) of a plugin that has no descriptor (panel) in descriptor_dir for its binary as it is now, and none being written: no add since it changed, or a binary that cannot be found | | `-530` | `PHD_ERR_DESCRIPTOR_PENDING` | descriptor_get (panel_get) of a plugin whose scan, started by its add, has not written its descriptor (panel) yet: ask again | | `-531` | `PHD_ERR_DESCRIPTOR_SCAN_FAILED` | descriptor_get (panel_get) of a plugin whose last scan wrote no descriptor (panel): the scanner refused the plugin, made no dump or no panel, or is not installed; the add it ran for stands, and stderr and the scan's log say why. The next add scans again | +| `-532` | `PHD_ERR_NO_SUCH_PLAN` | plan_get or plan_clear of a chain no plan planned | A worker's own refusal is returned as it said it. An `add` that killed its worker answers mod-host's `-102`. @@ -392,7 +404,8 @@ Events on the feedback port, one NUL-terminated line each: | `instance_restored` | ` ` | an instance is back in a worker | | `instance_replay_refused` | ` ` | a replayed add was refused, by its pin or by the worker, the refusal's own: no worker holds the instance until the next replay | | `instance_quarantined` | ` ` | the culprit of a pool or a chain death, placed own until cleared; a chain goes on without it | -| `chain_formed` | ` ,...` | a chain's worker is up with these members, in this order, after its declaration, a respawn or a member's quarantine; - for none | +| `chain_formed` | ` ,...` | a chain's worker is up with these members, in this order, after its declaration, a respawn or a member's quarantine; - for none. is the number chain declared, or . for segment k of a plan | +| `plan_reason` | ` :,...` | a plan of the chain changed: what plan_get answers now after the segments, - for none | | `supervisor_gave_up` | ` ` | the storm bound is spent; respawning ends | | `pin_changed` | ` files \| layout` | an add or a replay of a plugin was refused because its files, or the layout the worker found, differ from its pin; the pin is as it was until pin_accept or pin_set | | `lv2_adapter_refused` | ` auto \| omx-clap-host` | the LV2 adapter of the lv2-clap worker refused a plugin: is the hosting.* code after -102 on the worker's reply to that add (- when it named none); the last word is the lv2_worker in force, auto for a plugin the lv2 row runs from now on, omx-clap-host for an add answered (-512) | diff --git a/data/grouping.json b/data/grouping.json new file mode 100644 index 0000000..f7cba82 --- /dev/null +++ b/data/grouping.json @@ -0,0 +1,65 @@ +{ + "$schema": "../schema/omx-contract.schema.json", + "requires": { + "omx-contract-schema": ">= 3.3.0" + }, + "PLUGIN_HOSTD_GROUPING_REASONS": { + "kind": "words", + "since": "0.13.0", + "doc": "Why a member of a plan is not in a segment with its neighbours, or is in one on an assumption (the grouping-planner design note, The planner), in the order the planner's rules decide them. plan_get answers them as instance:reason; a plan_reason event carries the same. A controller reads these words and retypes none.", + "words": { + "refused": { + "word": "refused", + "doc": "The pin's verdict is refused: the plugin is not run in a group, and an add of it answers -516." + }, + "noPin": { + "word": "no-pin", + "doc": "The plugin has no pin and pin_mode is strict: an add of it answers -508." + }, + "noVerdict": { + "word": "no-verdict", + "doc": "The pin carries no verdict (a pin the daemon learned on first use): the plugin stays alone." + }, + "verdictIsolated": { + "word": "verdict-isolated", + "doc": "The pin's verdict is isolated: the plugin stays alone." + }, + "formatWorld": { + "word": "format-world", + "doc": "The plugin's format pins only in a world of its own and no world holding the plugin can be built for a shared worker: it stays alone." + }, + "costOver": { + "word": "cost-over", + "doc": "The plugin's cost, alone, is over group_cost_max: it stays alone." + }, + "formatMixed": { + "word": "format-mixed", + "doc": "No one worker program runs this plugin together with the open segment: it starts a new segment." + }, + "groupFull": { + "word": "group-full", + "doc": "The open segment holds group_max members: the plugin starts a new segment." + }, + "groupCost": { + "word": "group-cost", + "doc": "Adding the plugin's cost takes the open segment over group_cost_max: it starts a new segment." + }, + "costAssumed": { + "word": "cost-assumed", + "doc": "The plugin has no known cost and joined a segment at the median of the known costs of that segment, zero when none is known." + }, + "noNeighbour": { + "word": "no-neighbour", + "doc": "The plugin passed every rule but has no neighbour it can share a worker with: it is alone only because of the members around it." + }, + "groupingOff": { + "word": "grouping-off", + "doc": "The setting grouping is off: the plugin would share a worker, and the plan was answered and not applied." + }, + "planRefused": { + "word": "plan-refused", + "doc": "A segment could not be formed: the plan was not applied and nothing moved. The word is followed by a colon and the number of the refusal, plan-refused:-501." + } + } + } +} diff --git a/data/verbs.json b/data/verbs.json index 40160ac..1cc528d 100644 --- a/data/verbs.json +++ b/data/verbs.json @@ -133,6 +133,11 @@ "code": "reply.plugin-hostd.descriptor-scan-failed", "resp": -531, "doc": "PHD_ERR_DESCRIPTOR_SCAN_FAILED: the plugin's last scan wrote no descriptor (panel)" + }, + "noSuchPlan": { + "code": "reply.plugin-hostd.no-such-plan", + "resp": -532, + "doc": "PHD_ERR_NO_SUCH_PLAN: a chain no plan verb planned" } } }, @@ -492,6 +497,66 @@ "mod-host": "unsupported" }, "doc": "One preset, 0 to count - 1: the key preset_load takes for it, and its name." + }, + "plan": { + "verb": "plan", + "format": "plan %i %i ...", + "reply": "resp 0 ,...;,...;...", + "codes": [ + { + "ref": "PLUGIN_HOSTD_REPLY_CODES.chainUnknownInstance" + }, + { + "ref": "PLUGIN_HOSTD_REPLY_CODES.chainDuplicate" + }, + { + "ref": "PLUGIN_HOSTD_REPLY_CODES.chainInstanceBusy" + }, + { + "ref": "PLUGIN_HOSTD_REPLY_CODES.placementInvalid" + }, + { + "ref": "HOST_REPLY_CODES.invalidOperation" + } + ], + "implementedBy": { + "plugin-hostd": "required" + }, + "doc": "The instances, in signal order, are planned into the fewest workers their pins and costs allow: [ ...]. Each run of consecutive members that may share a worker becomes a chain of place chain:., the others stay alone in a worker of their own; the reply names the segments in order, the members of a segment joined by commas and the segments by semicolons. With the setting grouping off the plan is answered and kept, and nothing moves." + }, + "planGet": { + "verb": "plan_get", + "format": "plan_get %i", + "reply": "resp 0 ,...;,...;... :,...", + "codes": [ + { + "ref": "PLUGIN_HOSTD_REPLY_CODES.noSuchPlan" + }, + { + "ref": "HOST_REPLY_CODES.invalidOperation" + } + ], + "implementedBy": { + "plugin-hostd": "required" + }, + "doc": "Read the last plan of a chain back: its segments as plan answered them, then the reason of each member that has one (a word of PLUGIN_HOSTD_GROUPING_REASONS, plan-refused followed by the refusal's number), instance:reason joined by commas, - for none." + }, + "planClear": { + "verb": "plan_clear", + "format": "plan_clear %i", + "reply": "resp 0", + "codes": [ + { + "ref": "PLUGIN_HOSTD_REPLY_CODES.noSuchPlan" + }, + { + "ref": "HOST_REPLY_CODES.invalidOperation" + } + ], + "implementedBy": { + "plugin-hostd": "required" + }, + "doc": "Forget the plan: each member goes to a worker of its own, as chain_clear does." } } } diff --git a/debian/control b/debian/control index 4e6f4fa..8814219 100644 --- a/debian/control +++ b/debian/control @@ -64,5 +64,6 @@ Description: plugin-hostd's format tables and verbs, read at run time tables a scan maps a CLAP or LV2 plugin through (formats/clap.json, formats/lv2.json) and the verbs the daemon adds to mod-host's protocol, with their replies, their refusals and what each host owes them (verbs.json), + and the reasons a plan gives for a plugin it leaves alone (grouping.json), under /usr/share/omx-contract/contract.d/3/plugin-hostd. A program that reads them at run time depends on this package, never on the development one. diff --git a/doc/plugin-hostd.1 b/doc/plugin-hostd.1 index eb0b7e7..80f7e3b 100644 --- a/doc/plugin-hostd.1 +++ b/doc/plugin-hostd.1 @@ -72,7 +72,7 @@ or in one .\" BEGIN GENERATED protocol:man-keys .RI \(dq key " value" \(dq -per line: mod_host, clap_host, lv2_worker, lv2_path, state_root, pin_mode, pin_store, pin_file, pins_dir, descriptor_dir, qualification_dir, probe, ready_timeout_ms, rpc_timeout_ms, backoff_base_ms, backoff_max_ms, storm_deaths, storm_window_ms, suspect_window_ms, checkpoint_ms, idle_ms, pool_max, require_pins. +per line: mod_host, clap_host, lv2_worker, lv2_path, state_root, pin_mode, grouping, pin_store, pin_file, pins_dir, descriptor_dir, qualification_dir, probe, ready_timeout_ms, rpc_timeout_ms, backoff_base_ms, backoff_max_ms, storm_deaths, storm_window_ms, suspect_window_ms, checkpoint_ms, idle_ms, pool_max, group_max, group_cost_max, group_rate, group_quantum, require_pins. .\" END GENERATED protocol:man-keys .PP The development package installs the schemas of the documents the daemon and its probe write or read, beside diff --git a/include/plugin-hostd/plugin_hostd_host_verbs.h b/include/plugin-hostd/plugin_hostd_host_verbs.h index 99a017f..bddec21 100644 --- a/include/plugin-hostd/plugin_hostd_host_verbs.h +++ b/include/plugin-hostd/plugin_hostd_host_verbs.h @@ -166,7 +166,28 @@ struct omx_host_verb { #define OMX_VERB_PRESET_GET_CODES_COUNT 3u #define OMX_VERB_PRESET_GET_CODES_INIT { -515, -3, -902 } /* reply.plugin-hostd.no-preset-list, reply.host.instance-non-exists, reply.host.invalid-operation */ -#define PLUGIN_HOSTD_HOST_VERB_COUNT 19u +/* PLUGIN_HOSTD_VERBS.plan (data/verbs.json): The instances, in signal order, are planned into the fewest workers their pins and costs allow: [ ...]. Each run of consecutive members that may share a worker becomes a chain of place chain:., the others stay alone in a worker of their own; the reply names the segments in order, the members of a segment joined by commas and the segments by semicolons. With the setting grouping off the plan is answered and kept, and nothing moves. */ +#define OMX_VERB_PLAN "plan" +#define OMX_VERB_PLAN_FMT "plan %i %i ..." +#define OMX_VERB_PLAN_REPLY "resp 0 ,...;,...;..." +#define OMX_VERB_PLAN_CODES_COUNT 5u +#define OMX_VERB_PLAN_CODES_INIT { -518, -519, -520, -501, -902 } /* reply.plugin-hostd.chain-unknown-instance, reply.plugin-hostd.chain-duplicate, reply.plugin-hostd.chain-instance-busy, reply.plugin-hostd.placement-invalid, reply.host.invalid-operation */ + +/* PLUGIN_HOSTD_VERBS.planGet (data/verbs.json): Read the last plan of a chain back: its segments as plan answered them, then the reason of each member that has one (a word of PLUGIN_HOSTD_GROUPING_REASONS, plan-refused followed by the refusal's number), instance:reason joined by commas, - for none. */ +#define OMX_VERB_PLAN_GET "plan_get" +#define OMX_VERB_PLAN_GET_FMT "plan_get %i" +#define OMX_VERB_PLAN_GET_REPLY "resp 0 ,...;,...;... :,..." +#define OMX_VERB_PLAN_GET_CODES_COUNT 2u +#define OMX_VERB_PLAN_GET_CODES_INIT { -532, -902 } /* reply.plugin-hostd.no-such-plan, reply.host.invalid-operation */ + +/* PLUGIN_HOSTD_VERBS.planClear (data/verbs.json): Forget the plan: each member goes to a worker of its own, as chain_clear does. */ +#define OMX_VERB_PLAN_CLEAR "plan_clear" +#define OMX_VERB_PLAN_CLEAR_FMT "plan_clear %i" +#define OMX_VERB_PLAN_CLEAR_REPLY "resp 0" +#define OMX_VERB_PLAN_CLEAR_CODES_COUNT 2u +#define OMX_VERB_PLAN_CLEAR_CODES_INIT { -532, -902 } /* reply.plugin-hostd.no-such-plan, reply.host.invalid-operation */ + +#define PLUGIN_HOSTD_HOST_VERB_COUNT 22u /* Every verb, in the order of its items, with what each host owes it. */ static inline const struct omx_host_verb *plugin_hostd_host_verbs(void) { @@ -190,6 +211,9 @@ static inline const struct omx_host_verb *plugin_hostd_host_verbs(void) { { OMX_VERB_PANEL_GET, OMX_VERB_PANEL_GET_FMT, OMX_VERB_PANEL_GET_REPLY, { OMX_HOST_VERB_REQUIRED, OMX_HOST_VERB_UNSUPPORTED, OMX_HOST_VERB_UNSUPPORTED } }, { OMX_VERB_PRESET_LIST, OMX_VERB_PRESET_LIST_FMT, OMX_VERB_PRESET_LIST_REPLY, { OMX_HOST_VERB_OPTIONAL, OMX_HOST_VERB_OPTIONAL, OMX_HOST_VERB_UNSUPPORTED } }, { OMX_VERB_PRESET_GET, OMX_VERB_PRESET_GET_FMT, OMX_VERB_PRESET_GET_REPLY, { OMX_HOST_VERB_OPTIONAL, OMX_HOST_VERB_OPTIONAL, OMX_HOST_VERB_UNSUPPORTED } }, + { OMX_VERB_PLAN, OMX_VERB_PLAN_FMT, OMX_VERB_PLAN_REPLY, { OMX_HOST_VERB_REQUIRED, OMX_HOST_VERB_UNSUPPORTED, OMX_HOST_VERB_UNSUPPORTED } }, + { OMX_VERB_PLAN_GET, OMX_VERB_PLAN_GET_FMT, OMX_VERB_PLAN_GET_REPLY, { OMX_HOST_VERB_REQUIRED, OMX_HOST_VERB_UNSUPPORTED, OMX_HOST_VERB_UNSUPPORTED } }, + { OMX_VERB_PLAN_CLEAR, OMX_VERB_PLAN_CLEAR_FMT, OMX_VERB_PLAN_CLEAR_REPLY, { OMX_HOST_VERB_REQUIRED, OMX_HOST_VERB_UNSUPPORTED, OMX_HOST_VERB_UNSUPPORTED } }, }; return table; } diff --git a/include/plugin-hostd/plugin_hostd_vocabulary.h b/include/plugin-hostd/plugin_hostd_vocabulary.h index 48b93bb..9e2c831 100644 --- a/include/plugin-hostd/plugin_hostd_vocabulary.h +++ b/include/plugin-hostd/plugin_hostd_vocabulary.h @@ -10,6 +10,23 @@ * literal of its own. */ +/* PLUGIN_HOSTD_GROUPING_REASONS (data/grouping.json) */ +#define OMX_PLUGIN_HOSTD_GROUPING_REASONS_REFUSED "refused" ///< The pin's verdict is refused: the plugin is not run in a group, and an add of it answers -516. +#define OMX_PLUGIN_HOSTD_GROUPING_REASONS_NO_PIN "no-pin" ///< The plugin has no pin and pin_mode is strict: an add of it answers -508. +#define OMX_PLUGIN_HOSTD_GROUPING_REASONS_NO_VERDICT "no-verdict" ///< The pin carries no verdict (a pin the daemon learned on first use): the plugin stays alone. +#define OMX_PLUGIN_HOSTD_GROUPING_REASONS_VERDICT_ISOLATED "verdict-isolated" ///< The pin's verdict is isolated: the plugin stays alone. +#define OMX_PLUGIN_HOSTD_GROUPING_REASONS_FORMAT_WORLD "format-world" ///< The plugin's format pins only in a world of its own and no world holding the plugin can be built for a shared worker: it stays alone. +#define OMX_PLUGIN_HOSTD_GROUPING_REASONS_COST_OVER "cost-over" ///< The plugin's cost, alone, is over group_cost_max: it stays alone. +#define OMX_PLUGIN_HOSTD_GROUPING_REASONS_FORMAT_MIXED "format-mixed" ///< No one worker program runs this plugin together with the open segment: it starts a new segment. +#define OMX_PLUGIN_HOSTD_GROUPING_REASONS_GROUP_FULL "group-full" ///< The open segment holds group_max members: the plugin starts a new segment. +#define OMX_PLUGIN_HOSTD_GROUPING_REASONS_GROUP_COST "group-cost" ///< Adding the plugin's cost takes the open segment over group_cost_max: it starts a new segment. +#define OMX_PLUGIN_HOSTD_GROUPING_REASONS_COST_ASSUMED "cost-assumed" ///< The plugin has no known cost and joined a segment at the median of the known costs of that segment, zero when none is known. +#define OMX_PLUGIN_HOSTD_GROUPING_REASONS_NO_NEIGHBOUR "no-neighbour" ///< The plugin passed every rule but has no neighbour it can share a worker with: it is alone only because of the members around it. +#define OMX_PLUGIN_HOSTD_GROUPING_REASONS_GROUPING_OFF "grouping-off" ///< The setting grouping is off: the plugin would share a worker, and the plan was answered and not applied. +#define OMX_PLUGIN_HOSTD_GROUPING_REASONS_PLAN_REFUSED "plan-refused" ///< A segment could not be formed: the plan was not applied and nothing moved. The word is followed by a colon and the number of the refusal, plan-refused:-501. +#define OMX_PLUGIN_HOSTD_GROUPING_REASONS_COUNT 13u +#define OMX_PLUGIN_HOSTD_GROUPING_REASONS_INIT { "refused", "no-pin", "no-verdict", "verdict-isolated", "format-world", "cost-over", "format-mixed", "group-full", "group-cost", "cost-assumed", "no-neighbour", "grouping-off", "plan-refused" } + /* PLUGIN_HOSTD_REPLY_CODES (data/verbs.json) */ #define OMX_REPLY_PLUGIN_HOSTD_PLACEMENT_INVALID "reply.plugin-hostd.placement-invalid" ///< PHD_ERR_PLACEMENT_INVALID: not a placement, or not valid here #define OMX_REPLY_PLUGIN_HOSTD_PLACEMENT_INVALID_RESP (-501) @@ -61,5 +78,7 @@ #define OMX_REPLY_PLUGIN_HOSTD_DESCRIPTOR_PENDING_RESP (-530) #define OMX_REPLY_PLUGIN_HOSTD_DESCRIPTOR_SCAN_FAILED "reply.plugin-hostd.descriptor-scan-failed" ///< PHD_ERR_DESCRIPTOR_SCAN_FAILED: the plugin's last scan wrote no descriptor (panel) #define OMX_REPLY_PLUGIN_HOSTD_DESCRIPTOR_SCAN_FAILED_RESP (-531) +#define OMX_REPLY_PLUGIN_HOSTD_NO_SUCH_PLAN "reply.plugin-hostd.no-such-plan" ///< PHD_ERR_NO_SUCH_PLAN: a chain no plan verb planned +#define OMX_REPLY_PLUGIN_HOSTD_NO_SUCH_PLAN_RESP (-532) #endif /* PLUGIN_HOSTD_VOCABULARY_H */ diff --git a/include/plugin-hostd/protocol.h b/include/plugin-hostd/protocol.h index 9f4fc1b..7c80df5 100644 --- a/include/plugin-hostd/protocol.h +++ b/include/plugin-hostd/protocol.h @@ -72,6 +72,9 @@ #define PHD_PLACE_POOL_PREFIX "pool:" /* the place of a chain's worker in worker_list, chain:; no add names it, chain puts instances there */ #define PHD_PLACE_CHAIN_PREFIX "chain:" +/* the place of the worker of segment k of a plan, chain:., k counting from 0 in the plan's order; only plan + * puts instances there, and the plan's number is its own, apart from the numbers of the chains chain declares */ +#define PHD_PLACE_PLAN_SEPARATOR "." /* a full pool opens , n counting from 2 */ #define PHD_POOL_SIBLING_SEPARATOR "#" @@ -294,6 +297,7 @@ static inline int phd_pool_name_valid(const char *name) #define PHD_ERR_DESCRIPTOR_ABSENT (-529) #define PHD_ERR_DESCRIPTOR_PENDING (-530) #define PHD_ERR_DESCRIPTOR_SCAN_FAILED (-531) +#define PHD_ERR_NO_SUCH_PLAN (-532) /* X(id, meaning) */ #define PHD_ERRORS(X) \ @@ -367,7 +371,8 @@ static inline int phd_pool_name_valid(const char *name) X(DESCRIPTOR_SCAN_FAILED, OMX_VERB_DESCRIPTOR_GET " (" OMX_VERB_PANEL_GET ") of a plugin whose last scan wrote no " \ "descriptor (panel): the scanner refused the plugin, made no dump or no panel, or is not " \ "installed; the add it ran for stands, and stderr and the scan's log say why. The next add " \ - "scans again") + "scans again") \ + X(NO_SUCH_PLAN, OMX_VERB_PLAN_GET " or " OMX_VERB_PLAN_CLEAR " of a chain no " OMX_VERB_PLAN " planned") /* ---------------------------------------------------------------- verbs */ @@ -409,6 +414,13 @@ static inline int phd_pool_name_valid(const char *name) X(FIRST_USE, PHD_PIN_MODE_FIRST_USE, "add of a plugin with no pin records its pin, keeps it in the pin store and places " \ "the instance " PHD_PLACE_OWN "; a later change is refused until pin_accept") +/* the grouping modes: X(id, name, meaning) */ +#define PHD_GROUPING_OFF "off" +#define PHD_GROUPING_ON "on" +#define PHD_GROUPING_MODES(X) \ + X(OFF, PHD_GROUPING_OFF, OMX_VERB_PLAN " answers the plan it would make and applies none") \ + X(ON, PHD_GROUPING_ON, OMX_VERB_PLAN " applies its plan") + /* a manifest's IRI that holds this is percent-encoded */ #define PHD_PIN_PERCENT "%" @@ -489,6 +501,25 @@ static inline int phd_pool_name_valid(const char *name) "read a chain back: its worker (" PHD_WORD_NONE " when it has none), its members in the declared order, and the " \ "members its worker runs now in their order (" PHD_WORD_NONE " for none); a quarantined member is declared and " \ "not running; -522 for a chain no " OMX_VERB_CHAIN " declared, -902 for a chain that is not a number") \ + X(PLAN, OMX_VERB_PLAN, " [ ...]", OMX_VERB_PLAN_REPLY, \ + "the instances, in signal order, are planned (the grouping-planner design note): the daemon decides, from the " \ + "pin and the cost of each, which runs of consecutive members share a worker. A run of two or more is declared " \ + "as a chain of place " PHD_PLACE_CHAIN_PREFIX "" PHD_PLACE_PLAN_SEPARATOR ", k counting segments from 0; " \ + "a member that shares with no neighbour is placed " PHD_PLACE_OWN ". The reply names the segments in order, " \ + "the members of one joined by commas and the segments by semicolons. A chain planned again keeps the running " \ + "worker of every segment that did not change. The plan numbers are its own, apart from chain's. While the " \ + "setting grouping is off the plan is answered and kept for " OMX_VERB_PLAN_GET " and nothing moves. A segment " \ + "that cannot be formed refuses the whole plan with its own code, moves nothing and is told by " OMX_VERB_PLAN_GET \ + " as plan-refused:. -518 for an instance no add made, -519 for one named twice, -520 for one in a chain " \ + "or in the plan of another chain, -501 for a segment no world can be built for, -902 for a chain or an " \ + "instance that is not a number, or fewer than chain_members_min (" PHD_STR(PHD_CHAIN_MEMBERS_MIN) ") instances") \ + X(PLAN_GET, OMX_VERB_PLAN_GET, "", OMX_VERB_PLAN_GET_REPLY, \ + "read the last plan of a chain back: its segments as " OMX_VERB_PLAN " answered them, then the members that have " \ + "a reason, instance:reason joined by commas (" PHD_WORD_NONE " for none); the reasons are the words of " \ + "PLUGIN_HOSTD_GROUPING_REASONS (data/grouping.json); -532 for a chain no " OMX_VERB_PLAN " planned, -902 for " \ + "a chain that is not a number") \ + X(PLAN_CLEAR, OMX_VERB_PLAN_CLEAR, "", OMX_VERB_PLAN_CLEAR_REPLY, \ + "forget the plan: each member goes to a worker of its own; -532 for a chain no " OMX_VERB_PLAN " planned") \ X(DESCRIPTOR_GET, OMX_VERB_DESCRIPTOR_GET, "", OMX_VERB_DESCRIPTOR_GET_REPLY, \ "the scanned descriptor of the plugin, answered from descriptor_dir: the file named for its binary as it is now " \ "(" PHD_DESCRIPTOR_NAME_SYNTAX "), written by the scan the plugin's add ran. -529 when there is none, -530 while " \ @@ -826,6 +857,9 @@ static inline int phd_is_note_verb(const char *verb) X(OMX_VERB_CHAIN, DAEMON, REQUIRED, REQUIRED, REQUIRED, REQUIRED) \ X(OMX_VERB_CHAIN_CLEAR, DAEMON, REQUIRED, REQUIRED, REQUIRED, REQUIRED) \ X(OMX_VERB_CHAIN_GET, DAEMON, REQUIRED, REQUIRED, REQUIRED, REQUIRED) \ + X(OMX_VERB_PLAN, DAEMON, REQUIRED, REQUIRED, REQUIRED, REQUIRED) \ + X(OMX_VERB_PLAN_GET, DAEMON, REQUIRED, REQUIRED, REQUIRED, REQUIRED) \ + X(OMX_VERB_PLAN_CLEAR, DAEMON, REQUIRED, REQUIRED, REQUIRED, REQUIRED) \ X(OMX_VERB_DESCRIPTOR_GET, DAEMON, REQUIRED, REQUIRED, REQUIRED, REQUIRED) \ X(OMX_VERB_PANEL_GET, DAEMON, REQUIRED, REQUIRED, REQUIRED, REQUIRED) @@ -853,6 +887,7 @@ static inline int phd_is_note_verb(const char *verb) #define PHD_EVENT_PIN_CHANGED "pin_changed" #define PHD_EVENT_LV2_ADAPTER_REFUSED "lv2_adapter_refused" #define PHD_EVENT_CHAIN_FORMED "chain_formed" +#define PHD_EVENT_PLAN_REASON "plan_reason" /* written by a worker, relayed as it wrote it */ #define PHD_EVENT_REMOTE_PAGES_CHANGED "remote_pages_changed" #define PHD_EVENT_REMOTE_PAGE_SUGGEST "remote_page_suggest" @@ -882,7 +917,8 @@ static inline int phd_is_note_verb(const char *verb) #define PHD_EVENT_PARAM_SET_FMT PHD_EVENT_PARAM_SET " %d %s %s" #define PHD_EVENT_PIN_CHANGED_FMT PHD_EVENT_PIN_CHANGED " %s %s" #define PHD_EVENT_LV2_ADAPTER_REFUSED_FMT PHD_EVENT_LV2_ADAPTER_REFUSED " %s %s %s" -#define PHD_EVENT_CHAIN_FORMED_FMT PHD_EVENT_CHAIN_FORMED " %d w%d %s" +#define PHD_EVENT_CHAIN_FORMED_FMT PHD_EVENT_CHAIN_FORMED " %s w%d %s" +#define PHD_EVENT_PLAN_REASON_FMT PHD_EVENT_PLAN_REASON " %d %s" /* the table of the docs: X(name, fields, meaning), one row per shape of a line */ #define PHD_EVENTS(X) \ @@ -903,7 +939,10 @@ static inline int phd_is_note_verb(const char *verb) "the culprit of a pool or a chain death, placed own until cleared; a chain goes on without it") \ X(PHD_EVENT_CHAIN_FORMED, " ,...", \ "a chain's worker is up with these members, in this order, after its declaration, a respawn or a member's " \ - "quarantine; " PHD_WORD_NONE " for none") \ + "quarantine; " PHD_WORD_NONE " for none. is the number chain declared, or " PHD_PLACE_PLAN_SEPARATOR \ + " for segment k of a plan") \ + X(PHD_EVENT_PLAN_REASON, " :,...", \ + "a plan of the chain changed: what " OMX_VERB_PLAN_GET " answers now after the segments, " PHD_WORD_NONE " for none") \ X(PHD_EVENT_SUPERVISOR_GAVE_UP, " ", "the storm bound is spent; respawning ends") \ X(PHD_EVENT_PIN_CHANGED, " " PHD_PIN_CHANGED_FILES " | " PHD_PIN_CHANGED_LAYOUT, \ "an add or a replay of a plugin was refused because its files, or the layout the worker found, differ from its " \ @@ -968,6 +1007,12 @@ static inline int phd_is_note_verb(const char *verb) #define PHD_DEFAULT_CHECKPOINT_MS 5000 #define PHD_DEFAULT_IDLE_MS 25 #define PHD_DEFAULT_POOL_MAX 8 +#define PHD_DEFAULT_GROUPING PHD_GROUPING_OFF +/* starting values of the planner's limits; the benchmark sets the ones a release ships (the design note, The benchmark) */ +#define PHD_DEFAULT_GROUP_MAX 8 +#define PHD_DEFAULT_GROUP_COST_MAX 60 +#define PHD_DEFAULT_GROUP_RATE 48000 +#define PHD_DEFAULT_GROUP_QUANTUM 128 #define PHD_DEFAULT_REQUIRE_PINS 1 #define PHD_DEFAULT_PIN_MODE PHD_PIN_MODE_FIRST_USE #define PHD_DEFAULT_PINS_DIR "/usr/share/plugin-hostd/pins.d" @@ -1015,6 +1060,9 @@ static inline int phd_is_note_verb(const char *verb) X(state_root, path, PHD_DEFAULT_STATE_ROOT, "XDG_RUNTIME_DIR", "the daemon's checkpoints and worker logs") \ X(pin_mode, word, PHD_DEFAULT_PIN_MODE, "", "strict or first-use, while require_pins is 1: first-use records the pin " \ "of a plugin on its first add, strict admits only a plugin a controller, a pin_file or pins_dir pinned") \ + X(grouping, word, PHD_DEFAULT_GROUPING, "", "off or on: whether plan applies its plan (a run of consecutive members " \ + "that may share a worker is declared as a chain), or only answers it; the default stays off until the benchmark " \ + "has published a win") \ X(pin_store, path, "", "", "the file first-use pins are kept in; empty is $XDG_STATE_HOME/plugin-hostd/pins, " \ "else $HOME/.local/state/plugin-hostd/pins") \ X(pin_file, path, "", "", "a file of pin lines loaded at start, as plugin-qualify --all --pins writes it; empty is none") \ @@ -1043,6 +1091,12 @@ static inline int phd_is_note_verb(const char *verb) X(checkpoint_ms, PHD_DEFAULT_CHECKPOINT_MS, "ms", "a quiet interval with a changed ledger writes a checkpoint") \ X(idle_ms, PHD_DEFAULT_IDLE_MS, "ms", "the period of the idle tick: reap, respawn, checkpoint") \ X(pool_max, PHD_DEFAULT_POOL_MAX, "instances", "instances in a pool before a sibling opens") \ + X(group_max, PHD_DEFAULT_GROUP_MAX, "instances", "most members of one segment of a plan; 0 sets no limit") \ + X(group_cost_max, PHD_DEFAULT_GROUP_COST_MAX, "percent of a core", \ + "most the costs of the members of one segment of a plan add to, each a member's coreFractionP95 of its " \ + "qualification record at group_rate and group_quantum; a member over it alone stays alone; 0 sets no limit") \ + X(group_rate, PHD_DEFAULT_GROUP_RATE, "Hz", "the sample rate the cost of a member is read at") \ + X(group_quantum, PHD_DEFAULT_GROUP_QUANTUM, "frames", "the quantum the cost of a member is read at") \ X(require_pins, PHD_DEFAULT_REQUIRE_PINS, "", \ "1: add admits only a plugin that has a pin (see pin_mode), its files hashed before any worker sees it, and a plugin of a " \ "format whose pin holds only in a world of its own gets a worker of its own, its world the plugin's files (a " \ diff --git a/packaging/plugin-hostd.spec b/packaging/plugin-hostd.spec index cc6b26a..8fbf070 100644 --- a/packaging/plugin-hostd.spec +++ b/packaging/plugin-hostd.spec @@ -73,7 +73,8 @@ Requires: omx-clap-core-contract >= 0.22.0 plugin-hostd's package of omx-contract's contract directory: the format tables a scan maps a CLAP or LV2 plugin through (formats/clap.json, formats/lv2.json) and the verbs the daemon adds to mod-host's protocol, with their replies, -their refusals and what each host owes them (verbs.json), under +their refusals and what each host owes them (verbs.json), and the reasons a +plan gives for a plugin it leaves alone (grouping.json), under share/omx-contract/contract.d/3/plugin-hostd. A program that reads them at run time requires this package, never the development one. diff --git a/protocol/plugin-hostd.json b/protocol/plugin-hostd.json index 637f3bc..74f498e 100644 --- a/protocol/plugin-hostd.json +++ b/protocol/plugin-hostd.json @@ -56,7 +56,8 @@ {"name": "DIR_NOT_WRITABLE", "macro": "PHD_ERR_DIR_NOT_WRITABLE", "code": -528, "meaning": "descriptor_dir or qualification_dir cannot be made or written: the daemon does not start, and says which key and which path on stderr with this code; a directory that turns read-only later makes the scan of an add write nothing, and descriptor_get and panel_get of that plugin answer this code"}, {"name": "DESCRIPTOR_ABSENT", "macro": "PHD_ERR_DESCRIPTOR_ABSENT", "code": -529, "meaning": "descriptor_get (panel_get) of a plugin that has no descriptor (panel) in descriptor_dir for its binary as it is now, and none being written: no add since it changed, or a binary that cannot be found"}, {"name": "DESCRIPTOR_PENDING", "macro": "PHD_ERR_DESCRIPTOR_PENDING", "code": -530, "meaning": "descriptor_get (panel_get) of a plugin whose scan, started by its add, has not written its descriptor (panel) yet: ask again"}, - {"name": "DESCRIPTOR_SCAN_FAILED", "macro": "PHD_ERR_DESCRIPTOR_SCAN_FAILED", "code": -531, "meaning": "descriptor_get (panel_get) of a plugin whose last scan wrote no descriptor (panel): the scanner refused the plugin, made no dump or no panel, or is not installed; the add it ran for stands, and stderr and the scan's log say why. The next add scans again"} + {"name": "DESCRIPTOR_SCAN_FAILED", "macro": "PHD_ERR_DESCRIPTOR_SCAN_FAILED", "code": -531, "meaning": "descriptor_get (panel_get) of a plugin whose last scan wrote no descriptor (panel): the scanner refused the plugin, made no dump or no panel, or is not installed; the add it ran for stands, and stderr and the scan's log say why. The next add scans again"}, + {"name": "NO_SUCH_PLAN", "macro": "PHD_ERR_NO_SUCH_PLAN", "code": -532, "meaning": "plan_get or plan_clear of a chain no plan planned"} ], "verbs": [ {"name": "add", "arguments": " [own | pool: | default] [client_name]", "reply": "resp ", "meaning": "mod-host's add with a placement; a token after the instance that is not a placement is the jack client name"}, @@ -76,6 +77,9 @@ {"name": "chain", "arguments": " [ ...]", "reply": "resp 0", "meaning": "the instances, in this order, are one chain, run in one worker of place chain:: each member's state is saved, it is removed from the worker it was in and replayed in the chain's. A chain already declared takes the new order, and a member left out of it goes back to a worker of its own; while its worker is up that worker is kept: a new member joins it (its add, the order, its state, its ledger, its connections) and no other member is restarted, unless the new member's format narrows the worker's world, which forms the chain again. Members of clap and lv2-clap form one chain. A worker that runs chains in one client (clap, lv2-clap) is sent this line, with the members it holds, after the adds of its replay; one that does not (mod-host) never is, and a refusal of it is written to stderr and is no failure. Every instance verb still names one member by its own number. A member that crashes the worker is quarantined to a worker of its own and the chain goes on without it; quarantine_clear puts it back. -518 for an instance no add made, -519 for one named twice, -520 for one in another chain, -521 for members no one worker program runs together, -501 for a plugin that may not share a worker while require_pins is on (a CLAP plugin whose first-use pin is not in-process; an LV2 member whose bundle is not found, since a chain's worker sees a world narrowed to the union of its members' bundles), -902 for a chain or an instance that is not a number, or fewer than chain_members_min (1) instances"}, {"name": "chain_clear", "arguments": "", "reply": "resp 0", "meaning": "forget the chain: each member goes to a worker of its own; -522 for a chain no chain declared"}, {"name": "chain_get", "arguments": "", "reply": "resp 0 ,... ,...", "meaning": "read a chain back: its worker (- when it has none), its members in the declared order, and the members its worker runs now in their order (- for none); a quarantined member is declared and not running; -522 for a chain no chain declared, -902 for a chain that is not a number"}, + {"name": "plan", "arguments": " [ ...]", "reply": "resp 0 ,...;,...;...", "meaning": "the instances, in signal order, are planned (the grouping-planner design note): the daemon decides, from the pin and the cost of each, which runs of consecutive members share a worker. A run of two or more is declared as a chain of place chain:., k counting segments from 0; a member that shares with no neighbour is placed own. The reply names the segments in order, the members of one joined by commas and the segments by semicolons. A chain planned again keeps the running worker of every segment that did not change. The plan numbers are its own, apart from chain's. While the setting grouping is off the plan is answered and kept for plan_get and nothing moves. A segment that cannot be formed refuses the whole plan with its own code, moves nothing and is told by plan_get as plan-refused:. -518 for an instance no add made, -519 for one named twice, -520 for one in a chain or in the plan of another chain, -501 for a segment no world can be built for, -902 for a chain or an instance that is not a number, or fewer than chain_members_min (1) instances"}, + {"name": "plan_get", "arguments": "", "reply": "resp 0 ,...;,...;... :,...", "meaning": "read the last plan of a chain back: its segments as plan answered them, then the members that have a reason, instance:reason joined by commas (- for none); the reasons are the words of PLUGIN_HOSTD_GROUPING_REASONS (data/grouping.json); -532 for a chain no plan planned, -902 for a chain that is not a number"}, + {"name": "plan_clear", "arguments": "", "reply": "resp 0", "meaning": "forget the plan: each member goes to a worker of its own; -532 for a chain no plan planned"}, {"name": "descriptor_get", "arguments": "", "reply": "resp 0 \"\"", "meaning": "the scanned descriptor of the plugin, answered from descriptor_dir: the file named for its binary as it is now (@-.json), written by the scan the plugin's add ran. -529 when there is none, -530 while its scan runs, -531 when its last scan wrote nothing, -528 when the directory could not be written"}, {"name": "panel_get", "arguments": "", "reply": "resp 0 \"\"", "meaning": "the panel the plugin publishes (org.openmixer.panel-description/1), answered from descriptor_dir: the file panels/@-.json, named as its descriptor, which the same scan writes once the descriptor is in place, {\"plugin\": {\"format\", \"id\"}, \"panel\": }; an LV2 plugin's panel is null. The codes are descriptor_get's: -529 when there is none, -530 while its scan runs, -531 when its last scan wrote nothing, -528 when the directory could not be written"} ], @@ -168,6 +172,9 @@ {"name": "chain", "source": "daemon", "levels": {"lv2-worker": "required", "clap-worker": "required", "console": "required", "zynthian": "required"}}, {"name": "chain_clear", "source": "daemon", "levels": {"lv2-worker": "required", "clap-worker": "required", "console": "required", "zynthian": "required"}}, {"name": "chain_get", "source": "daemon", "levels": {"lv2-worker": "required", "clap-worker": "required", "console": "required", "zynthian": "required"}}, + {"name": "plan", "source": "daemon", "levels": {"lv2-worker": "required", "clap-worker": "required", "console": "required", "zynthian": "required"}}, + {"name": "plan_get", "source": "daemon", "levels": {"lv2-worker": "required", "clap-worker": "required", "console": "required", "zynthian": "required"}}, + {"name": "plan_clear", "source": "daemon", "levels": {"lv2-worker": "required", "clap-worker": "required", "console": "required", "zynthian": "required"}}, {"name": "descriptor_get", "source": "daemon", "levels": {"lv2-worker": "required", "clap-worker": "required", "console": "required", "zynthian": "required"}}, {"name": "panel_get", "source": "daemon", "levels": {"lv2-worker": "required", "clap-worker": "required", "console": "required", "zynthian": "required"}} ], @@ -188,7 +195,8 @@ {"name": "instance_restored", "fields": " ", "meaning": "an instance is back in a worker"}, {"name": "instance_replay_refused", "fields": " ", "meaning": "a replayed add was refused, by its pin or by the worker, the refusal's own: no worker holds the instance until the next replay"}, {"name": "instance_quarantined", "fields": " ", "meaning": "the culprit of a pool or a chain death, placed own until cleared; a chain goes on without it"}, - {"name": "chain_formed", "fields": " ,...", "meaning": "a chain's worker is up with these members, in this order, after its declaration, a respawn or a member's quarantine; - for none"}, + {"name": "chain_formed", "fields": " ,...", "meaning": "a chain's worker is up with these members, in this order, after its declaration, a respawn or a member's quarantine; - for none. is the number chain declared, or . for segment k of a plan"}, + {"name": "plan_reason", "fields": " :,...", "meaning": "a plan of the chain changed: what plan_get answers now after the segments, - for none"}, {"name": "supervisor_gave_up", "fields": " ", "meaning": "the storm bound is spent; respawning ends"}, {"name": "pin_changed", "fields": " files | layout", "meaning": "an add or a replay of a plugin was refused because its files, or the layout the worker found, differ from its pin; the pin is as it was until pin_accept or pin_set"}, {"name": "lv2_adapter_refused", "fields": " auto | omx-clap-host", "meaning": "the LV2 adapter of the lv2-clap worker refused a plugin: is the hosting.* code after -102 on the worker's reply to that add (- when it named none); the last word is the lv2_worker in force, auto for a plugin the lv2 row runs from now on, omx-clap-host for an add answered (-512)"}, @@ -225,6 +233,7 @@ {"key": "lv2_path", "type": "string", "default": "/usr/lib64/lv2:/usr/lib/lv2:/usr/local/lib/lv2", "env": "LV2_PATH", "meaning": "where bundles are searched"}, {"key": "state_root", "type": "string", "default": "/tmp", "env": "XDG_RUNTIME_DIR", "meaning": "the daemon's checkpoints and worker logs"}, {"key": "pin_mode", "type": "string", "default": "first-use", "env": "", "meaning": "strict or first-use, while require_pins is 1: first-use records the pin of a plugin on its first add, strict admits only a plugin a controller, a pin_file or pins_dir pinned"}, + {"key": "grouping", "type": "string", "default": "off", "env": "", "meaning": "off or on: whether plan applies its plan (a run of consecutive members that may share a worker is declared as a chain), or only answers it; the default stays off until the benchmark has published a win"}, {"key": "pin_store", "type": "string", "default": "", "env": "", "meaning": "the file first-use pins are kept in; empty is $XDG_STATE_HOME/plugin-hostd/pins, else $HOME/.local/state/plugin-hostd/pins"}, {"key": "pin_file", "type": "string", "default": "", "env": "", "meaning": "a file of pin lines loaded at start, as plugin-qualify --all --pins writes it; empty is none"}, {"key": "pins_dir", "type": "string", "default": "/usr/share/plugin-hostd/pins.d", "env": "", "meaning": "the read-only pins of packages: every file in it whose name ends in .pins, read at start in name order, under the store, the pin_file and pin_set, except that a pin of verdict=refused there always holds; empty is none"}, @@ -241,6 +250,10 @@ {"key": "checkpoint_ms", "type": "int", "default": 5000, "unit": "ms", "meaning": "a quiet interval with a changed ledger writes a checkpoint"}, {"key": "idle_ms", "type": "int", "default": 25, "unit": "ms", "meaning": "the period of the idle tick: reap, respawn, checkpoint"}, {"key": "pool_max", "type": "int", "default": 8, "unit": "instances", "meaning": "instances in a pool before a sibling opens"}, + {"key": "group_max", "type": "int", "default": 8, "unit": "instances", "meaning": "most members of one segment of a plan; 0 sets no limit"}, + {"key": "group_cost_max", "type": "int", "default": 60, "unit": "percent of a core", "meaning": "most the costs of the members of one segment of a plan add to, each a member's coreFractionP95 of its qualification record at group_rate and group_quantum; a member over it alone stays alone; 0 sets no limit"}, + {"key": "group_rate", "type": "int", "default": 48000, "unit": "Hz", "meaning": "the sample rate the cost of a member is read at"}, + {"key": "group_quantum", "type": "int", "default": 128, "unit": "frames", "meaning": "the quantum the cost of a member is read at"}, {"key": "require_pins", "type": "int", "default": 1, "unit": "", "meaning": "1: add admits only a plugin that has a pin (see pin_mode), its files hashed before any worker sees it, and a plugin of a format whose pin holds only in a world of its own gets a worker of its own, its world the plugin's files (a default that resolves to a pool is placed own); 0: no pin is checked"} ] } From c7b5500aa4e5d9209a47d500d50a1ebf97124ebe Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Tue, 27 Oct 2026 10:35:50 +0100 Subject: [PATCH 11/16] Add the grouping planner as a pure function, and the reader of the cost in a qualification record planner_plan takes the members in signal order with their pin, class, world and cost and answers consecutive segments and a reason for every member left alone; the same members give the same bytes. The rules are the design note's, in its order. The history is a seam: a member carries an optional summary that no rule reads yet, and member_cost and history_reason are where the history lane plugs in. qualcost_read gives the p95 cost of a plugin at the daemon's rate and quantum from the records in qualification_dir. One test for each reason, for the median a member of unknown cost counts as, for the text of a plan and for determinism. --- Makefile | 22 +- src/planner.c | 235 ++++++++++++++++ src/planner.h | 116 ++++++++ src/qualcost.c | 114 ++++++++ src/qualcost.h | 21 ++ tests/planner_test.c | 609 ++++++++++++++++++++++++++++++++++++++++++ tests/qualcost_test.c | 146 ++++++++++ 7 files changed, 1259 insertions(+), 4 deletions(-) create mode 100644 src/planner.c create mode 100644 src/planner.h create mode 100644 src/qualcost.c create mode 100644 src/qualcost.h create mode 100644 tests/planner_test.c create mode 100644 tests/qualcost_test.c diff --git a/Makefile b/Makefile index af03f88..4b75972 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,8 @@ VERSION = $(shell sed -n "s/^Version: *//p" packaging/plugin-hostd.spec) FAKE = tests/fake-host FORMAT_TEST = tests/format-test PIN_TEST = tests/pin-test +PLANNER_TEST = tests/planner-test +QUALCOST_TEST = tests/qualcost-test RELAY_TEST = tests/relay-test GESTURE_ALLOC_TEST = tests/gesture-alloc-test LV2_ALLOC_TEST = tests/lv2-alloc-test @@ -57,7 +59,7 @@ LDFLAGS += -Wl,--no-undefined # source and object files SRC = src/main.c src/conf.c src/proc.c src/supervisor.c src/verbs.c src/pins.c src/relay.c src/formats.c src/lv2.c \ - src/descriptors.c src/describe.c src/transforms.c src/json.c + src/descriptors.c src/describe.c src/transforms.c src/json.c src/planner.c src/qualcost.c OBJ = $(SRC:.c=.o) # the C names of the daemon's verbs and reply codes, rendered from data/verbs.json (render-verbs below); protocol.h @@ -251,12 +253,12 @@ fuzz-corpus-check: # clean rule clean: - @rm -f src/*.o $(PROG) $(PROBE) tests/probe-child tests/lv2-probe-child tests/rt_interpose.so $(FAKE) $(PIN_TEST) $(RELAY_TEST) tests/relay-test-relay.o $(FORMAT_TEST) $(GESTURE_ALLOC_TEST) $(LV2_ALLOC_TEST) tests/lv2-alloc-lv2.o $(TRANSFORMS_TEST) $(GEN) tests/stress.clap tests/jack_levels tests/host_scenarios $(LV2_FIXTURE) + @rm -f src/*.o $(PROG) $(PROBE) tests/probe-child tests/lv2-probe-child tests/rt_interpose.so $(FAKE) $(PIN_TEST) $(PLANNER_TEST) $(QUALCOST_TEST) $(RELAY_TEST) tests/relay-test-relay.o $(FORMAT_TEST) $(GESTURE_ALLOC_TEST) $(LV2_ALLOC_TEST) tests/lv2-alloc-lv2.o $(TRANSFORMS_TEST) $(GEN) tests/stress.clap tests/jack_levels tests/host_scenarios $(LV2_FIXTURE) @rm -rf build # the daemon against workers that are not plugin hosts at all (tests/fake-host speaks the same protocol and can be # made to die on cue): placement, forwarding, ledger, replay, attribution, the storm bound; no jack, no plugin -test: test-protocol-lib test-pin test-transforms test-format-coverage test-describe test-relay test-alloc test-format test-daemon test-verbs-declared check-rendered check-generated check-schema test-probe-schema test-vocabulary test-probe test-contract test-consumer test-perturbation check-readme check-own-host-page check-no-pins check-ci-guards test-mod-host-json test-socket check-verb-table check-preset-schema test-install-schemas +test: test-protocol-lib test-pin test-planner test-transforms test-format-coverage test-describe test-relay test-alloc test-format test-daemon test-verbs-declared check-rendered check-generated check-schema test-probe-schema test-vocabulary test-probe test-contract test-consumer test-perturbation check-readme check-own-host-page check-no-pins check-ci-guards test-mod-host-json test-socket check-verb-table check-preset-schema test-install-schemas MOD_HOST_DIR=$(MOD_HOST_DIR) python3 tests/verbs_contract.py # the protocol clients plugin-hostd-tests ships (tests/lib/), against a peer that frames by NUL, and their sabotage @@ -295,7 +297,7 @@ test-format: $(FORMAT_TEST) $(FAKE) PLUGIN_HOSTD_FORMAT_TEST_DIR=$${TMPDIR:-$${XDG_RUNTIME_DIR:-.}} ./$(FORMAT_TEST) ./$(FAKE) # the daemon's parts without its main and its verbs: the test calls the supervisor as the verbs do -FORMAT_OBJ = src/conf.o src/proc.o src/supervisor.o src/pins.o src/relay.o src/formats.o src/lv2.o +FORMAT_OBJ = src/conf.o src/proc.o src/supervisor.o src/pins.o src/relay.o src/formats.o src/lv2.o src/planner.o src/qualcost.o src/json.o $(FORMAT_TEST): tests/format_test.c $(FORMAT_OBJ) $(PROTOCOL_LIB) $(CC) $(INCS) -Isrc $(CFLAGS) -Werror -o $@ tests/format_test.c $(FORMAT_OBJ) $(PROTOCOL_LIBS) -pthread -lm @@ -307,6 +309,18 @@ test-pin: $(PIN_TEST) $(PIN_TEST): tests/pin_test.c include/plugin-hostd/pin.h $(CC) -Iinclude $(CFLAGS) -Werror -o $@ tests/pin_test.c +# the grouping planner (src/planner.c, the grouping-planner design note), on its own: each reason it gives, the order its +# rules decide in, the text of a plan, and that the same members give the same bytes +test-planner: $(PLANNER_TEST) $(QUALCOST_TEST) + ./$(PLANNER_TEST) + ./$(QUALCOST_TEST) + +$(QUALCOST_TEST): tests/qualcost_test.c src/qualcost.c src/qualcost.h src/json.c src/json.h + $(CC) -Isrc $(CFLAGS) -Werror -o $@ tests/qualcost_test.c src/qualcost.c src/json.c -lm + +$(PLANNER_TEST): tests/planner_test.c src/planner.c src/planner.h include/plugin-hostd/protocol.h $(RENDERED) + $(CC) -Iinclude -Isrc $(CFLAGS) -Werror -o $@ tests/planner_test.c src/planner.c + # the transforms the format tables name (src/transforms.c, docs/formats.md): every one and its inverse over its vectors, # and a lookup that is not one to one refused test-transforms: $(TRANSFORMS_TEST) diff --git a/src/planner.c b/src/planner.c new file mode 100644 index 0000000..c24385a --- /dev/null +++ b/src/planner.c @@ -0,0 +1,235 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * This file is part of plugin-hostd. + * + * plugin-hostd is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * plugin-hostd is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with plugin-hostd. If not, see . + * + * Copyright (C) 2026 Pau Aliagas + */ + +#include +#include +#include + +#include +#include + +#include "planner.h" + +static const char *const g_reason_words[] = OMX_PLUGIN_HOSTD_GROUPING_REASONS_INIT; + +const char *planner_reason_word(int reason) +{ + if (reason < 0 || reason >= PLANNER_REASON_COUNT || (size_t)reason >= sizeof(g_reason_words) / sizeof(g_reason_words[0])) + return NULL; + return g_reason_words[reason]; +} + +/* ---------------------------------------------------------------- the seams the history record fills in */ + +/* The cost a member counts for in a segment. Today it is the qualification cost the supervisor read. With the history + * it is the larger of that and the member's cpu_p95_solo (the design note, The history record), which is this + * function's one line to change. PLANNER_COST_UNKNOWN when nobody measured it. */ +static double member_cost(const planner_member_t *m) +{ + return m->cost; +} + +/* The reason the history keeps a member alone, or PLANNER_REASON_NONE. history-crashed and history-xruns are the rules + * of the history lane: until they exist a history is neutral, whatever it holds, so a plugin with none and a plugin + * with some are judged alike. */ +static int history_reason(const planner_member_t *m) +{ + (void)m; + return PLANNER_REASON_NONE; +} + +/* ---------------------------------------------------------------- the rules */ + +/* The rules that leave a member alone, in the order of the design note's table, the first that matches deciding. The + * pin rules are read only while require_pins is on, as the daemon reads a pin only then. */ +static int alone_reason(const planner_settings_t *s, const planner_member_t *m) +{ + int r; + + if (s->require_pins) + { + if (m->pin == PLANNER_PIN_REFUSED) + return PLANNER_REASON_REFUSED; + if (m->pin == PLANNER_PIN_NONE && !s->pins_first_use) + return PLANNER_REASON_NO_PIN; + /* a pin of first use has no verdict, and neither has a pin that was cleared after the add */ + if ((m->pin == PLANNER_PIN_NONE || m->pin == PLANNER_PIN_NO_VERDICT) && s->pins_first_use) + return PLANNER_REASON_NO_VERDICT; + if (m->pin == PLANNER_PIN_ISOLATED) + return PLANNER_REASON_VERDICT_ISOLATED; + if (!m->world_ok) + return PLANNER_REASON_FORMAT_WORLD; + } + r = history_reason(m); + if (r != PLANNER_REASON_NONE) + return r; + if (s->group_cost_max > 0 && member_cost(m) != PLANNER_COST_UNKNOWN && member_cost(m) > s->group_cost_max) + return PLANNER_REASON_COST_OVER; + return PLANNER_REASON_NONE; +} + +/* the median of the costs the segment holds that are known; 0 when none is: a member of unknown cost counts as that */ +static double assumed_cost(const planner_member_t *m, int first, int count) +{ + double known[PLANNER_MEMBERS_MAX]; + int k, j, n = 0; + + for (k = first; k < first + count; k++) + if (member_cost(&m[k]) != PLANNER_COST_UNKNOWN) + { + for (j = n++; j > 0 && known[j - 1] > member_cost(&m[k]); j--) + known[j] = known[j - 1]; + known[j] = member_cost(&m[k]); + } + if (!n) + return 0; + return n % 2 ? known[n / 2] : (known[n / 2 - 1] + known[n / 2]) / 2; +} + +int planner_plan(const planner_settings_t *s, const planner_member_t *m, int n, planner_plan_t *out) +{ + int k, first = 0, open = 0; /* the open segment: m[first .. first + open) */ + double sum = 0; /* what its members' costs add to, the assumed ones included */ + + if (n < 1 || n > PLANNER_MEMBERS_MAX) + return -1; + memset(out, 0, sizeof(*out)); + for (k = 0; k < n; k++) + out->reason[k] = PLANNER_REASON_NONE; + + for (k = 0; k < n; k++) + { + int r = alone_reason(s, &m[k]); + double cost = member_cost(&m[k]); + + if (r != PLANNER_REASON_NONE) + { + /* it shares with no one: the open segment ends before it, and it is a segment of one */ + if (open) + out->seg_len[out->nseg++] = open; + out->seg_len[out->nseg++] = 1; + out->reason[k] = r; + first = k + 1; + open = 0; + sum = 0; + continue; + } + if (open) + { + double own = cost == PLANNER_COST_UNKNOWN ? assumed_cost(m, first, open) : cost; + + if (strcmp(m[first].klass, m[k].klass)) + r = PLANNER_REASON_FORMAT_MIXED; + else if (s->group_max > 0 && open >= s->group_max) + r = PLANNER_REASON_GROUP_FULL; + else if (s->group_cost_max > 0 && sum + own > s->group_cost_max) + r = PLANNER_REASON_GROUP_COST; + if (r == PLANNER_REASON_NONE) + { + open++; + sum += own; + continue; + } + /* it starts a new segment, and says why */ + out->seg_len[out->nseg++] = open; + out->reason[k] = r; + first = k; + open = 0; + sum = 0; + } + if (!open) + { + first = k; + open = 1; + sum = cost == PLANNER_COST_UNKNOWN ? 0 : cost; + } + } + if (open) + out->seg_len[out->nseg++] = open; + + /* what the members that said nothing have to say */ + for (k = 0, first = 0; first < out->nseg; first++) + { + int len = out->seg_len[first], j; + + for (j = k; j < k + len; j++) + { + if (out->reason[j] != PLANNER_REASON_NONE) + continue; + if (len == 1) + out->reason[j] = PLANNER_REASON_NO_NEIGHBOUR; + else if (!s->grouping_on) + out->reason[j] = PLANNER_REASON_GROUPING_OFF; + else if (member_cost(&m[j]) == PLANNER_COST_UNKNOWN) + out->reason[j] = PLANNER_REASON_COST_ASSUMED; + } + k += len; + } + return 0; +} + +/* ---------------------------------------------------------------- the text of a plan */ + +static void put(char *buf, size_t size, size_t *used, const char *fmt, ...) +{ + va_list ap; + int w; + + va_start(ap, fmt); + w = vsnprintf(*used < size ? buf + *used : NULL, *used < size ? size - *used : 0, fmt, ap); + va_end(ap); + if (w > 0) + *used += (size_t)w; +} + +size_t planner_format_segments(const planner_member_t *m, const planner_plan_t *p, char *buf, size_t size) +{ + size_t used = 0; + int s, k = 0, j; + + for (s = 0; s < p->nseg; s++) + { + if (s) + put(buf, size, &used, ";"); + for (j = 0; j < p->seg_len[s]; j++, k++) + put(buf, size, &used, "%s%d", j ? "," : "", m[k].id); + } + return used; +} + +size_t planner_format_reasons(const planner_member_t *m, const planner_plan_t *p, int refused_code, char *buf, size_t size) +{ + size_t used = 0; + int k, n = 0, count = 0; + + for (k = 0; k < p->nseg; k++) + n += p->seg_len[k]; + for (k = 0; k < n; k++) + { + if (refused_code) + put(buf, size, &used, "%s%d:%s:%d", count++ ? "," : "", m[k].id, + g_reason_words[PLANNER_REASON_PLAN_REFUSED], refused_code); + else if (p->reason[k] != PLANNER_REASON_NONE) + put(buf, size, &used, "%s%d:%s", count++ ? "," : "", m[k].id, g_reason_words[p->reason[k]]); + } + if (!count) + put(buf, size, &used, PHD_WORD_NONE); + return used; +} diff --git a/src/planner.h b/src/planner.h new file mode 100644 index 0000000..ac9478e --- /dev/null +++ b/src/planner.h @@ -0,0 +1,116 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * This file is part of plugin-hostd. + * + * plugin-hostd is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * plugin-hostd is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with plugin-hostd. If not, see . + * + * Copyright (C) 2026 Pau Aliagas + */ + +/* The grouping planner (the grouping-planner design note, The planner): given plugins in signal order, which runs of + * consecutive ones may share a worker. A pure function: the same members and settings give the same plan, byte for + * byte; it reads no clock, no file, no pin store and no global, and allocates nothing. What it is given is gathered + * by the supervisor (src/supervisor.c), what it answers is applied there. */ + +#ifndef PLANNER_H +#define PLANNER_H + +#include + +/* the most members of one plan: the words of a plan line less the verb and the chain */ +#define PLANNER_MEMBERS_MAX 62 + +/* a cost nobody measured */ +#define PLANNER_COST_UNKNOWN (-1.0) + +/* what a member's pin says, as far as the rules read it */ +enum { + PLANNER_PIN_NONE, /* no pin */ + PLANNER_PIN_NO_VERDICT, /* a pin without a verdict: one the daemon learned on first use */ + PLANNER_PIN_IN_PROCESS, + PLANNER_PIN_ISOLATED, + PLANNER_PIN_REFUSED +}; + +/* The reasons, in the order of PLUGIN_HOSTD_GROUPING_REASONS (data/grouping.json): the test holds each to its word. + * PLANNER_REASON_NONE is a member with nothing to say. */ +enum { + PLANNER_REASON_NONE = -1, + PLANNER_REASON_REFUSED, + PLANNER_REASON_NO_PIN, + PLANNER_REASON_NO_VERDICT, + PLANNER_REASON_VERDICT_ISOLATED, + PLANNER_REASON_FORMAT_WORLD, + PLANNER_REASON_COST_OVER, + PLANNER_REASON_FORMAT_MIXED, + PLANNER_REASON_GROUP_FULL, + PLANNER_REASON_GROUP_COST, + PLANNER_REASON_COST_ASSUMED, + PLANNER_REASON_NO_NEIGHBOUR, + PLANNER_REASON_GROUPING_OFF, + PLANNER_REASON_PLAN_REFUSED, + PLANNER_REASON_COUNT +}; + +/* What the history of a plugin on this host class comes to (the design note, The history record), for the rules that + * read it. Nothing reads it yet: the history record and the rules that use it are the next lane's. A member's history + * is NULL when there is none, which is neutral. */ +typedef struct PLANNER_HISTORY_T { + int crashed; /* a crash within history_keep_days */ + int raises_xruns; /* more xrun windows grouped than alone */ + double cpu_p95_solo; /* fraction of one core over the windows run alone, PLANNER_COST_UNKNOWN for none */ +} planner_history_t; + +typedef struct PLANNER_MEMBER_T { + int id; /* the instance */ + const char *klass; /* two members share a segment only when their classes are equal: the worker program that + * runs chains in one client, else the row alone */ + int pin; /* PLANNER_PIN_* */ + int world_ok; /* a world holding this plugin can be built for a shared worker, or none needs building */ + double cost; /* a fraction of one core at the daemon's rate and quantum, PLANNER_COST_UNKNOWN for none */ + const planner_history_t *history; +} planner_member_t; + +typedef struct PLANNER_SETTINGS_T { + int require_pins; /* the pin rules are read only while this is set */ + int pins_first_use; /* pin_mode is first-use, else strict */ + int grouping_on; /* off: the plan is the plan the daemon would apply */ + int group_max; /* most members of a segment, 0 for no limit */ + double group_cost_max; /* most the costs of a segment add to, a fraction of one core, 0 for no limit */ +} planner_settings_t; + +/* seg_len[0..nseg) are the lengths of the segments, which are consecutive runs of the members in the order given; + * reason[k] is the reason of member k, PLANNER_REASON_NONE for none */ +typedef struct PLANNER_PLAN_T { + int nseg; + int seg_len[PLANNER_MEMBERS_MAX]; + int reason[PLANNER_MEMBERS_MAX]; +} planner_plan_t; + +/* plans n members (1 to PLANNER_MEMBERS_MAX) into *out; 0, or -1 for a number of members outside that */ +int planner_plan(const planner_settings_t *s, const planner_member_t *m, int n, planner_plan_t *out); + +/* The word of a reason: the one of data/grouping.json, NULL outside PLANNER_REASON_* */ +const char *planner_reason_word(int reason); + +/* The segments as plan and plan_get answer them: the members of a segment joined by commas, the segments by + * semicolons, "1,2;3". Returns the length the text needs; truncates to size like snprintf. */ +size_t planner_format_segments(const planner_member_t *m, const planner_plan_t *p, char *buf, size_t size); + +/* The reasons as plan_get answers them: instance:reason joined by commas for the members that have one, "-" for none. + * A refused plan (refused_code is not 0) gives every member plan-refused:. Returns the length the text + * needs. */ +size_t planner_format_reasons(const planner_member_t *m, const planner_plan_t *p, int refused_code, char *buf, size_t size); + +#endif diff --git a/src/qualcost.c b/src/qualcost.c new file mode 100644 index 0000000..7529411 --- /dev/null +++ b/src/qualcost.c @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * This file is part of plugin-hostd. + * + * Copyright (C) 2026 Pau Aliagas + */ + +#include +#include +#include +#include +#include + +#include "json.h" +#include "qualcost.h" + +#define RECORD_SUFFIX ".qualification.json" +#define RECORD_MAX (4u << 20) + +/* the id as plugin-qualify spells it in a file name: each character outside [A-Za-z0-9._-] is an underscore */ +static void safe(const char *id, char *out, size_t size) +{ + size_t n; + + for (n = 0; id[n] && n + 1 < size; n++) + out[n] = (id[n] >= 'a' && id[n] <= 'z') || (id[n] >= 'A' && id[n] <= 'Z') || (id[n] >= '0' && id[n] <= '9') || + id[n] == '.' || id[n] == '_' || id[n] == '-' ? id[n] : '_'; + out[n] = '\0'; +} + +static char *slurp(const char *path) +{ + FILE *f = fopen(path, "r"); + char *text; + long size; + + if (!f) + return NULL; + if (fseek(f, 0, SEEK_END) != 0 || (size = ftell(f)) < 0 || (unsigned long)size > RECORD_MAX || fseek(f, 0, SEEK_SET) != 0) + { + fclose(f); + return NULL; + } + text = malloc((size_t)size + 1); + if (text && fread(text, 1, (size_t)size, f) != (size_t)size) + { + free(text); + text = NULL; + } + else if (text) + text[size] = '\0'; + fclose(f); + return text; +} + +/* the fraction of one core of the record's grid point, or -1.0 */ +static double point_of(const jnode_t *record, const char *id, int rate, int quantum) +{ + const jnode_t *plugin = jget(record, "plugin"), *grid = jget(jget(record, "measurements"), "grid"); + const char *got = plugin ? jstr(plugin, "id") : NULL; + size_t k; + + if (!got || strcmp(got, id) || !grid || grid->type != J_ARR) + return -1.0; + for (k = 0; k < grid->n; k++) + { + const jnode_t *point = grid->kid[k], *percentile = jget(jget(point, "cost"), "percentile"); + double r, q, ns; + + if (!point || !jnum(point, "rate", &r) || !jnum(point, "quantum", &q) || r != rate || q != quantum) + continue; + if (percentile && jnum(percentile, "value", &ns) && ns >= 0) + return ns * rate / 1e9; + } + return -1.0; +} + +double qualcost_read(const char *dir, const char *id, int rate, int quantum) +{ + char prefix[PATH_MAX], path[PATH_MAX * 2]; + double best = -1.0; + struct dirent *e; + size_t plen, slen = strlen(RECORD_SUFFIX); + DIR *d; + + if (!dir || !*dir || !id || !*id || !(d = opendir(dir))) + return -1.0; + safe(id, prefix, sizeof(prefix) - 2); + strcat(prefix, "@"); + plen = strlen(prefix); + while ((e = readdir(d)) != NULL) + { + size_t len = strlen(e->d_name); + char *text; + jnode_t *record; + double v; + + if (len <= plen + slen || strncmp(e->d_name, prefix, plen) || strcmp(e->d_name + len - slen, RECORD_SUFFIX)) + continue; + snprintf(path, sizeof(path), "%s/%s", dir, e->d_name); + if (!(text = slurp(path))) + continue; + record = jparse(text); + free(text); + if (!record) + continue; + v = point_of(record, id, rate, quantum); + if (v > best) + best = v; + jfree(record); + } + closedir(d); + return best; +} diff --git a/src/qualcost.h b/src/qualcost.h new file mode 100644 index 0000000..aa804e4 --- /dev/null +++ b/src/qualcost.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * This file is part of plugin-hostd. + * + * Copyright (C) 2026 Pau Aliagas + */ + +/* What a plugin costs, as plugin-qualify measured it: the qualification records of qualification_dir, one file per + * binary and offer (@-..qualification.json), each holding the figures of the probe's grid + * per rate and quantum. The daemon reads them and writes none; a record gates no add. */ + +#ifndef PLUGIN_HOSTD_QUALCOST_H +#define PLUGIN_HOSTD_QUALCOST_H + +/* The cost of the plugin with this id (the CLAP id, or the LV2 URI) at this rate and quantum, as a fraction of one core: + * the 95th-percentile nanoseconds per sample of the record's grid point, times the rate, over 1e9. When several records + * hold the point (a plugin tested in several versions) the largest wins, since the planner must not be optimistic. -1.0 + * when no record holds the point, the directory cannot be read, or a record is not one. */ +double qualcost_read(const char *dir, const char *id, int rate, int quantum); + +#endif diff --git a/tests/planner_test.c b/tests/planner_test.c new file mode 100644 index 0000000..0adf269 --- /dev/null +++ b/tests/planner_test.c @@ -0,0 +1,609 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * The grouping planner (src/planner.c) on its own, against plans worked out by hand: one test for each reason it gives, + * the order the rules decide in, the median a member of unknown cost counts as, the text of a plan, and that the same + * members give the same bytes. tests/sabotage.py breaks one rule at a time and names the test that must go red. + * + * Usage: planner-test [] one test, or all of them; "ok " for each that passes, exit 1 if one fails + */ + +#include +#include +#include + +#include +#include + +#include "planner.h" + +static int g_failed; + +#define CHECK(cond, ...) \ + do { \ + if (!(cond)) \ + { \ + fprintf(stderr, "FAIL %s:%d: ", __func__, __LINE__); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, "\n"); \ + g_failed = 1; \ + return; \ + } \ + } while (0) + +#define MEMBERS(a) ((int)(sizeof(a) / sizeof((a)[0]))) + +/* a member of the CLAP program's chain class with a pin that lets it share and a small cost */ +static planner_member_t member(int id) +{ + planner_member_t m = { id, "omx-clap-host", PLANNER_PIN_IN_PROCESS, 1, 0.01, NULL }; + + return m; +} + +static planner_member_t unknown(int id) +{ + planner_member_t m = member(id); + + m.cost = PLANNER_COST_UNKNOWN; + return m; +} + +static planner_member_t costed(int id, double cost) +{ + planner_member_t m = member(id); + + m.cost = cost; + return m; +} + +/* require_pins, first-use pins, grouping on, no limits */ +static planner_settings_t settings(void) +{ + planner_settings_t s = { 1, 1, 1, 0, 0 }; + + return s; +} + +static char *segments(const planner_member_t *m, const planner_plan_t *p) +{ + static char text[256]; + + planner_format_segments(m, p, text, sizeof(text)); + return text; +} + +static char *reasons(const planner_member_t *m, const planner_plan_t *p) +{ + static char text[512]; + + planner_format_reasons(m, p, 0, text, sizeof(text)); + return text; +} + +static int plan(const planner_settings_t *s, const planner_member_t *m, int n, planner_plan_t *p) +{ + memset(p, 0xaa, sizeof(*p)); + return planner_plan(s, m, n, p); +} + +/* ---------------------------------------------------------------- the words are the declaration's */ + +static void the_reasons_are_the_words_of_the_declaration(void) +{ + static const char *const want[] = { + OMX_PLUGIN_HOSTD_GROUPING_REASONS_REFUSED, OMX_PLUGIN_HOSTD_GROUPING_REASONS_NO_PIN, + OMX_PLUGIN_HOSTD_GROUPING_REASONS_NO_VERDICT, OMX_PLUGIN_HOSTD_GROUPING_REASONS_VERDICT_ISOLATED, + OMX_PLUGIN_HOSTD_GROUPING_REASONS_FORMAT_WORLD, OMX_PLUGIN_HOSTD_GROUPING_REASONS_COST_OVER, + OMX_PLUGIN_HOSTD_GROUPING_REASONS_FORMAT_MIXED, OMX_PLUGIN_HOSTD_GROUPING_REASONS_GROUP_FULL, + OMX_PLUGIN_HOSTD_GROUPING_REASONS_GROUP_COST, OMX_PLUGIN_HOSTD_GROUPING_REASONS_COST_ASSUMED, + OMX_PLUGIN_HOSTD_GROUPING_REASONS_NO_NEIGHBOUR, OMX_PLUGIN_HOSTD_GROUPING_REASONS_GROUPING_OFF, + OMX_PLUGIN_HOSTD_GROUPING_REASONS_PLAN_REFUSED, + }; + int r; + + CHECK(MEMBERS(want) == PLANNER_REASON_COUNT && OMX_PLUGIN_HOSTD_GROUPING_REASONS_COUNT == PLANNER_REASON_COUNT, + "the declaration has %u reasons, the planner %d", OMX_PLUGIN_HOSTD_GROUPING_REASONS_COUNT, PLANNER_REASON_COUNT); + for (r = 0; r < PLANNER_REASON_COUNT; r++) + CHECK(planner_reason_word(r) && !strcmp(planner_reason_word(r), want[r]), "reason %d is %s, not %s", r, + planner_reason_word(r), want[r]); + CHECK(planner_reason_word(PLANNER_REASON_NONE) == NULL && planner_reason_word(PLANNER_REASON_COUNT) == NULL, + "a number that is no reason has no word"); +} + +/* ---------------------------------------------------------------- one test per reason */ + +/* A B C D E with the middle one unfit becomes [A B] [C] [D E]: C is alone for `reason`, the others have nothing to say */ +static void unfit_middle(const char *what, planner_member_t c, const planner_settings_t *s, int reason, const char *word) +{ + planner_member_t m[5] = { member(1), member(2), c, member(4), member(5) }; + planner_plan_t p; + char want[64]; + + c.id = 3; + m[2] = c; + CHECK(plan(s, m, 5, &p) == 0, "%s: planned", what); + CHECK(!strcmp(segments(m, &p), "1,2;3;4,5"), "%s: the segments are %s", what, segments(m, &p)); + snprintf(want, sizeof(want), "3:%s", word); + CHECK(p.reason[2] == reason, "%s: member 3 has reason %d, not %d", what, p.reason[2], reason); + CHECK(!strcmp(reasons(m, &p), want), "%s: the reasons are %s, not %s", what, reasons(m, &p), want); + CHECK(p.nseg == 3 && p.seg_len[0] == 2 && p.seg_len[1] == 1 && p.seg_len[2] == 2, "%s: the lengths", what); +} + +static void refused_leaves_the_member_alone(void) +{ + planner_settings_t s = settings(); + planner_member_t c = member(3); + + c.pin = PLANNER_PIN_REFUSED; + unfit_middle("a refused pin", c, &s, PLANNER_REASON_REFUSED, "refused"); +} + +static void no_pin_under_strict_leaves_the_member_alone(void) +{ + planner_settings_t s = settings(); + planner_member_t c = member(3); + + s.pins_first_use = 0; + c.pin = PLANNER_PIN_NONE; + unfit_middle("no pin under strict", c, &s, PLANNER_REASON_NO_PIN, "no-pin"); +} + +static void no_verdict_leaves_the_member_alone(void) +{ + planner_settings_t s = settings(); + planner_member_t c = member(3); + + c.pin = PLANNER_PIN_NO_VERDICT; + unfit_middle("a learned pin", c, &s, PLANNER_REASON_NO_VERDICT, "no-verdict"); +} + +static void a_pin_that_is_gone_under_first_use_has_no_verdict(void) +{ + planner_settings_t s = settings(); + planner_member_t c = member(3); + + c.pin = PLANNER_PIN_NONE; + unfit_middle("no pin under first-use", c, &s, PLANNER_REASON_NO_VERDICT, "no-verdict"); +} + +static void verdict_isolated_leaves_the_member_alone(void) +{ + planner_settings_t s = settings(); + planner_member_t c = member(3); + + c.pin = PLANNER_PIN_ISOLATED; + unfit_middle("verdict isolated", c, &s, PLANNER_REASON_VERDICT_ISOLATED, "verdict-isolated"); +} + +static void format_world_leaves_the_member_alone(void) +{ + planner_settings_t s = settings(); + planner_member_t c = member(3); + + c.world_ok = 0; + unfit_middle("no world for the format", c, &s, PLANNER_REASON_FORMAT_WORLD, "format-world"); +} + +static void cost_over_leaves_the_member_alone(void) +{ + planner_settings_t s = settings(); + + s.group_cost_max = 0.5; + unfit_middle("one member over the limit", costed(3, 0.51), &s, PLANNER_REASON_COST_OVER, "cost-over"); +} + +/* a member that exactly meets the limit is not over it */ +static void a_cost_at_the_limit_is_not_over(void) +{ + planner_settings_t s = settings(); + planner_member_t m[2] = { costed(1, 0.5), costed(2, 0.0) }; + planner_plan_t p; + + s.group_cost_max = 0.5; + CHECK(plan(&s, m, 2, &p) == 0 && !strcmp(segments(m, &p), "1,2"), "0.5 against 0.5 joins: %s", segments(m, &p)); +} + +static void format_mixed_starts_a_new_segment(void) +{ + planner_settings_t s = settings(); + planner_member_t m[4] = { member(1), member(2), member(3), member(4) }; + planner_plan_t p; + + m[2].klass = "mod-host"; + m[3].klass = "mod-host"; + CHECK(plan(&s, m, 4, &p) == 0, "planned"); + CHECK(!strcmp(segments(m, &p), "1,2;3,4"), "a program that does not run the others' chains starts a segment: %s", segments(m, &p)); + CHECK(p.reason[2] == PLANNER_REASON_FORMAT_MIXED && p.reason[0] == PLANNER_REASON_NONE && + p.reason[1] == PLANNER_REASON_NONE && p.reason[3] == PLANNER_REASON_NONE, "member 3 says why, the others nothing"); + CHECK(!strcmp(reasons(m, &p), "3:format-mixed"), "the reasons are %s", reasons(m, &p)); +} + +static void group_full_starts_a_new_segment(void) +{ + planner_settings_t s = settings(); + planner_member_t m[5] = { member(1), member(2), member(3), member(4), member(5) }; + planner_plan_t p; + + s.group_max = 2; + CHECK(plan(&s, m, 5, &p) == 0, "planned"); + CHECK(!strcmp(segments(m, &p), "1,2;3,4;5"), "two to a segment: %s", segments(m, &p)); + CHECK(p.reason[2] == PLANNER_REASON_GROUP_FULL && p.reason[3] == PLANNER_REASON_NONE && + p.reason[4] == PLANNER_REASON_GROUP_FULL, "the members that start a segment say why: %s", reasons(m, &p)); + CHECK(!strcmp(reasons(m, &p), "3:group-full,5:group-full"), "the reasons are %s", reasons(m, &p)); +} + +static void group_max_zero_is_no_limit(void) +{ + planner_settings_t s = settings(); + planner_member_t m[PLANNER_MEMBERS_MAX]; + planner_plan_t p; + int k; + + for (k = 0; k < PLANNER_MEMBERS_MAX; k++) + m[k] = member(k + 1); + CHECK(plan(&s, m, PLANNER_MEMBERS_MAX, &p) == 0 && p.nseg == 1 && p.seg_len[0] == PLANNER_MEMBERS_MAX, + "all %d in one segment, not %d", PLANNER_MEMBERS_MAX, p.nseg); +} + +static void group_cost_starts_a_new_segment(void) +{ + planner_settings_t s = settings(); + planner_member_t m[4] = { costed(1, 0.25), costed(2, 0.25), costed(3, 0.25), costed(4, 0.25) }; + planner_plan_t p; + + s.group_cost_max = 0.6; + CHECK(plan(&s, m, 4, &p) == 0, "planned"); + CHECK(!strcmp(segments(m, &p), "1,2;3,4"), "0.5 fits under 0.6, 0.75 does not: %s", segments(m, &p)); + CHECK(!strcmp(reasons(m, &p), "3:group-cost"), "the reasons are %s", reasons(m, &p)); +} + +/* a segment's costs add: three of 0.25 are over 0.7 on the third, though each is far under it alone */ +static void the_costs_of_a_segment_add_up(void) +{ + planner_settings_t s = settings(); + planner_member_t m[3] = { costed(1, 0.25), costed(2, 0.25), costed(3, 0.25) }; + planner_plan_t p; + + s.group_cost_max = 0.7; + CHECK(plan(&s, m, 3, &p) == 0 && !strcmp(segments(m, &p), "1,2;3"), "the third is over: %s", segments(m, &p)); + CHECK(p.reason[2] == PLANNER_REASON_GROUP_COST, "and says group-cost"); + s.group_cost_max = 0.75; + CHECK(plan(&s, m, 3, &p) == 0 && !strcmp(segments(m, &p), "1,2,3"), "0.75 holds all three: %s", segments(m, &p)); +} + +/* a member of unknown cost counts as the median of the known costs of its segment: three known at 1/8, 1/2 and 1/4 add + * 7/8 and have the median 1/4, so the first unknown member makes 9/8 and the second 11/8, which is over 5/4 (the costs + * are halves and quarters, so the sums are exact) */ +static void a_member_of_unknown_cost_counts_as_the_median_of_its_segment(void) +{ + planner_settings_t s = settings(); + planner_member_t m[5] = { costed(1, 0.125), costed(2, 0.5), costed(3, 0.25), unknown(4), unknown(5) }; + planner_plan_t p; + + s.group_cost_max = 1.25; + CHECK(plan(&s, m, 5, &p) == 0, "planned"); + CHECK(!strcmp(segments(m, &p), "1,2,3,4;5"), "9/8 fits 5/4, 11/8 does not: %s", segments(m, &p)); + CHECK(p.reason[3] == PLANNER_REASON_COST_ASSUMED && p.reason[4] == PLANNER_REASON_GROUP_COST, "the reasons: %s", reasons(m, &p)); + CHECK(!strcmp(reasons(m, &p), "4:cost-assumed,5:group-cost"), "the reasons are %s", reasons(m, &p)); +} + +/* the median of an even number is the mean of the middle two */ +static void the_median_of_an_even_number_is_the_mean_of_the_middle_two(void) +{ + planner_settings_t s = settings(); + planner_member_t m[4] = { costed(1, 0.0625), costed(2, 0.1875), unknown(3), unknown(4) }; + planner_plan_t p; + + /* 1/16 and 3/16 known: the median is 1/8, so the third makes 3/8 and the fourth 1/2 */ + s.group_cost_max = 0.45; + CHECK(plan(&s, m, 4, &p) == 0 && !strcmp(segments(m, &p), "1,2,3;4"), "1/8 more is over 0.45 on the fourth: %s", segments(m, &p)); + s.group_cost_max = 0.5; + CHECK(plan(&s, m, 4, &p) == 0 && !strcmp(segments(m, &p), "1,2,3,4"), "1/2 holds all four: %s", segments(m, &p)); +} + +/* with no cost known in the segment a member counts for nothing, and says so */ +static void members_of_unknown_cost_alone_in_a_segment_count_for_nothing(void) +{ + planner_settings_t s = settings(); + planner_member_t m[3] = { unknown(1), unknown(2), unknown(3) }; + planner_plan_t p; + + s.group_cost_max = 0.01; + CHECK(plan(&s, m, 3, &p) == 0 && !strcmp(segments(m, &p), "1,2,3"), "no cost, no limit reached: %s", segments(m, &p)); + CHECK(!strcmp(reasons(m, &p), "1:cost-assumed,2:cost-assumed,3:cost-assumed"), "each says so: %s", reasons(m, &p)); +} + +static void cost_assumed_is_said_of_a_member_that_joined_on_an_assumption(void) +{ + planner_settings_t s = settings(); + planner_member_t m[3] = { costed(1, 0.1), unknown(2), costed(3, 0.1) }; + planner_plan_t p; + + CHECK(plan(&s, m, 3, &p) == 0 && !strcmp(segments(m, &p), "1,2,3"), "one segment: %s", segments(m, &p)); + CHECK(!strcmp(reasons(m, &p), "2:cost-assumed"), "only the member without a cost says so: %s", reasons(m, &p)); +} + +static void no_neighbour_is_said_of_a_member_alone_for_the_members_around_it(void) +{ + planner_settings_t s = settings(); + planner_member_t m[3] = { member(1), member(2), member(3) }; + planner_plan_t p; + + m[1].pin = PLANNER_PIN_ISOLATED; + CHECK(plan(&s, m, 3, &p) == 0 && !strcmp(segments(m, &p), "1;2;3"), "all alone: %s", segments(m, &p)); + CHECK(!strcmp(reasons(m, &p), "1:no-neighbour,2:verdict-isolated,3:no-neighbour"), + "the unfit one says why, its neighbours that they have no one: %s", reasons(m, &p)); + CHECK(plan(&s, m, 1, &p) == 0 && !strcmp(segments(m, &p), "1") && !strcmp(reasons(m, &p), "1:no-neighbour"), + "a plan of one member is a member with no neighbour: %s", reasons(m, &p)); +} + +static void grouping_off_says_what_would_have_shared(void) +{ + planner_settings_t s = settings(); + planner_member_t m[4] = { member(1), member(2), member(3), costed(4, 0.1) }; + planner_plan_t p; + + m[2].pin = PLANNER_PIN_ISOLATED; + s.grouping_on = 0; + CHECK(plan(&s, m, 4, &p) == 0, "planned"); + CHECK(!strcmp(segments(m, &p), "1,2;3;4"), "the plan is the one grouping on would make: %s", segments(m, &p)); + CHECK(!strcmp(reasons(m, &p), "1:grouping-off,2:grouping-off,3:verdict-isolated,4:no-neighbour"), + "the members of a segment that is not applied say so, the others as before: %s", reasons(m, &p)); + s.grouping_on = 1; + CHECK(plan(&s, m, 4, &p) == 0 && !strcmp(segments(m, &p), "1,2;3;4"), "on: the same segments"); + CHECK(!strcmp(reasons(m, &p), "3:verdict-isolated,4:no-neighbour"), "on: %s", reasons(m, &p)); +} + +static void a_refused_plan_gives_every_member_the_refusal(void) +{ + planner_settings_t s = settings(); + planner_member_t m[3] = { member(7), member(8), member(9) }; + planner_plan_t p; + char text[128]; + + CHECK(plan(&s, m, 3, &p) == 0, "planned"); + planner_format_reasons(m, &p, PHD_ERR_PLACEMENT_INVALID, text, sizeof(text)); + CHECK(!strcmp(text, "7:plan-refused:-501,8:plan-refused:-501,9:plan-refused:-501"), "the reasons are %s", text); + CHECK(!strncmp(text + 2, planner_reason_word(PLANNER_REASON_PLAN_REFUSED), strlen(planner_reason_word(PLANNER_REASON_PLAN_REFUSED))), + "and the word is the declared one"); +} + +/* ---------------------------------------------------------------- the rules together */ + +static void the_first_rule_that_matches_decides(void) +{ + planner_settings_t s = settings(); + planner_member_t m[1]; + planner_plan_t p; + + s.group_cost_max = 0.1; + m[0] = costed(1, 0.9); + m[0].pin = PLANNER_PIN_REFUSED; + m[0].world_ok = 0; + CHECK(plan(&s, m, 1, &p) == 0 && p.reason[0] == PLANNER_REASON_REFUSED, "refused before the world and the cost"); + m[0].pin = PLANNER_PIN_ISOLATED; + CHECK(plan(&s, m, 1, &p) == 0 && p.reason[0] == PLANNER_REASON_VERDICT_ISOLATED, "isolated before the world and the cost"); + m[0].pin = PLANNER_PIN_NO_VERDICT; + CHECK(plan(&s, m, 1, &p) == 0 && p.reason[0] == PLANNER_REASON_NO_VERDICT, "no verdict before isolated and the world"); + m[0].pin = PLANNER_PIN_IN_PROCESS; + CHECK(plan(&s, m, 1, &p) == 0 && p.reason[0] == PLANNER_REASON_FORMAT_WORLD, "the world before the cost"); + m[0].world_ok = 1; + CHECK(plan(&s, m, 1, &p) == 0 && p.reason[0] == PLANNER_REASON_COST_OVER, "then the cost"); +} + +/* a member that meets format-mixed, group-full and group-cost at once is told the first of them */ +static void format_mixed_comes_before_group_full_and_group_cost(void) +{ + planner_settings_t s = settings(); + planner_member_t m[2] = { costed(1, 0.5), costed(2, 0.5) }; + planner_plan_t p; + + s.group_max = 1; + s.group_cost_max = 0.6; + CHECK(plan(&s, m, 2, &p) == 0 && p.reason[1] == PLANNER_REASON_GROUP_FULL, "full before cost: %s", reasons(m, &p)); + m[1].klass = "mod-host"; + CHECK(plan(&s, m, 2, &p) == 0 && p.reason[1] == PLANNER_REASON_FORMAT_MIXED, "mixed before full: %s", reasons(m, &p)); +} + +static void without_require_pins_no_pin_is_read(void) +{ + planner_settings_t s = settings(); + planner_member_t m[3] = { member(1), member(2), member(3) }; + planner_plan_t p; + + s.require_pins = 0; + m[0].pin = PLANNER_PIN_REFUSED; + m[1].pin = PLANNER_PIN_NONE; + m[2].pin = PLANNER_PIN_ISOLATED; + m[2].world_ok = 0; + CHECK(plan(&s, m, 3, &p) == 0 && !strcmp(segments(m, &p), "1,2,3"), "pins and worlds are not read: %s", segments(m, &p)); +} + +/* under strict a pin the daemon did not learn needs no verdict to share: the chain verb lets it */ +static void under_strict_a_pin_without_a_verdict_shares(void) +{ + planner_settings_t s = settings(); + planner_member_t m[2] = { member(1), member(2) }; + planner_plan_t p; + + s.pins_first_use = 0; + m[0].pin = PLANNER_PIN_NO_VERDICT; + CHECK(plan(&s, m, 2, &p) == 0 && !strcmp(segments(m, &p), "1,2"), "shares: %s", segments(m, &p)); +} + +static void a_plan_of_unfit_members_is_all_segments_of_one(void) +{ + planner_settings_t s = settings(); + planner_member_t m[3] = { member(1), member(2), member(3) }; + planner_plan_t p; + int k; + + for (k = 0; k < 3; k++) + m[k].pin = PLANNER_PIN_NO_VERDICT; + CHECK(plan(&s, m, 3, &p) == 0 && !strcmp(segments(m, &p), "1;2;3") && p.nseg == 3, "three alone: %s", segments(m, &p)); + CHECK(!strcmp(reasons(m, &p), "1:no-verdict,2:no-verdict,3:no-verdict"), "each says why: %s", reasons(m, &p)); +} + +static void the_plan_keeps_the_signal_order(void) +{ + planner_settings_t s = settings(); + planner_member_t m[4] = { member(40), member(7), member(23), member(1) }; + planner_plan_t p; + + m[1].pin = PLANNER_PIN_ISOLATED; + CHECK(plan(&s, m, 4, &p) == 0 && !strcmp(segments(m, &p), "40;7;23,1"), "the order given, not the numbers': %s", segments(m, &p)); +} + +static void a_plan_of_no_member_or_too_many_is_refused(void) +{ + planner_settings_t s = settings(); + planner_member_t m[PLANNER_MEMBERS_MAX + 1]; + planner_plan_t p; + int k; + + for (k = 0; k <= PLANNER_MEMBERS_MAX; k++) + m[k] = member(k); + CHECK(planner_plan(&s, m, 0, &p) == -1, "none"); + CHECK(planner_plan(&s, m, PLANNER_MEMBERS_MAX + 1, &p) == -1, "too many"); + CHECK(planner_plan(&s, m, PLANNER_MEMBERS_MAX, &p) == 0, "the most"); +} + +/* ---------------------------------------------------------------- the text */ + +static void the_text_of_a_plan_is_the_one_the_bench_parses(void) +{ + planner_settings_t s = settings(); + planner_member_t m[5] = { member(1), member(2), member(3), member(4), member(5) }; + planner_plan_t p; + char text[64]; + size_t need; + + m[2].pin = PLANNER_PIN_NO_VERDICT; + CHECK(plan(&s, m, 5, &p) == 0, "planned"); + snprintf(text, sizeof(text), "resp 0 %s", segments(m, &p)); + CHECK(!strcmp(text, "resp 0 1,2;3;4,5"), "the reply is resp 0 i,i;i: %s", text); + need = planner_format_segments(m, &p, text, 4); + CHECK(need == strlen("1,2;3;4,5") && !strcmp(text, "1,2"), "a short buffer is cut and the need said: %zu '%s'", need, text); + need = planner_format_reasons(m, &p, 0, NULL, 0); + CHECK(need == strlen("3:no-verdict"), "the reasons can be measured: %zu", need); +} + +static void no_reason_is_a_dash(void) +{ + planner_settings_t s = settings(); + planner_member_t m[2] = { costed(1, 0.1), costed(2, 0.1) }; + planner_plan_t p; + + CHECK(plan(&s, m, 2, &p) == 0 && !strcmp(reasons(m, &p), "-"), "nothing to say is -: %s", reasons(m, &p)); +} + +/* ---------------------------------------------------------------- the same members, the same plan */ + +static void the_same_members_give_the_same_bytes(void) +{ + planner_settings_t s = settings(); + planner_member_t a[6] = { costed(10, 0.2), member(11), costed(12, 0.3), costed(13, 0.35), member(14), costed(15, 0.1) }; + planner_member_t b[3] = { member(1), member(2), member(3) }; + planner_plan_t first, again, other; + char seg[128], why[256], seg2[128], why2[256]; + int round; + + s.group_max = 3; + s.group_cost_max = 0.7; + a[1].pin = PLANNER_PIN_ISOLATED; + a[4].pin = PLANNER_PIN_NO_VERDICT; + memset(&first, 0x00, sizeof(first)); + CHECK(planner_plan(&s, a, 6, &first) == 0, "planned"); + planner_format_segments(a, &first, seg, sizeof(seg)); + planner_format_reasons(a, &first, 0, why, sizeof(why)); + for (round = 0; round < 50; round++) + { + /* the output starts as other bytes, and other plans were made between: nothing carries over */ + memset(&again, round % 2 ? 0xff : 0x5a, sizeof(again)); + CHECK(planner_plan(&s, b, 3, &other) == 0, "another plan"); + CHECK(planner_plan(&s, a, 6, &again) == 0, "planned again"); + planner_format_segments(a, &again, seg2, sizeof(seg2)); + planner_format_reasons(a, &again, 0, why2, sizeof(why2)); + CHECK(!strcmp(seg, seg2) && !strcmp(why, why2), "round %d: '%s' '%s' against '%s' '%s'", round, seg, why, seg2, why2); + CHECK(again.nseg == first.nseg && !memcmp(again.seg_len, first.seg_len, sizeof(int) * (size_t)first.nseg) && + !memcmp(again.reason, first.reason, sizeof(int) * 6), "round %d: the plan itself differs", round); + } + CHECK(!strcmp(seg, "10;11;12,13;14;15"), "and the plan is the worked one: %s (%s)", seg, why); +} + +/* ---------------------------------------------------------------- the seam of the history lane */ + +static void a_history_is_neutral_until_the_history_rules_land(void) +{ + planner_settings_t s = settings(); + planner_member_t m[3] = { member(1), member(2), member(3) }; + planner_history_t bad = { 1, 1, 0.9 }; + planner_plan_t p; + + m[1].history = &bad; + CHECK(plan(&s, m, 3, &p) == 0 && !strcmp(segments(m, &p), "1,2,3"), + "a history that marks a plugin crashed changes nothing in this lane: %s", segments(m, &p)); +} + +static const struct { const char *name; void (*fn)(void); } g_tests[] = { +#define T(f) { #f, f } + T(the_reasons_are_the_words_of_the_declaration), + T(refused_leaves_the_member_alone), + T(no_pin_under_strict_leaves_the_member_alone), + T(no_verdict_leaves_the_member_alone), + T(a_pin_that_is_gone_under_first_use_has_no_verdict), + T(verdict_isolated_leaves_the_member_alone), + T(format_world_leaves_the_member_alone), + T(cost_over_leaves_the_member_alone), + T(a_cost_at_the_limit_is_not_over), + T(format_mixed_starts_a_new_segment), + T(group_full_starts_a_new_segment), + T(group_max_zero_is_no_limit), + T(group_cost_starts_a_new_segment), + T(the_costs_of_a_segment_add_up), + T(a_member_of_unknown_cost_counts_as_the_median_of_its_segment), + T(the_median_of_an_even_number_is_the_mean_of_the_middle_two), + T(members_of_unknown_cost_alone_in_a_segment_count_for_nothing), + T(cost_assumed_is_said_of_a_member_that_joined_on_an_assumption), + T(no_neighbour_is_said_of_a_member_alone_for_the_members_around_it), + T(grouping_off_says_what_would_have_shared), + T(a_refused_plan_gives_every_member_the_refusal), + T(the_first_rule_that_matches_decides), + T(format_mixed_comes_before_group_full_and_group_cost), + T(without_require_pins_no_pin_is_read), + T(under_strict_a_pin_without_a_verdict_shares), + T(a_plan_of_unfit_members_is_all_segments_of_one), + T(the_plan_keeps_the_signal_order), + T(a_plan_of_no_member_or_too_many_is_refused), + T(the_text_of_a_plan_is_the_one_the_bench_parses), + T(no_reason_is_a_dash), + T(the_same_members_give_the_same_bytes), + T(a_history_is_neutral_until_the_history_rules_land), +#undef T +}; + +int main(int argc, char **argv) +{ + size_t n, ran = 0; + int bad = 0; + + for (n = 0; n < sizeof(g_tests) / sizeof(g_tests[0]); n++) + { + if (argc > 1 && strcmp(argv[1], g_tests[n].name)) + continue; + g_failed = 0; + g_tests[n].fn(); + ran++; + if (g_failed) + bad++; + else + printf("ok %s\n", g_tests[n].name); + } + if (!ran) + { + fprintf(stderr, "no test named %s\n", argc > 1 ? argv[1] : ""); + return 2; + } + printf("planner tests %s (%zu run, %d failed)\n", bad ? "FAILED" : "ok", ran, bad); + return bad ? 1 : 0; +} diff --git a/tests/qualcost_test.c b/tests/qualcost_test.c new file mode 100644 index 0000000..042a930 --- /dev/null +++ b/tests/qualcost_test.c @@ -0,0 +1,146 @@ +/* SPDX-License-Identifier: GPL-3.0-or-later */ +/* + * The cost the daemon reads from qualification records (src/qualcost.c), against records written out by hand: the + * figure of the right rate and quantum, the larger of two records, the records that are not the plugin's, and the files + * that are no records. tests/sabotage.py breaks one rule at a time. + * + * Usage: qualcost-test [] + */ + +#include +#include +#include +#include +#include +#include + +#include "qualcost.h" + +static int g_failed; +static char g_dir[256]; + +#define CHECK(cond, ...) \ + do { \ + if (!(cond)) \ + { \ + fprintf(stderr, "FAIL %s:%d: ", __func__, __LINE__); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, "\n"); \ + g_failed = 1; \ + return; \ + } \ + } while (0) + +static void put(const char *name, const char *text) +{ + char path[512]; + FILE *f; + + snprintf(path, sizeof(path), "%s/%s", g_dir, name); + f = fopen(path, "w"); + if (!f || fputs(text, f) < 0) + exit(2); + fclose(f); +} + +static void record(const char *name, const char *id, int rate, int quantum, double ns) +{ + char text[1024]; + + snprintf(text, sizeof(text), + "{\"schema\":\"plugin-qualify.qualification-record/1\",\"plugin\":{\"format\":\"clap\",\"id\":\"%s\"}," + "\"measurements\":{\"grid\":[{\"rate\":96000,\"quantum\":64,\"outcome\":\"probe.ok\",\"cost\":{\"median\":1," + "\"percentile\":{\"p\":95,\"value\":1},\"blocks\":1,\"warmupBlocks\":0}},{\"rate\":%d,\"quantum\":%d," + "\"outcome\":\"probe.ok\",\"cost\":{\"median\":1,\"percentile\":{\"p\":95,\"value\":%.1f},\"blocks\":1," + "\"warmupBlocks\":0}}]}}", + id, rate, quantum, ns); + put(name, text); +} + +static void the_figure_is_the_p95_at_the_rate_and_quantum_as_a_fraction_of_a_core(void) +{ + double c = qualcost_read(g_dir, "org.example.a", 48000, 128); + + CHECK(fabs(c - 0.24) < 1e-12, "5000 ns per sample at 48 kHz is 0.24 of a core, not %g", c); + c = qualcost_read(g_dir, "org.example.a", 96000, 64); + CHECK(fabs(c - 96000.0 / 1e9) < 1e-12, "the other grid point of the record is its own figure: %g", c); +} + +static void a_rate_or_quantum_the_record_lacks_is_unknown(void) +{ + CHECK(qualcost_read(g_dir, "org.example.a", 44100, 128) == -1.0, "no such rate"); + CHECK(qualcost_read(g_dir, "org.example.a", 48000, 256) == -1.0, "no such quantum"); +} + +static void the_larger_of_two_records_of_one_plugin_stands(void) +{ + double c = qualcost_read(g_dir, "org.example.b", 48000, 128); + + CHECK(fabs(c - 0.48) < 1e-12, "10000 ns beats 2000 ns: %g", c); +} + +static void a_record_of_another_plugin_is_not_this_ones(void) +{ + CHECK(qualcost_read(g_dir, "org.example.c", 48000, 128) == -1.0, "c has no record"); + /* the file name begins with the id, the id inside is another */ + CHECK(qualcost_read(g_dir, "org.example.liar", 48000, 128) == -1.0, "the id inside the record decides"); +} + +static void a_file_that_is_no_record_is_skipped(void) +{ + CHECK(qualcost_read(g_dir, "org.example.junk", 48000, 128) == -1.0, "junk"); + CHECK(qualcost_read(g_dir, "org.example.empty", 48000, 128) == -1.0, "empty"); +} + +static void a_directory_that_is_not_there_gives_no_figure(void) +{ + CHECK(qualcost_read("/nonexistent/qualification", "org.example.a", 48000, 128) == -1.0, "no directory"); + CHECK(qualcost_read(NULL, "org.example.a", 48000, 128) == -1.0, "no name"); +} + +static const struct { const char *name; void (*fn)(void); } g_tests[] = { +#define T(f) { #f, f } + T(the_figure_is_the_p95_at_the_rate_and_quantum_as_a_fraction_of_a_core), + T(a_rate_or_quantum_the_record_lacks_is_unknown), + T(the_larger_of_two_records_of_one_plugin_stands), + T(a_record_of_another_plugin_is_not_this_ones), + T(a_file_that_is_no_record_is_skipped), + T(a_directory_that_is_not_there_gives_no_figure), +#undef T +}; + +int main(int argc, char **argv) +{ + size_t n, ran = 0; + int bad = 0; + char cmd[300]; + + snprintf(g_dir, sizeof(g_dir), "/tmp/qualcost-test.%d", (int)getpid()); + if (mkdir(g_dir, 0700) != 0) + return 2; + record("org.example.a@1.0-aaaaaaaaaaaa.default.qualification.json", "org.example.a", 48000, 128, 5000); + record("org.example.b@1.0-bbbbbbbbbbbb.default.qualification.json", "org.example.b", 48000, 128, 2000); + record("org.example.b@2.0-cccccccccccc.default.qualification.json", "org.example.b", 48000, 128, 10000); + record("org.example.liar@1.0-dddddddddddd.default.qualification.json", "org.example.other", 48000, 128, 5000); + put("org.example.junk@1.0-eeeeeeeeeeee.default.qualification.json", "{not json"); + put("org.example.empty@1.0-ffffffffffff.default.qualification.json", ""); + for (n = 0; n < sizeof(g_tests) / sizeof(g_tests[0]); n++) + { + if (argc > 1 && strcmp(argv[1], g_tests[n].name)) + continue; + g_failed = 0; + g_tests[n].fn(); + ran++; + if (g_failed) + bad++; + else + printf("ok %s\n", g_tests[n].name); + } + snprintf(cmd, sizeof(cmd), "rm -rf %s", g_dir); + if (system(cmd) != 0) + bad++; + if (!ran) + return 2; + printf("qualcost tests %s (%zu run, %d failed)\n", bad ? "FAILED" : "ok", ran, bad); + return bad ? 1 : 0; +} From e90de1a15116dc72db8d4bff40e56b6842a0d3b5 Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Tue, 27 Oct 2026 10:35:59 +0100 Subject: [PATCH 12/16] Answer plan, plan_get and plan_clear: gather the members, apply the segments as chains, keep the ones that did not change plan reads each member's pin, worker program, world and cost, asks the planner, checks every segment before it makes one, and with grouping on declares each run of two or more as a chain of place chain:. and gives every other member a worker of its own. A plan again keeps the worker of a segment that did not change, wherever it now stands. With grouping off the plan is answered and kept for plan_get and nothing moves. chain_formed names a segment as ., and a plan_reason event carries the reasons when a plan changes. The fuzz recorder, its corpus and the line citations of its records follow the new handlers; the sabotage script breaks each rule of the planner, the reader of the records and the daemon's use of them, and names the test that must go red. --- fuzz/corpus/command/plugin-hostd-plan | Bin 0 -> 11 bytes fuzz/corpus/command/plugin-hostd-plan-extra | Bin 0 -> 19 bytes fuzz/corpus/command/plugin-hostd-plan-full | Bin 0 -> 13 bytes fuzz/corpus/command/plugin-hostd-plan-refused | Bin 0 -> 11 bytes fuzz/corpus/command/plugin-hostd-plan-tabs | Bin 0 -> 13 bytes fuzz/corpus/command/plugin-hostd-plan_clear | Bin 0 -> 15 bytes .../command/plugin-hostd-plan_clear-extra | Bin 0 -> 21 bytes .../command/plugin-hostd-plan_clear-refused | Bin 0 -> 15 bytes .../command/plugin-hostd-plan_clear-tabs | Bin 0 -> 17 bytes fuzz/corpus/command/plugin-hostd-plan_get | Bin 0 -> 13 bytes .../command/plugin-hostd-plan_get-extra | Bin 0 -> 19 bytes .../command/plugin-hostd-plan_get-refused | Bin 0 -> 13 bytes .../corpus/command/plugin-hostd-plan_get-tabs | Bin 0 -> 15 bytes fuzz/fuzz_command.c | 35 +- fuzz/recorder.c | 33 ++ fuzz/recorder.h | 2 +- src/conf.c | 18 + src/conf.h | 2 + src/pins.c | 7 + src/pins.h | 3 + src/supervisor.c | 364 +++++++++++++++++- src/supervisor.h | 9 + src/verbs.c | 27 ++ tests/daemon_test.py | 302 +++++++++++++++ tests/sabotage.py | 213 +++++++++- tests/verbs_declared.py | 3 + 26 files changed, 994 insertions(+), 24 deletions(-) create mode 100644 fuzz/corpus/command/plugin-hostd-plan create mode 100644 fuzz/corpus/command/plugin-hostd-plan-extra create mode 100644 fuzz/corpus/command/plugin-hostd-plan-full create mode 100644 fuzz/corpus/command/plugin-hostd-plan-refused create mode 100644 fuzz/corpus/command/plugin-hostd-plan-tabs create mode 100644 fuzz/corpus/command/plugin-hostd-plan_clear create mode 100644 fuzz/corpus/command/plugin-hostd-plan_clear-extra create mode 100644 fuzz/corpus/command/plugin-hostd-plan_clear-refused create mode 100644 fuzz/corpus/command/plugin-hostd-plan_clear-tabs create mode 100644 fuzz/corpus/command/plugin-hostd-plan_get create mode 100644 fuzz/corpus/command/plugin-hostd-plan_get-extra create mode 100644 fuzz/corpus/command/plugin-hostd-plan_get-refused create mode 100644 fuzz/corpus/command/plugin-hostd-plan_get-tabs diff --git a/fuzz/corpus/command/plugin-hostd-plan b/fuzz/corpus/command/plugin-hostd-plan new file mode 100644 index 0000000000000000000000000000000000000000..a56a835f9362dd821b9466c22f6db44f0464f6c7 GIT binary patch literal 11 ScmZSJU?|8*%u_H_FaQ7yz5-PM literal 0 HcmV?d00001 diff --git a/fuzz/corpus/command/plugin-hostd-plan-extra b/fuzz/corpus/command/plugin-hostd-plan-extra new file mode 100644 index 0000000000000000000000000000000000000000..4e14643074d7280dc8b43610d0cb491070c801e7 GIT binary patch literal 19 YcmZP&U?|8*%u_H_FaW~Tijty404ab3@c;k- literal 0 HcmV?d00001 diff --git a/fuzz/corpus/command/plugin-hostd-plan-full b/fuzz/corpus/command/plugin-hostd-plan-full new file mode 100644 index 0000000000000000000000000000000000000000..5c502750f7cb5821e0c7ff17f074171d070c96d5 GIT binary patch literal 13 UcmZSJVkpQ-%u_H_Fi `text + 1, strlen(text)` + * mutant: src/verbs.c:133 `text, strlen(text) + 1` -> `text + 1, strlen(text)` * input: add-default * mutant: src/verbs.c:90 `return rc;` -> `return SUCCESS;` * input: plugin-hostd-pin_set-refused - * mutant: src/verbs.c:220 `return reply_text;` -> `return sup_resp(SUCCESS);` + * mutant: src/verbs.c:247 `return reply_text;` -> `return sup_resp(SUCCESS);` * input: mod-host-set_bpm-refused */ if (strcmp(g_rec.reply, g_rec.answer)) abort(); @@ -215,7 +215,7 @@ __attribute__((noinline)) static void check_reply_answer(const struct words *w) } /* no call answered: the reply is one the daemon makes itself, a refusal or the reply of quit, which check_reply * holds to "resp " of a declared code and to "resp 0" - * mutant: src/verbs.c:135 `return sup_resp(ERR_INVALID_OPERATION);` -> `return sup_resp(SUCCESS);` + * mutant: src/verbs.c:148 `return sup_resp(ERR_INVALID_OPERATION);` -> `return sup_resp(SUCCESS);` * input: add-too-few */ if (sup_reply_code(g_rec.reply) >= 0 && !(w->n == 1 && verb_is(w->w[0], QUIT))) abort(); @@ -229,12 +229,12 @@ __attribute__((noinline)) static void check_reply(const struct words *w) int code; /* one reply, and nothing sent empty - * mutant: src/verbs.c:238 `reply(msg->sender_id, handle(line, tok, ntok));` -> `free(handle(line, tok, ntok));` + * mutant: src/verbs.c:265 `reply(msg->sender_id, handle(line, tok, ntok));` -> `free(handle(line, tok, ntok));` * input: add-default */ if (g_rec.nreplies != 1 || g_rec.reply_size == 0) abort(); /* the reply is a string that ends where the message the socket sends ends - * mutant: src/verbs.c:120 `strlen(text) + 1` -> `strlen(text)` + * mutant: src/verbs.c:133 `strlen(text) + 1` -> `strlen(text)` * input: add-default */ if (g_rec.reply[g_rec.reply_size - 1] != '\0' || memchr(g_rec.reply, '\0', g_rec.reply_size - 1)) abort(); @@ -252,7 +252,7 @@ __attribute__((noinline)) static void check_reply(const struct words *w) } shape = w->n ? rec_shape_of(w->w[0], strlen(w->w[0])) : REC_MOD_HOST_SHAPE; /* anything else has the shape its verb declares, or mod-host's - * mutant: src/verbs.c:209 `sup_resp(SUCCESS)` -> `sup_resp(1)` + * mutant: src/verbs.c:236 `sup_resp(SUCCESS)` -> `sup_resp(1)` * input: mod-host-quit */ if (!syntax_reply_matches(shape, g_rec.reply) && !syntax_reply_matches(REC_MOD_HOST_SHAPE, g_rec.reply)) abort(); @@ -286,9 +286,9 @@ __attribute__((noinline)) static void check_arity(const struct words *w) else syntax_arity(g_parsed[n].syntax, &min, &max); /* acted on, by its own call or by a broadcast, with an argument count the declaration allows - * mutant: src/verbs.c:175 `ntok == 3` -> `ntok >= 3` + * mutant: src/verbs.c:188 `ntok == 3` -> `ntok >= 3` * input: plugin-hostd-pool_config-extra - * mutant: src/verbs.c:152 `verb_is(verb, EFFECT_REMOVE)` -> `verb_is(verb, EFFECT_REMOVE) && ntok == 2` + * mutant: src/verbs.c:165 `verb_is(verb, EFFECT_REMOVE)` -> `verb_is(verb, EFFECT_REMOVE) && ntok == 2` * input: mod-host-remove-extra */ if (read - 1 < min || read - 1 > max) abort(); @@ -322,6 +322,9 @@ static const char *const g_call_words[REC_FN_COUNT] = { [REC_CHAIN] = "#1 * *", [REC_CHAIN_CLEAR] = "#1", [REC_CHAIN_GET] = "#1", + [REC_PLAN] = "#1 * *", + [REC_PLAN_GET] = "#1", + [REC_PLAN_CLEAR] = "#1", [REC_AFTER_REPLY] = "#1 0", [REC_PINS_SET] = "1 2 3 *", [REC_PINS_CLEAR] = "1", @@ -346,13 +349,13 @@ __attribute__((noinline)) static void check_words_add(const struct rec_call *cal const char *client = read > next ? w[next] : "-"; /* the placement and the client, each where the message has it - * mutant: src/verbs.c:142 `placement, client` -> `client, placement` + * mutant: src/verbs.c:155 `placement, client` -> `client, placement` * input: add-pool-p-fuzz_client */ if (call->nargs != 4 || strcmp(call->args[2], placement) || strcmp(call->args[3], client)) abort(); } -/* the words property for chain [ ...]: the members are word 2 and every word after it, as +/* the words property for chain and plan, [ ...]: the members are word 2 and every word after it, as * numbers, and their count is how many they are */ __attribute__((noinline)) static void check_words_chain(const struct rec_call *call, const char *const *w, int read) { @@ -429,14 +432,14 @@ __attribute__((noinline)) static void check_words(const struct words *w) } item += strcspn(item, " "); /* the argument is the word the declaration puts there, as the controller wrote it - * mutant: src/verbs.c:142 `atoi(tok[2])` -> `atoi(tok[1])` + * mutant: src/verbs.c:155 `atoi(tok[2])` -> `atoi(tok[1])` * input: add-instance-past-the-table */ if (want && (call->args[a] == NULL || strcmp(call->args[a], want))) abort(); } if (call->fn == REC_ADD) check_words_add(call, (const char *const *)w->w, read); - else if (call->fn == REC_CHAIN) + else if (call->fn == REC_CHAIN || call->fn == REC_PLAN) check_words_chain(call, (const char *const *)w->w, read); else if (call->fn == REC_PINS_SET) check_words_pin_set(call, (const char *const *)w->w, read); @@ -450,7 +453,7 @@ __attribute__((noinline)) static void check_refusal(uint64_t before) if (g_rec.reply == NULL || sup_reply_code(g_rec.reply) >= 0) return; /* nothing changed: no accepted call, no follow-up, no quit - * mutant: src/verbs.c:144 `reply_text && sup_reply_code(reply_text) >= 0` -> `reply_text` + * mutant: src/verbs.c:157 `reply_text && sup_reply_code(reply_text) >= 0` -> `reply_text` * input: add-instance-past-the-table */ if (g_rec.state != before || g_quitting) abort(); @@ -485,17 +488,17 @@ __attribute__((noinline)) static void check_round_trip(const struct words *w, ui free(line); /* the same calls with the same arguments - * mutant: src/verbs.c:232 `strtok_r(copy, " \t", &save)` -> `strtok_r(copy, " ", &save)` + * mutant: src/verbs.c:259 `strtok_r(copy, " \t", &save)` -> `strtok_r(copy, " ", &save)` * input: mod-host-add-tabs */ if (!rec_calls_equal(first, nfirst, g_rec.calls, g_rec.ncalls)) abort(); /* the same reply - * mutant: src/verbs.c:206 `ntok != 1` -> `ntok != 1 || line[0] != ' '` + * mutant: src/verbs.c:233 `ntok != 1` -> `ntok != 1 || line[0] != ' '` * input: quit-leading-space */ if (g_rec.reply == NULL || strcmp(g_rec.reply, reply_first)) abort(); /* the same state, the same host settings, and quitting the same - * mutant: src/verbs.c:208 `g_quitting = 1;` -> `g_quitting = line[0] == ' ';` + * mutant: src/verbs.c:235 `g_quitting = 1;` -> `g_quitting = line[0] == ' ';` * input: quit-leading-space */ if (g_quitting != quit_first || g_rec.state != state_first || g_rec.host_settings != host_first) abort(); diff --git a/fuzz/recorder.c b/fuzz/recorder.c index bd14e04..7a5b230 100644 --- a/fuzz/recorder.c +++ b/fuzz/recorder.c @@ -494,6 +494,39 @@ char *sup_chain_get(int id) return answer(c, daemon_shape(OMX_VERB_CHAIN_GET), 1); } +char *sup_plan(int id, const int *ids, int n) +{ + struct rec_call *c = record(REC_PLAN); + char *list = xmalloc((size_t)n * 12 + 1); + size_t o = 0; + int k; + + arg_i(c, id); + arg_i(c, n); + list[0] = '\0'; + for (k = 0; k < n; k++) + o += (size_t)sprintf(list + o, k ? " %d" : "%d", ids[k]); + arg_s(c, list); + free(list); + return answer(c, daemon_shape(OMX_VERB_PLAN), 1); +} + +char *sup_plan_get(int id) +{ + struct rec_call *c = record(REC_PLAN_GET); + + arg_i(c, id); + return answer(c, daemon_shape(OMX_VERB_PLAN_GET), 0); +} + +char *sup_plan_clear(int id) +{ + struct rec_call *c = record(REC_PLAN_CLEAR); + + arg_i(c, id); + return answer(c, daemon_shape(OMX_VERB_PLAN_CLEAR), 1); +} + /* a checkpoint of what a preset or patch changed: it writes down state, it changes none */ void sup_after_reply(int id, const char *verb) { diff --git a/fuzz/recorder.h b/fuzz/recorder.h index f4367d0..e917466 100644 --- a/fuzz/recorder.h +++ b/fuzz/recorder.h @@ -32,7 +32,7 @@ enum { REC_ADD, REC_CALL, REC_REMOVE, REC_REMOVE_ALL, REC_CONNECT, REC_BROADCAST, REC_HOST_SETTING, REC_INSTANCE_VERB, REC_WORKER_LIST, REC_INSTANCE_INFO, REC_RESET, REC_QUARANTINE_CLEAR, REC_POLICY_SET, REC_PIN_ACCEPT, - REC_POOL_CONFIG, REC_LV2_WORKER_SET, REC_WORKER_ENV, REC_CHAIN, REC_CHAIN_CLEAR, REC_CHAIN_GET, REC_AFTER_REPLY, REC_PINS_SET, + REC_POOL_CONFIG, REC_LV2_WORKER_SET, REC_WORKER_ENV, REC_CHAIN, REC_CHAIN_CLEAR, REC_CHAIN_GET, REC_PLAN, REC_PLAN_GET, REC_PLAN_CLEAR, REC_AFTER_REPLY, REC_PINS_SET, REC_PINS_CLEAR, REC_DESCRIPTORS_ADD, REC_DESCRIPTORS_GET, REC_DESCRIPTORS_PANEL_GET, REC_LV2_REFUSALS_FORGET, REC_FN_COUNT }; diff --git a/src/conf.c b/src/conf.c index d664303..053bf89 100644 --- a/src/conf.c +++ b/src/conf.c @@ -124,6 +124,19 @@ int conf_check(const conf_t *conf) return -1; } ok = 0; +#define X(id, name, meaning) ok |= !strcmp(conf->grouping, name); + PHD_GROUPING_MODES(X) +#undef X + if (!ok) + { + fprintf(stderr, "grouping '%s' is not one of:", conf->grouping); +#define X(id, name, meaning) fprintf(stderr, " %s", name); + PHD_GROUPING_MODES(X) +#undef X + fprintf(stderr, "\n"); + return -1; + } + ok = 0; #define X(id, name, meaning) ok |= !strcmp(conf->lv2_worker, name); PHD_LV2_WORKERS(X) #undef X @@ -143,3 +156,8 @@ int conf_pins_first_use(const conf_t *conf) { return !strcmp(conf->pin_mode, PHD_PIN_MODE_FIRST_USE); } + +int conf_grouping_on(const conf_t *conf) +{ + return !strcmp(conf->grouping, PHD_GROUPING_ON); +} diff --git a/src/conf.h b/src/conf.h index 42fdd75..9e87b28 100644 --- a/src/conf.h +++ b/src/conf.h @@ -47,5 +47,7 @@ int conf_load(conf_t *conf, const char *path, int required); int conf_check(const conf_t *conf); /* whether pin_mode is first-use */ int conf_pins_first_use(const conf_t *conf); +/* whether grouping is on */ +int conf_grouping_on(const conf_t *conf); #endif diff --git a/src/pins.c b/src/pins.c index 85cc54a..b55bd84 100644 --- a/src/pins.c +++ b/src/pins.c @@ -628,6 +628,13 @@ int pins_in_process(const char *uri) return p && !strcmp(p->verdict, PHD_PIN_VERDICT_IN_PROCESS); } +const char *pins_verdict(const char *uri) +{ + const pin_t *p = find(bare(uri)); + + return p ? p->verdict : NULL; +} + int pins_clear(const char *uri) { pin_t *p; diff --git a/src/pins.h b/src/pins.h index b18acba..ad0810b 100644 --- a/src/pins.h +++ b/src/pins.h @@ -48,6 +48,9 @@ int pins_clear(const char *uri); /* whether the uri has a pin, and whether it is one a verdict of in-process qualified */ int pins_known(const char *uri); int pins_in_process(const char *uri); +/* the verdict of the uri's pin as the pin line spells it (one of PHD_PIN_VERDICTS), "" for a pin without one, NULL when + * there is no pin; valid until the next pin verb */ +const char *pins_verdict(const char *uri); /* first use: the files the format loads for the uri, hashed, recorded with an unknown layout and written to the store. * SUCCESS, or PHD_ERR_PIN_BINARY_MISMATCH when they cannot be collected (nothing is recorded) */ diff --git a/src/supervisor.c b/src/supervisor.c index 183e069..757751c 100644 --- a/src/supervisor.c +++ b/src/supervisor.c @@ -38,6 +38,8 @@ /* mod-host's pseudo port of the host's own bypass, which param_info names no parameter */ #include "pins.h" +#include "planner.h" +#include "qualcost.h" #include "relay.h" #include "supervisor.h" #include "verbs.h" @@ -72,8 +74,30 @@ static worker_env_t g_env[FORMATS_MAX]; static chain_t *g_chain[MAX_INSTANCE]; static int g_nchains; +/* what plan last answered for a chain: its segments, and the chain of each segment of two or more that runs */ +typedef struct PLAN_T { + int id; + int n; + int *ids; /* the members, in the order plan was given them */ + chain_t **seg; /* the segments of two or more members that run, in plan order */ + int nseg; + char *segments; /* plan's answer after resp 0 */ + char *reasons; /* plan_get's second word */ +} plan_t; +static plan_t *g_plan[MAX_INSTANCE]; +static int g_nplans; + static void chain_form(chain_t *c); +/* a chain as the controller names it: the number chain declared, or . for segment k of a plan */ +static void chain_word(const chain_t *c, char *word, size_t size) +{ + if (c->planned) + snprintf(word, size, "%d" PHD_PLACE_PLAN_SEPARATOR "%d", c->id, c->seg); + else + snprintf(word, size, "%d", c->id); +} + static const char *const g_state_states[W_COUNT] = { #define X(id, name, meaning) name, PHD_WORKER_STATES(X) @@ -1048,7 +1072,7 @@ static int replay_conns(worker_t *w, instance_t *i) /* chain_formed: the order a chain's worker runs now, the members it holds and replayed */ static void chain_formed(worker_t *w) { - char ids[1024]; + char ids[1024], word[24]; size_t used = 0; int n; @@ -1056,7 +1080,8 @@ static void chain_formed(worker_t *w) for (n = 0; n < w->ninst && used + 8 < sizeof(ids); n++) if (!w->inst[n]->unreplayed) used += snprintf(ids + used, sizeof(ids) - used, "%s%d", used ? "," : "", w->inst[n]->id); - event(PHD_EVENT_CHAIN_FORMED_FMT, w->chain->id, w->k, used ? ids : PHD_WORD_NONE); + chain_word(w->chain, word, sizeof(word)); + event(PHD_EVENT_CHAIN_FORMED_FMT, word, w->k, used ? ids : PHD_WORD_NONE); } static void replay(worker_t *w, int64_t started) @@ -1163,6 +1188,19 @@ void sup_tick(void) if (!chain_whole(g_chain[n])) chain_form(g_chain[n]); } + for (n = 0; g_nplans && n < MAX_INSTANCE; n++) + { + int k; + + /* chain_form retires and makes workers, not segments: the list holds while it runs */ + for (k = 0; g_plan[n] && k < g_plan[n]->nseg; k++) + if (g_plan[n]->seg[k]->rejoin) + { + g_plan[n]->seg[k]->rejoin = 0; + if (!chain_whole(g_plan[n]->seg[k])) + chain_form(g_plan[n]->seg[k]); + } + } } /* ---------------------------------------------------------------- init and finish */ @@ -1512,8 +1550,20 @@ static void chain_free(chain_t *c) g_inst[c->ids[n]]->chain = NULL; if (c->w) c->w->chain = NULL; - g_chain[c->id] = NULL; - g_nchains--; + if (c->planned) + { + plan_t *p = g_plan[c->id]; + int k; + + for (k = 0; p && k < p->nseg; k++) + if (p->seg[k] == c) + p->seg[k] = p->seg[--p->nseg]; + } + else + { + g_chain[c->id] = NULL; + g_nchains--; + } free(c->ids); free(c); } @@ -1938,8 +1988,14 @@ char *sup_quarantine_clear(const char *which) if (g_meta[id]) g_meta[id]->quarantined = 0; for (id = 0; id < MAX_INSTANCE; id++) + { + int k; + if (g_chain[id]) g_chain[id]->rejoin = 1; + for (k = 0; g_plan[id] && k < g_plan[id]->nseg; k++) + g_plan[id]->seg[k]->rejoin = 1; + } return sup_resp(SUCCESS); } id = atoi(which); @@ -2092,11 +2148,12 @@ static void to_own(instance_t *i) /* the chain's members that are not quarantined, gathered in order in one new worker and replayed there */ static void chain_form(chain_t *c) { - char place[48]; + char place[48], word[24]; worker_t *w = NULL; int n; - snprintf(place, sizeof(place), PHD_PLACE_CHAIN_PREFIX "%d", c->id); + chain_word(c, word, sizeof(word)); + snprintf(place, sizeof(place), PHD_PLACE_CHAIN_PREFIX "%s", word); for (n = 0; n < c->n; n++) { instance_t *i = g_inst[c->ids[n]]; @@ -2339,3 +2396,298 @@ char *sup_chain_get(int id) put(&buf, &used, &cap, PHD_WORD_NONE); return buf; } + +/* ---------------------------------------------------------------- plans */ + +/* the id the qualification records name a plugin by: the CLAP id after the '#' of clap:#, else the uri */ +static const char *record_id(const char *uri) +{ + const char *hash = strrchr(uri, '#'); + + return hash ? hash + 1 : uri; +} + +/* what the planner is told of an instance: its worker program's class, its pin, whether a world can be built for it, its cost */ +static void plan_member(const instance_t *i, planner_member_t *m) +{ + const format_t *f = format_at(i->fmt); + const char *bin = bin_of(i->fmt), *verdict = pins_verdict(i->uri); + + memset(m, 0, sizeof(*m)); + m->id = i->id; + m->klass = f->native_chain && bin ? bin : f->name; + if (!verdict) + m->pin = PLANNER_PIN_NONE; + else if (!strcmp(verdict, PHD_PIN_VERDICT_IN_PROCESS)) + m->pin = PLANNER_PIN_IN_PROCESS; + else if (!strcmp(verdict, PHD_PIN_VERDICT_ISOLATED)) + m->pin = PLANNER_PIN_ISOLATED; + else if (!strcmp(verdict, PHD_PIN_VERDICT_REFUSED)) + m->pin = PLANNER_PIN_REFUSED; + else + m->pin = PLANNER_PIN_NO_VERDICT; + m->world_ok = chain_world_found(&m->id, 1); + m->cost = g_conf.group_rate > 0 && g_conf.group_quantum > 0 + ? qualcost_read(g_conf.qualification_dir, record_id(i->uri), g_conf.group_rate, g_conf.group_quantum) + : PLANNER_COST_UNKNOWN; + if (m->cost < 0) + m->cost = PLANNER_COST_UNKNOWN; + m->history = NULL; +} + +/* the refusal chain would give the members of a segment of two or more, apart from the ones of the numbers: 0 for none */ +static int segment_refusal(const int *ids, int n) +{ + int k; + + for (k = 0; k < n; k++) + { + if (!chain_rows_together(g_inst[ids[k]]->fmt, g_inst[ids[0]]->fmt)) + return PHD_ERR_CHAIN_MIXED_FORMAT; + if (must_run_own(g_inst[ids[k]])) + return PHD_ERR_PLACEMENT_INVALID; + } + return chain_world_found(ids, n) ? 0 : PHD_ERR_PLACEMENT_INVALID; +} + +static void plan_free(plan_t *p) +{ + if (!p) + return; + free(p->ids); + free(p->seg); + free(p->segments); + free(p->reasons); + free(p); +} + +/* a member is alone in a worker of its own, or is made so */ +static void make_alone(instance_t *i) +{ + if (i->w && i->w->ninst == 1 && !i->w->pool && !i->w->chain) + return; + to_own(i); +} + +/* a text of the planner's, malloc'd */ +static char *plan_text(size_t (*format)(const planner_member_t *, const planner_plan_t *, int, char *, size_t), + const planner_member_t *m, const planner_plan_t *pp, int refused) +{ + size_t need = format(m, pp, refused, NULL, 0); + char *text = malloc(need + 1); + + format(m, pp, refused, text, need + 1); + return text; +} + +static size_t segments_text(const planner_member_t *m, const planner_plan_t *pp, int refused, char *buf, size_t size) +{ + (void)refused; + return planner_format_segments(m, pp, buf, size); +} + +/* the plan p, made over the ids, is applied: the running segment of the old plan that did not change is kept, wherever + * it now stands, the others are dissolved, a member that shares with no one gets a worker of its own, and a segment that + * is new forms in a worker of its own. p->seg holds the segments of two or more, in plan order. */ +static void plan_apply(plan_t *old, plan_t *p, const planner_plan_t *pp, const int *ids) +{ + chain_t **at = calloc(pp->nseg, sizeof(chain_t *)), **was = NULL; + char *kept = NULL, *formed = calloc(pp->nseg, 1); + int k, j, first, nwas = old ? old->nseg : 0; + + if (nwas) + { + was = malloc(nwas * sizeof(chain_t *)); + memcpy(was, old->seg, nwas * sizeof(chain_t *)); + kept = calloc(nwas, 1); + } + for (k = 0, first = 0; k < pp->nseg; first += pp->seg_len[k++]) + for (j = 0; pp->seg_len[k] > 1 && j < nwas && !at[k]; j++) + if (!kept[j] && was[j]->n == pp->seg_len[k] && !memcmp(was[j]->ids, ids + first, pp->seg_len[k] * sizeof(int))) + { + kept[j] = 1; + at[k] = was[j]; + at[k]->seg = k; + if (at[k]->w) + { + char word[24]; + + chain_word(at[k], word, sizeof(word)); + snprintf(at[k]->w->place, sizeof(at[k]->w->place), PHD_PLACE_CHAIN_PREFIX "%s", word); + } + } + for (j = 0; j < nwas; j++) + if (!kept[j]) + { + chain_t *c = was[j]; + + /* a member the new plan leaves out goes back to a worker of its own, as one chain leaves out does */ + for (k = 0; k < c->n; k++) + { + instance_t *i = g_inst[c->ids[k]]; + int in_plan = 0, u; + + for (u = 0; u < p->n; u++) + in_plan |= ids[u] == c->ids[k]; + if (i && !in_plan) + { + i->chain = NULL; + if (c->w && i->w == c->w) + to_own(i); + } + } + chain_free(c); + } + free(was); + free(kept); + for (k = 0, first = 0; k < pp->nseg; first += pp->seg_len[k++]) + if (pp->seg_len[k] == 1) + make_alone(g_inst[ids[first]]); + for (k = 0, first = 0; k < pp->nseg; first += pp->seg_len[k++]) + if (pp->seg_len[k] > 1 && !at[k]) + { + chain_t *c = calloc(1, sizeof(chain_t)); + + c->id = p->id; + c->planned = 1; + c->seg = k; + c->n = pp->seg_len[k]; + c->ids = malloc(c->n * sizeof(int)); + memcpy(c->ids, ids + first, c->n * sizeof(int)); + for (j = 0; j < c->n; j++) + g_inst[c->ids[j]]->chain = c; + at[k] = c; + formed[k] = 1; + } + for (k = 0; k < pp->nseg; k++) + if (at[k]) + p->seg[p->nseg++] = at[k]; + /* the new plan stands before its segments form: a member that leaves one finds it there */ + g_plan[p->id] = p; + for (k = 0; k < pp->nseg; k++) + if (formed[k]) + chain_form(at[k]); + free(formed); + free(at); +} + +char *sup_plan(int id, const int *ids, int n) +{ + plan_t *old = (id >= 0 && id < MAX_INSTANCE) ? g_plan[id] : NULL, *p; + planner_member_t m[PLANNER_MEMBERS_MAX]; + planner_settings_t s; + planner_plan_t pp; + int k, j, first, refused = 0; + char *reply; + size_t used = 0, cap = 64; + + if (id < 0 || id >= MAX_INSTANCE || n < PHD_CHAIN_MEMBERS_MIN || n > PLANNER_MEMBERS_MAX) + return sup_resp(ERR_INVALID_OPERATION); + for (k = 0; k < n; k++) + { + instance_t *i = (ids[k] >= 0 && ids[k] < MAX_INSTANCE) ? g_inst[ids[k]] : NULL; + + if (!i) + return sup_resp(PHD_ERR_CHAIN_UNKNOWN_INSTANCE); + for (j = 0; j < k; j++) + if (ids[j] == ids[k]) + return sup_resp(PHD_ERR_CHAIN_DUPLICATE); + /* the member of a chain, or of the plan of another chain, is not this plan's to place */ + if (i->chain && !(i->chain->planned && i->chain->id == id)) + return sup_resp(PHD_ERR_CHAIN_INSTANCE_BUSY); + plan_member(i, &m[k]); + } + s.require_pins = g_conf.require_pins; + s.pins_first_use = conf_pins_first_use(&g_conf); + s.grouping_on = conf_grouping_on(&g_conf); + s.group_max = g_conf.group_max; + s.group_cost_max = g_conf.group_cost_max / 100.0; + if (planner_plan(&s, m, n, &pp) != 0) + return sup_resp(ERR_INVALID_OPERATION); + + /* every segment is checked before any is made: a plan that cannot be formed moves nothing */ + for (k = 0, first = 0; s.grouping_on && !refused && k < pp.nseg; first += pp.seg_len[k++]) + if (pp.seg_len[k] > 1) + refused = segment_refusal(ids + first, pp.seg_len[k]); + + p = calloc(1, sizeof(*p)); + p->id = id; + p->n = n; + p->ids = malloc(n * sizeof(int)); + memcpy(p->ids, ids, n * sizeof(int)); + p->seg = calloc(pp.nseg, sizeof(chain_t *)); + p->segments = plan_text(segments_text, m, &pp, 0); + p->reasons = plan_text(planner_format_reasons, m, &pp, refused); + + if (s.grouping_on && !refused) + plan_apply(old, p, &pp, ids); + else + { + /* nothing moves: the segments that run stay as they are, and the plan is kept to be read back */ + if (old) + { + memcpy(p->seg, old->seg, old->nseg * sizeof(chain_t *)); + p->nseg = old->nseg; + } + g_plan[id] = p; + } + if (!old || strcmp(old->segments, p->segments) || strcmp(old->reasons, p->reasons)) + event(PHD_EVENT_PLAN_REASON_FMT, id, p->reasons); + if (old) + { + /* the old plan's list is dropped, not its chains: they are p's now, or were freed */ + old->nseg = 0; + plan_free(old); + } + else + g_nplans++; + if (refused) + return sup_resp(refused); + reply = malloc(cap); + put(&reply, &used, &cap, "resp 0 %s", p->segments); + return reply; +} + +char *sup_plan_get(int id) +{ + plan_t *p = (id >= 0 && id < MAX_INSTANCE) ? g_plan[id] : NULL; + size_t used = 0, cap = 64; + char *buf; + + if (!p) + return sup_resp(PHD_ERR_NO_SUCH_PLAN); + buf = malloc(cap); + put(&buf, &used, &cap, "resp 0 %s %s", p->segments, p->reasons); + return buf; +} + +char *sup_plan_clear(int id) +{ + plan_t *p = (id >= 0 && id < MAX_INSTANCE) ? g_plan[id] : NULL; + int j; + + if (!p) + return sup_resp(PHD_ERR_NO_SUCH_PLAN); + while (p->nseg) + { + chain_t *c = p->seg[p->nseg - 1]; + + for (j = 0; j < c->n; j++) + { + instance_t *i = g_inst[c->ids[j]]; + + if (!i) + continue; + i->chain = NULL; + /* one already alone, quarantined or split, stays where it is */ + if (c->w && i->w == c->w) + to_own(i); + } + chain_free(c); + } + /* the members that share with no one are alone already: a plan that is cleared leaves them where they stand */ + g_plan[id] = NULL; + g_nplans--; + plan_free(p); + return sup_resp(SUCCESS); +} diff --git a/src/supervisor.h b/src/supervisor.h index 4ef297d..eb73e92 100644 --- a/src/supervisor.h +++ b/src/supervisor.h @@ -104,6 +104,9 @@ typedef struct CHAIN_T { int n; worker_t *w; /* NULL while no worker holds it */ int rejoin; /* its members are to be gathered in one worker again, by the tick */ + int planned; /* a segment of a plan, not a chain the controller declared: id is the plan's number, seg the + * segment's place in the plan, and no g_chain row holds it */ + int seg; } chain_t; void sup_init(const conf_t *conf); @@ -146,6 +149,12 @@ char *sup_chain(int id, const int *ids, int n); char *sup_chain_clear(int id); /* chain_get: the chain's worker, its members as declared and the ones its worker runs, each in order */ char *sup_chain_get(int id); +/* plan: the instances ids, in signal order, planned into segments (src/planner.c); applied as chains of place + * chain:. and workers of their own while grouping is on, answered and kept while it is off. plan_get reads the + * last plan back, plan_clear gives each member a worker of its own */ +char *sup_plan(int id, const int *ids, int n); +char *sup_plan_get(int id); +char *sup_plan_clear(int id); /* the idle tick: reap the dead, respawn the due, checkpoint the quiet */ void sup_tick(void); diff --git a/src/verbs.c b/src/verbs.c index 4dc630a..04c1781 100644 --- a/src/verbs.c +++ b/src/verbs.c @@ -115,6 +115,19 @@ static char *chain_words(char **tok, int ntok) return sup_chain(id, ids, ntok - 2); } +/* plan [ ...]: at least PHD_CHAIN_MEMBERS_MIN members */ +static char *plan_words(char **tok, int ntok) +{ + int id, ids[MAX_TOKENS], n; + + if (ntok < 2 + PHD_CHAIN_MEMBERS_MIN || !number(tok[1], &id)) + return sup_resp(ERR_INVALID_OPERATION); + for (n = 2; n < ntok; n++) + if (!number(tok[n], &ids[n - 2])) + return sup_resp(ERR_INVALID_OPERATION); + return sup_plan(id, ids, ntok - 2); +} + static void reply(int fd, char *text) { socket_send(fd, text, strlen(text) + 1); @@ -189,6 +202,20 @@ static char *handle(char *line, char **tok, int ntok) return ntok == 2 && number(tok[1], &id) ? sup_chain_get(id) : sup_resp(ERR_INVALID_OPERATION); } + if (!strcmp(verb, OMX_VERB_PLAN)) + return plan_words(tok, ntok); + if (!strcmp(verb, OMX_VERB_PLAN_GET)) + { + int id; + + return ntok == 2 && number(tok[1], &id) ? sup_plan_get(id) : sup_resp(ERR_INVALID_OPERATION); + } + if (!strcmp(verb, OMX_VERB_PLAN_CLEAR)) + { + int id; + + return ntok == 2 && number(tok[1], &id) ? sup_plan_clear(id) : sup_resp(ERR_INVALID_OPERATION); + } if (!strcmp(verb, OMX_VERB_PIN_SET)) return sup_resp(pin_set_words(line, tok, ntok)); if (!strcmp(verb, OMX_VERB_PIN_ACCEPT)) diff --git a/tests/daemon_test.py b/tests/daemon_test.py index 815b159..fe13503 100644 --- a/tests/daemon_test.py +++ b/tests/daemon_test.py @@ -2902,6 +2902,308 @@ def a_chain_declared_again_keeps_its_running_worker_and_restarts_no_survivor(): os.unlink(log) +# ---------------------------------------------------------------- plan (the grouping planner, src/planner.c) + +def plan_of(d, plan): + """{k: worker} of the segment workers of a plan, place chain:., by the segment's k""" + prefix = "chain:%d." % plan + return {int(w["place"][len(prefix):]): dict(w, name=name) for name, w in d.workers().items() if w["place"].startswith(prefix)} + + +def clap_n(p, n): + """the uri of the n-th plugin of one file: five plugins in one .clap, each with a pin of its own""" + return "clap:%s#org.openmixer.fault.%d" % (p.clap, n) + + +def pin_n(p, n, verdict=None): + return "pin_set %s omx-fault.clap=%s %s%s" % (clap_n(p, n), sha(p.clap), fake_layout(), " verdict=" + verdict if verdict else "") + + +def record_for(d, plugin_id, ns_p95, rate=48000, quantum=128, name=None): + """a qualification record of plugin-qualify, as far as the daemon reads one: the cost of one grid point""" + point = {"rate": rate, "quantum": quantum, "outcome": "probe.ok", + "cost": {"median": ns_p95 / 2, "percentile": {"p": 95, "value": ns_p95}, "blocks": 100, "warmupBlocks": 10}} + doc = {"schema": "plugin-qualify.qualification-record/1", "plugin": {"format": "clap", "id": plugin_id}, + "measurements": {"grid": [point]}} + path = os.path.join(d.tmp, "qualification", name or "%s@1.0-0123456789ab.default.qualification.json" % plugin_id) + with open(path, "w") as f: + json.dump(doc, f) + + +def plan_run(conf=None): + """five pinned plugins of one file under first-use, 3 of verdict isolated and 6 never given a verdict, a daemon with + grouping on""" + p = Plugins() + c = {"pin_mode": "first-use", "grouping": "on"} + c.update(conf or {}) + d = p.daemon(conf=c) + for n in (1, 2, 4, 5): + d.expect(pin_n(p, n, "in-process"), "resp 0") + d.expect(pin_n(p, 3, "isolated"), "resp 0") + for n in range(1, 7): + d.expect("add %s %d" % (clap_n(p, n), n), "resp %d" % n) + return p, d + + +@test +def a_plan_forms_the_runs_that_may_share_and_leaves_the_member_that_may_not_in_a_worker_of_its_own(): + """A B C D E with C isolated is [A B] [C] [D E]: chain workers of place chain:1.0 and chain:1.2 (k counts the segments of + the plan, the lone one too), and C own""" + p, d = plan_run() + try: + for n in range(1, 6): + d.expect("param_set %d gain 0.%d000" % (n, n), "resp 0") + mark = d.mark() + d.expect("plan 1 1 2 3 4 5", "resp 0 1,2;3;4,5") + e0 = d.wait_event("chain_formed 1.0 ", since=mark) + e1 = d.wait_event("chain_formed 1.2 ", since=mark) + segs = plan_of(d, 1) + check(sorted(segs) == [0, 2] and segs[0]["inst"] == [1, 2] and segs[2]["inst"] == [4, 5], "two chain workers: %s" % d.workers()) + check(e0 == "chain_formed 1.0 %s 1,2" % segs[0]["name"] and e1 == "chain_formed 1.2 %s 4,5" % segs[2]["name"], + "the events name the segments: %s / %s" % (e0, e1)) + _, w3 = d.wait_up(3) + check(w3["place"] == "own" and w3["inst"] == [3], "the isolated member is alone: %s" % w3) + gains(d, (1, 2, 3, 4, 5)) + d.expect("plan_get 1", "resp 0 1,2;3;4,5 1:cost-assumed,2:cost-assumed,3:verdict-isolated,4:cost-assumed,5:cost-assumed") + check(d.wait_event("plan_reason 1 ", since=mark) == "plan_reason 1 1:cost-assumed,2:cost-assumed,3:verdict-isolated," + "4:cost-assumed,5:cost-assumed", "the plan_reason event carries the reasons") + # a member that was never given a verdict is alone as well + d.expect("plan 2 6", "resp 0 6") + d.expect("plan_get 2", "resp 0 6 6:no-verdict") + d.expect("plan 3 5", resp("CHAIN_INSTANCE_BUSY")) + finally: + d.close() + p.close() + + +@test +def a_plan_again_keeps_the_worker_of_a_segment_that_did_not_change_and_forms_the_ones_that_did(): + p, d = plan_run() + try: + d.expect("plan 1 1 2 3 4 5", "resp 0 1,2;3;4,5") + d.wait_event("chain_formed 1.2 ") + segs = plan_of(d, 1) + first, second = segs[0], segs[2] + d.expect("plan 1 1 2 3 4 5", "resp 0 1,2;3;4,5") + again = plan_of(d, 1) + check(again[0]["pid"] == first["pid"] and again[2]["pid"] == second["pid"] and again[0]["name"] == first["name"], + "the same plan moves nothing: %s" % d.workers()) + # the plan loses its last two members: [1 2] stays where it runs, 4 and 5 are alone + d.expect("plan 1 1 2 3", "resp 0 1,2;3") + now = plan_of(d, 1) + check(sorted(now) == [0] and now[0]["pid"] == first["pid"], "the segment that did not change keeps its worker: %s" % d.workers()) + for n in (4, 5): + _, w = d.wait_up(n) + check(w["place"] == "own" and w["inst"] == [n], "%d is alone: %s" % (n, w)) + # 3 earns an in-process verdict: all of 1 2 3 are one segment, in a worker that is formed again + d.expect(pin_n(p, 3, "in-process"), "resp 0") + mark = d.mark() + d.expect("plan 1 1 2 3", "resp 0 1,2,3") + d.wait_event("chain_formed 1.0 ", since=mark) + now = plan_of(d, 1) + check(sorted(now) == [0] and now[0]["inst"] == [1, 2, 3], "one segment of three: %s" % d.workers()) + # a segment that moves to another place in the plan keeps its worker and takes the place's name + d.expect("plan 1 4 1 2 3", "resp 0 4,1,2,3") + moved = plan_of(d, 1) + check(sorted(moved) == [0] and moved[0]["inst"] == [4, 1, 2, 3], "four in one: %s" % d.workers()) + d.expect(pin_n(p, 4, "isolated"), "resp 0") + d.expect("plan 1 4 1 2 3", "resp 0 4;1,2,3") + moved2 = plan_of(d, 1) + check(sorted(moved2) == [1] and moved2[1]["inst"] == [1, 2, 3], + "the segment of 1 2 3 is the plan's second now and its worker is named for it: %s" % d.workers()) + check(any(w["place"] == "own" and w["inst"] == [4] for w in d.workers().values()), "and 4 is alone: %s" % d.workers()) + finally: + d.close() + p.close() + + +@test +def a_plan_that_keeps_a_segment_in_another_place_renames_its_worker_and_restarts_nothing(): + p, d = plan_run() + try: + d.expect(pin_n(p, 1, "isolated"), "resp 0") + d.expect("plan 1 2 3 4 5", "resp 0 2;3;4,5") + d.wait_event("chain_formed 1.2 ") + was = plan_of(d, 1)[2] + check(was["inst"] == [4, 5], "4 5 is the plan's third segment: %s" % d.workers()) + d.expect("plan 1 1 2 3 4 5", "resp 0 1;2;3;4,5") + now = plan_of(d, 1) + check(sorted(now) == [3] and now[3]["pid"] == was["pid"] and now[3]["inst"] == [4, 5], + "the same worker, named for the place it stands in now: %s" % d.workers()) + finally: + d.close() + p.close() + + +@test +def while_grouping_is_off_a_plan_is_answered_and_kept_and_nothing_moves(): + d = daemon() + try: + for i in (1, 2, 3): + d.expect("add clap:fake#m%d %d" % (i, i), "resp %d" % i) + before = d.workers() + d.expect("plan 1 1 2 3", "resp 0 1,2,3") + check(d.workers() == before and not any(w["place"].startswith("chain:") for w in d.workers().values()), + "nothing moved: %s" % d.workers()) + d.expect("plan_get 1", "resp 0 1,2,3 1:grouping-off,2:grouping-off,3:grouping-off") + d.expect("plan_clear 1", "resp 0") + check(d.workers() == before, "and clearing it moves nothing: %s" % d.workers()) + d.expect("plan_get 1", resp("NO_SUCH_PLAN")) + finally: + d.close() + + +@test +def a_setting_that_is_none_of_the_grouping_modes_stops_the_daemon_and_names_the_modes(): + p = Plugins() + try: + try: + d = p.daemon(conf={"grouping": "maybe"}) + except Fail as e: + check("grouping 'maybe' is not one of: off on" in str(e), "stderr names the modes: %s" % e) + else: + d.close() + raise Fail("a daemon started with grouping maybe") + finally: + p.close() + + +@test +def a_plan_is_refused_for_what_chain_refuses_and_moves_nothing(): + d = daemon(conf={"grouping": "on"}) + try: + for i in (1, 2, 3): + d.expect("add clap:fake#m%d %d" % (i, i), "resp %d" % i) + before = d.workers() + d.expect("plan 1 1 2 77", resp("CHAIN_UNKNOWN_INSTANCE")) + d.expect("plan 1 1 2 1", resp("CHAIN_DUPLICATE")) + for bad in ("plan 1", "plan x 1 2", "plan 1 1 two", "plan -1 1 2", "plan_get", "plan_get x", "plan_clear", "plan_clear x"): + d.expect(bad, "resp -902") + d.expect("plan_get 1", resp("NO_SUCH_PLAN")) + d.expect("plan_clear 1", resp("NO_SUCH_PLAN")) + check(d.workers() == before, "a refused plan moves nothing: %s" % d.workers()) + # a member of a chain, or of another chain's plan, is not the plan's to place + d.expect("chain 9 1 2", "resp 0") + d.expect("plan 1 2 3", resp("CHAIN_INSTANCE_BUSY")) + d.expect("chain_clear 9", "resp 0") + d.expect("plan 1 2 3", "resp 0 2,3") + d.expect("plan 2 3", resp("CHAIN_INSTANCE_BUSY")) + d.expect("chain 9 3 1", resp("CHAIN_INSTANCE_BUSY")) + # a plan and a chain of one number are two things + d.expect("chain 1 1", "resp 0") + d.expect("chain_get 1", "resp 0 " + [k for k, w in d.workers().items() if w["place"] == "chain:1"][0] + " 1 1") + d.expect("plan_get 1", "resp 0 2,3 2:cost-assumed,3:cost-assumed") + finally: + d.close() + + +@test +def plan_clear_gives_every_member_a_worker_of_its_own_and_forgets_the_plan(): + d = daemon(conf={"grouping": "on"}) + try: + for i in (1, 2, 3): + d.expect("add clap:fake#m%d %d" % (i, i), "resp %d" % i) + d.expect("param_set %d gain 0.%d000" % (i, i), "resp 0") + mark = d.mark() + d.expect("plan 4 1 2 3", "resp 0 1,2,3") + d.wait_event("chain_formed 4.0 ", since=mark) + check(plan_of(d, 4)[0]["inst"] == [1, 2, 3], "one worker: %s" % d.workers()) + d.expect("plan_clear 4", "resp 0") + for i in (1, 2, 3): + _, w = d.wait_up(i) + check(w["place"] == "own" and w["inst"] == [i], "plan_clear gives %d a worker of its own: %s" % (i, w)) + check(plan_of(d, 4) == {}, "the segment worker is gone: %s" % d.workers()) + gains(d, (1, 2, 3)) + d.expect("plan_get 4", resp("NO_SUCH_PLAN")) + d.expect("plan_clear 4", resp("NO_SUCH_PLAN")) + d.expect("chain 4 1 2", "resp 0") + finally: + d.close() + + +@test +def a_member_that_is_removed_leaves_its_segment_and_the_worker_is_told_the_order_without_it(): + log = tempfile.mktemp(prefix="plugin-hostd-fake-log.") + d = daemon(conf={"grouping": "on"}, env={"FAKE_LOG": log}) + try: + for i in (1, 2, 3): + d.expect("add clap:fake#m%d %d" % (i, i), "resp %d" % i) + d.expect("plan 1 1 2 3", "resp 0 1,2,3") + w = plan_of(d, 1)[0] + d.expect("remove 2", "resp 0") + check(plan_of(d, 1)[0]["inst"] == [1, 3], "the segment goes on without it: %s" % d.workers()) + check(sent_to(log, w["pid"])[-1] == "chain 1 1 3", "the worker is told: %s" % sent_to(log, w["pid"])[-3:]) + d.expect("remove 1", "resp 0") + d.expect("remove 3", "resp 0") + check(plan_of(d, 1) == {}, "no segment is left: %s" % d.workers()) + d.expect("plan_clear 1", "resp 0") + finally: + d.close() + if os.path.exists(log): + os.unlink(log) + + +@test +def a_segment_whose_worker_dies_is_respawned_with_its_members_in_order_and_its_plan_is_still_read_back(): + d = daemon(conf={"grouping": "on"}) + try: + for i in (10, 11, 12): + d.expect("add clap:fake#m%d %d" % (i, i), "resp %d" % i) + d.expect("plan 7 10 11 12", "resp 0 10,11,12") + d.wait_event("chain_formed 7.0 ") + os.kill(plan_of(d, 7)[0]["pid"], signal.SIGKILL) + _, w = d.wait_up(10) + check(w["place"] == "chain:7.0" and w["inst"] == [10, 11, 12], "the same place, the same order: %s" % d.workers()) + d.expect("plan_get 7", "resp 0 10,11,12 10:cost-assumed,11:cost-assumed,12:cost-assumed") + finally: + d.close() + + +@test +def the_cost_a_qualification_record_holds_decides_the_segments_and_a_missing_one_is_assumed(): + """a record's p95 nanoseconds per sample at group_rate and group_quantum is a fraction of a core: 5000 ns at 48 kHz is + 24 %, and the segment adds them up against group_cost_max""" + p, d = plan_run(conf={"group_cost_max": 50, "group_rate": 48000, "group_quantum": 128, "group_max": 3}) + try: + for n in (1, 2, 4, 5): + d.expect(pin_n(p, n, "in-process"), "resp 0") + d.expect(pin_n(p, 3, "in-process"), "resp 0") + record_for(d, "org.openmixer.fault.1", 5000) + record_for(d, "org.openmixer.fault.2", 5000) + record_for(d, "org.openmixer.fault.3", 5000) + record_for(d, "org.openmixer.fault.4", 12500) # 60 %: over the limit alone + record_for(d, "org.openmixer.fault.5", 5000, rate=96000) # another rate: not this plan's figure + d.expect("plan 1 1 2 3 4 5", "resp 0 1,2;3;4;5") + d.expect("plan_get 1", "resp 0 1,2;3;4;5 3:group-cost,4:cost-over,5:no-neighbour") + # a record of the plugin at a lower cost, in another version, is not preferred: the larger figure stands + record_for(d, "org.openmixer.fault.2", 1000, name="org.openmixer.fault.2@0.9-fedcba987654.default.qualification.json") + d.expect("plan 1 1 2 3 4 5", "resp 0 1,2;3;4;5") + # the plugin's figure lowered, in the record that held it: 24 + 24 + 0.5 % fits 50 + record_for(d, "org.openmixer.fault.3", 100, name="org.openmixer.fault.3@1.0-0123456789ab.default.qualification.json") + d.expect("plan 1 1 2 3", "resp 0 1,2,3") + d.expect("plan_get 1", "resp 0 1,2,3 -") + finally: + d.close() + p.close() + + +@test +def the_same_members_are_planned_into_the_same_segments_and_reasons_by_every_daemon_and_every_time(): + answers = [] + for _ in range(2): + p, d = plan_run() + try: + run = [] + for _ in range(3): + d.expect("plan 1 1 2 3 4 5", "resp 0 1,2;3;4,5") + run.append(d.send("plan_get 1")) + answers.append(run) + finally: + d.close() + p.close() + check(len(set(answers[0] + answers[1])) == 1, "six plan_get answers, one text: %s" % answers) + + @test def a_clap_member_and_an_lv2_member_of_the_adapter_row_form_one_chain(): """clap and lv2-clap are two rows of one program, both running chains in one client; lv2 (mod-host) and clap are two diff --git a/tests/sabotage.py b/tests/sabotage.py index 71ff56b..7bc024d 100644 --- a/tests/sabotage.py +++ b/tests/sabotage.py @@ -752,6 +752,198 @@ " if (!strcmp(verb, OMX_VERB_WORKER_LIST))\n", "verbs_declared", "verbs"), ("a declared verb is typed by hand in the daemon", "src/verbs.c", "if (!strcmp(verb, OMX_VERB_CHAIN_GET))", 'if (!strcmp(verb, "chain_get"))', "verbs_contract", "verbs"), + ('a refused pin is no reason to stay alone', 'src/planner.c', + 'if (m->pin == PLANNER_PIN_REFUSED)', + 'if (0 && m->pin == PLANNER_PIN_REFUSED)', + 'refused_leaves_the_member_alone', "planner"), + ('no pin under strict shares', 'src/planner.c', + 'if (m->pin == PLANNER_PIN_NONE && !s->pins_first_use)', + 'if (0 && m->pin == PLANNER_PIN_NONE && !s->pins_first_use)', + 'no_pin_under_strict_leaves_the_member_alone', "planner"), + ('a pin without a verdict shares under first-use', 'src/planner.c', + 'if ((m->pin == PLANNER_PIN_NONE || m->pin == PLANNER_PIN_NO_VERDICT) && s->pins_first_use)', + 'if (0 && (m->pin == PLANNER_PIN_NONE || m->pin == PLANNER_PIN_NO_VERDICT) && s->pins_first_use)', + 'no_verdict_leaves_the_member_alone', "planner"), + ('a pin that is gone under first-use shares', 'src/planner.c', + '(m->pin == PLANNER_PIN_NONE || m->pin == PLANNER_PIN_NO_VERDICT) && s->pins_first_use)', + '(m->pin == PLANNER_PIN_NO_VERDICT) && s->pins_first_use)', + 'a_pin_that_is_gone_under_first_use_has_no_verdict', "planner"), + ('a pin without a verdict is alone under strict as well', 'src/planner.c', + 'm->pin == PLANNER_PIN_NO_VERDICT) && s->pins_first_use)', + 'm->pin == PLANNER_PIN_NO_VERDICT))', + 'under_strict_a_pin_without_a_verdict_shares', "planner"), + ('verdict isolated shares', 'src/planner.c', + 'if (m->pin == PLANNER_PIN_ISOLATED)', + 'if (0 && m->pin == PLANNER_PIN_ISOLATED)', + 'verdict_isolated_leaves_the_member_alone', "planner"), + ('a format with no world shares', 'src/planner.c', + 'if (!m->world_ok)', + 'if (0)', + 'format_world_leaves_the_member_alone', "planner"), + ('the world is judged before the verdict', 'src/planner.c', + ' if (m->pin == PLANNER_PIN_ISOLATED)\n return PLANNER_REASON_VERDICT_ISOLATED;\n if (!m->world_ok)\n return PLANNER_REASON_FORMAT_WORLD;', + ' if (!m->world_ok)\n return PLANNER_REASON_FORMAT_WORLD;\n if (m->pin == PLANNER_PIN_ISOLATED)\n return PLANNER_REASON_VERDICT_ISOLATED;', + 'the_first_rule_that_matches_decides', "planner"), + ('the pins are read without require_pins', 'src/planner.c', + ' if (s->require_pins)\n {', + ' if (1)\n {', + 'without_require_pins_no_pin_is_read', "planner"), + ('a member over the cost limit shares', 'src/planner.c', + ' return PLANNER_REASON_COST_OVER;', + ' return PLANNER_REASON_NONE;', + 'cost_over_leaves_the_member_alone', "planner"), + ('a cost at the limit is over it', 'src/planner.c', + 'member_cost(m) > s->group_cost_max', + 'member_cost(m) >= s->group_cost_max', + 'a_cost_at_the_limit_is_not_over', "planner"), + ("a program that runs no chain of the others' joins the segment", 'src/planner.c', + 'if (strcmp(m[first].klass, m[k].klass))', + 'if (0)', + 'format_mixed_starts_a_new_segment', "planner"), + ('a full segment takes another member', 'src/planner.c', + 'else if (s->group_max > 0 && open >= s->group_max)', + 'else if (0)', + 'group_full_starts_a_new_segment', "planner"), + ('group_max 0 is a limit of 0', 'src/planner.c', + 'else if (s->group_max > 0 && open >= s->group_max)', + 'else if (open >= s->group_max)', + 'group_max_zero_is_no_limit', "planner"), + ('a segment over its cost takes another member', 'src/planner.c', + 'else if (s->group_cost_max > 0 && sum + own > s->group_cost_max)', + 'else if (s->group_cost_max > 0 && sum + own > s->group_cost_max * 1e9)', + 'group_cost_starts_a_new_segment', "planner"), + ('a segment exactly at its cost limit is full', 'src/planner.c', + 'sum + own > s->group_cost_max', + 'sum + own >= s->group_cost_max', + 'the_costs_of_a_segment_add_up', "planner"), + ('the costs of a segment do not add', 'src/planner.c', + ' open++;\n sum += own;', + ' open++;\n sum += 0 * own;', + 'the_costs_of_a_segment_add_up', "planner"), + ('a member of unknown cost counts for nothing', 'src/planner.c', + 'double own = cost == PLANNER_COST_UNKNOWN ? assumed_cost(m, first, open) : cost;', + 'double own = cost == PLANNER_COST_UNKNOWN ? 0 * assumed_cost(m, first, open) : cost;', + 'a_member_of_unknown_cost_counts_as_the_median_of_its_segment', "planner"), + ('the median of an odd number is the first', 'src/planner.c', + 'return n % 2 ? known[n / 2] :', + 'return n % 2 ? known[0] :', + 'a_member_of_unknown_cost_counts_as_the_median_of_its_segment', "planner"), + ('the median of an even number is the upper middle', 'src/planner.c', + '(known[n / 2 - 1] + known[n / 2]) / 2', + 'known[n / 2]', + 'the_median_of_an_even_number_is_the_mean_of_the_middle_two', "planner"), + ('cost-assumed is never said', 'src/planner.c', + 'out->reason[j] = PLANNER_REASON_COST_ASSUMED;', + 'out->reason[j] = PLANNER_REASON_NONE;', + 'cost_assumed_is_said_of_a_member_that_joined_on_an_assumption', "planner"), + ('no-neighbour is never said', 'src/planner.c', + 'out->reason[j] = PLANNER_REASON_NO_NEIGHBOUR;', + 'out->reason[j] = PLANNER_REASON_NONE;', + 'no_neighbour_is_said_of_a_member_alone_for_the_members_around_it', "planner"), + ('grouping-off is never said', 'src/planner.c', + 'else if (!s->grouping_on)', + 'else if (0)', + 'grouping_off_says_what_would_have_shared', "planner"), + ('group-full is judged before format-mixed', 'src/planner.c', + 'if (strcmp(m[first].klass, m[k].klass))\n r = PLANNER_REASON_FORMAT_MIXED;\n else if (s->group_max > 0 && open >= s->group_max)\n r = PLANNER_REASON_GROUP_FULL;', + 'if (s->group_max > 0 && open >= s->group_max)\n r = PLANNER_REASON_GROUP_FULL;\n else if (strcmp(m[first].klass, m[k].klass))\n r = PLANNER_REASON_FORMAT_MIXED;', + 'format_mixed_comes_before_group_full_and_group_cost', "planner"), + ('a plan of too many members is planned', 'src/planner.c', + 'if (n < 1 || n > PLANNER_MEMBERS_MAX)', + 'if (n < 1)', + 'a_plan_of_no_member_or_too_many_is_refused', "planner"), + ('the planner remembers what it planned before', 'src/planner.c', + 'sum + own > s->group_cost_max', + 'sum + own > s->group_cost_max - ({ static double drift; drift += 0.02; drift; })', + 'the_same_members_give_the_same_bytes', "planner"), + ('the segments are joined by commas', 'src/planner.c', + ' put(buf, size, &used, ";");', + ' put(buf, size, &used, ",");', + 'the_text_of_a_plan_is_the_one_the_bench_parses', "planner"), + ('no reason is the empty text', 'src/planner.c', + ' put(buf, size, &used, PHD_WORD_NONE);', + ' put(buf, size, &used, "");', + 'no_reason_is_a_dash', "planner"), + ('a refused plan is told another way', 'src/planner.c', + '"%s%d:%s:%d"', + '"%s%d:%s=%d"', + 'a_refused_plan_gives_every_member_the_refusal', "planner"), + ('the reasons are not the declared words', 'src/planner.c', + 'const char *planner_reason_word(int reason)\n{', + 'const char *planner_reason_word(int reason)\n{\n if (reason == PLANNER_REASON_NO_PIN)\n return "no_pin";', + 'the_reasons_are_the_words_of_the_declaration', "planner"), + ('the rate is not read', 'src/qualcost.c', + 'r != rate ||', + '0 ||', + 'a_rate_or_quantum_the_record_lacks_is_unknown', "qualcost"), + ('the quantum is not read', 'src/qualcost.c', + 'q != quantum)', + '(q != quantum && 0))', + 'a_rate_or_quantum_the_record_lacks_is_unknown', "qualcost"), + ('the first record of a plugin stands', 'src/qualcost.c', + 'if (v > best)', + 'if (best < 0)', + 'the_larger_of_two_records_of_one_plugin_stands', "qualcost"), + ('the id inside the record is not read', 'src/qualcost.c', + 'if (!got || strcmp(got, id) || !grid', + 'if (!got || (strcmp(got, id) && 0) || !grid', + 'a_record_of_another_plugin_is_not_this_ones', "qualcost"), + ('the fraction is nanoseconds over a second', 'src/qualcost.c', + 'return ns * rate / 1e9;', + 'return ns / 1e9;', + 'the_figure_is_the_p95_at_the_rate_and_quantum_as_a_fraction_of_a_core', "qualcost"), + ('a plan again forms every segment again', 'src/supervisor.c', + 'if (!kept[j] && was[j]->n == pp->seg_len[k]', + 'if (0 && !kept[j] && was[j]->n == pp->seg_len[k]', + 'a_plan_again_keeps_the_worker_of_a_segment_that_did_not_change_and_forms_the_ones_that_did'), + ('a kept segment keeps the name of its old place', 'src/supervisor.c', + ' snprintf(at[k]->w->place, sizeof(at[k]->w->place), PHD_PLACE_CHAIN_PREFIX "%s", word);\n', + '', + 'a_plan_that_keeps_a_segment_in_another_place_renames_its_worker_and_restarts_nothing'), + ('a member the new plan leaves out stays in the worker of its old segment', 'src/supervisor.c', + ' if (c->w && i->w == c->w)\n to_own(i);\n }\n }\n chain_free(c);', + ' }\n }\n chain_free(c);', + 'a_plan_again_keeps_the_worker_of_a_segment_that_did_not_change_and_forms_the_ones_that_did'), + ('a plan under grouping off is applied', 'src/supervisor.c', + 'if (s.grouping_on && !refused)\n plan_apply', + 'if (!refused)\n plan_apply', + 'while_grouping_is_off_a_plan_is_answered_and_kept_and_nothing_moves'), + ("a member of another chain's plan is the plan's to place", 'src/supervisor.c', + 'if (i->chain && !(i->chain->planned && i->chain->id == id))', + 'if (0)', + 'a_plan_is_refused_for_what_chain_refuses_and_moves_nothing'), + ("plan_clear leaves the members in the segment's worker", 'src/supervisor.c', + ' if (c->w && i->w == c->w)\n to_own(i);\n }\n chain_free(c);\n }\n /* the members that share', + ' }\n chain_free(c);\n }\n /* the members that share', + 'plan_clear_gives_every_member_a_worker_of_its_own_and_forgets_the_plan'), + ('a changed plan is not announced', 'src/supervisor.c', + ' event(PHD_EVENT_PLAN_REASON_FMT, id, p->reasons);', + ' ;', + 'a_plan_forms_the_runs_that_may_share_and_leaves_the_member_that_may_not_in_a_worker_of_its_own'), + ("a segment's worker is named for the plan alone", 'src/supervisor.c', + ' snprintf(place, sizeof(place), PHD_PLACE_CHAIN_PREFIX "%s", word);', + ' snprintf(place, sizeof(place), PHD_PLACE_CHAIN_PREFIX "%d", c->id);', + 'a_plan_forms_the_runs_that_may_share_and_leaves_the_member_that_may_not_in_a_worker_of_its_own'), + ('a cost is never read from a record', 'src/supervisor.c', + '? qualcost_read(g_conf.qualification_dir, record_id(i->uri), g_conf.group_rate, g_conf.group_quantum)', + '? PLANNER_COST_UNKNOWN', + 'the_cost_a_qualification_record_holds_decides_the_segments_and_a_missing_one_is_assumed'), + ('group_cost_max is read as a fraction already', 'src/supervisor.c', + 's.group_cost_max = g_conf.group_cost_max / 100.0;', + 's.group_cost_max = g_conf.group_cost_max / 10.0;', + 'the_cost_a_qualification_record_holds_decides_the_segments_and_a_missing_one_is_assumed'), + ('a CLAP plugin is found by its path in the records', 'src/supervisor.c', + 'return hash ? hash + 1 : uri;', + 'return uri;', + 'the_cost_a_qualification_record_holds_decides_the_segments_and_a_missing_one_is_assumed'), + ('a verdict of isolated is read as none', 'src/supervisor.c', + ' else if (!strcmp(verdict, PHD_PIN_VERDICT_ISOLATED))\n m->pin = PLANNER_PIN_ISOLATED;\n', + '', + 'a_plan_forms_the_runs_that_may_share_and_leaves_the_member_that_may_not_in_a_worker_of_its_own'), + ("any word is a grouping mode", "src/conf.c", "ok |= !strcmp(conf->grouping, name);", "ok |= 1;", + "a_setting_that_is_none_of_the_grouping_modes_stops_the_daemon_and_names_the_modes"), + ("the handler of a declared verb is deleted: plan", "src/verbs.c", + " if (!strcmp(verb, OMX_VERB_PLAN))\n return plan_words(tok, ntok);\n", "", "verbs_declared", "verbs"), ] @@ -835,7 +1027,22 @@ def run_verbs_test(tree, name): return p.returncode, p.stdout + p.stderr + ("ok %s\n" % name if p.returncode == 0 else "") +def run_unit_test(tree, target, source, name): + """a C unit test of the tree's sources, built by the Makefile's rule for `target`; it runs the one test and prints + "ok " when it passes""" + os.makedirs(os.path.join(tree, "tests"), exist_ok=True) + shutil.copy(os.path.join(ROOT, "tests", source), os.path.join(tree, "tests")) + subprocess.run(["make", "-s", "-B", "MOD_HOST_DIR=" + MOD_HOST_DIR, target], cwd=tree, check=True, + stdout=subprocess.DEVNULL) + p = subprocess.run([os.path.join(tree, target), name], capture_output=True, text=True, timeout=60) + return p.returncode, p.stdout + p.stderr + + def run_test(tree, name, jack=False): + if jack == "planner": + return run_unit_test(tree, "tests/planner-test", "planner_test.c", name) + if jack == "qualcost": + return run_unit_test(tree, "tests/qualcost-test", "qualcost_test.c", name) if jack == "verbs": return run_verbs_test(tree, name) if jack == "consumer": @@ -953,6 +1160,10 @@ def run_test(tree, name, jack=False): shutil.copy(os.path.join(ROOT, f), tree) os.makedirs(os.path.join(tree, "packaging")) shutil.copy(os.path.join(ROOT, "packaging", "plugin-hostd.spec"), os.path.join(tree, "packaging")) + elif jack in ("planner", "qualcost"): + shutil.copytree(os.path.join(ROOT, "src"), os.path.join(tree, "src")) + shutil.copytree(os.path.join(ROOT, "include"), os.path.join(tree, "include")) + shutil.copy(os.path.join(ROOT, "Makefile"), tree) elif jack in ("consumer", "verbtable"): shutil.copytree(os.path.join(ROOT, "include"), os.path.join(tree, "include")) elif jack == "nopins": @@ -991,7 +1202,7 @@ def run_test(tree, name, jack=False): if jack == "probe": build_probe(tree) elif jack not in ("readme", "nopins", "ciguards", "format", "relay", "contract", "consumer", "probeschema", "presetschema", "vocabulary", "alloc", - "verbtable"): + "verbtable", "planner", "qualcost"): for f in os.listdir(os.path.join(tree, "src")): if f.endswith(".o"): os.unlink(os.path.join(tree, "src", f)) diff --git a/tests/verbs_declared.py b/tests/verbs_declared.py index 644452c..e1eee0f 100644 --- a/tests/verbs_declared.py +++ b/tests/verbs_declared.py @@ -202,6 +202,9 @@ def code(reply): "chain": ("7 1 2", "7", ("8 99", "8 1 1")), "chain_get": ("7", "x", ("9",)), "chain_clear": ("7", "x", ("7",)), + "plan": ("7 1 2", "7", ("8 99", "8 1 1")), + "plan_get": ("7", "x", ("9",)), + "plan_clear": ("7", "x", ("7",)), "pin_clear": ("all", "", ()), } ORDER = list(LINES) From aa7fdfa717ff0d941330d00aaa1795932bcbbf69 Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Tue, 27 Oct 2026 10:40:55 +0100 Subject: [PATCH 13/16] fuzz: the chain mutant applies past the new plan handler plan_words sits after chain_words, so the patch's trailing context moved. --- fuzz/sabotage/command-words-chain.patch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzz/sabotage/command-words-chain.patch b/fuzz/sabotage/command-words-chain.patch index 0bd3008..47a1d05 100644 --- a/fuzz/sabotage/command-words-chain.patch +++ b/fuzz/sabotage/command-words-chain.patch @@ -13,4 +13,4 @@ index 3b7feba..b464de3 100644 + return sup_chain(id, ids, ntok - 3); } - static void reply(int fd, char *text) + /* plan [ ...]: at least PHD_CHAIN_MEMBERS_MIN members */ From 893ea7f78a5b92b523e54325c467f76d49f252f3 Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Tue, 27 Oct 2026 11:14:44 +0100 Subject: [PATCH 14/16] fuzz: point the mutant citations at src/verbs.c and fuzz_command.c as the batch lays them out The history verbs, the plan verbs and the planner stack moved the lines the fuzz checks cite: src/verbs.c:N in the mutant comments of fuzz/fuzz_command.c, and fuzz/fuzz_command.c:N in the first line of five sabotage patches. make fuzz-records meets all 34 records again. --- fuzz/fuzz_command.c | 12 ++++++------ fuzz/sabotage/command-refusal.patch | 2 +- fuzz/sabotage/command-round-trip.patch | 2 +- fuzz/sabotage/command-words-add.patch | 2 +- fuzz/sabotage/command-words-chain.patch | 2 +- fuzz/sabotage/command-words-pin-set.patch | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/fuzz/fuzz_command.c b/fuzz/fuzz_command.c index 66a6add..ffde610 100644 --- a/fuzz/fuzz_command.c +++ b/fuzz/fuzz_command.c @@ -207,7 +207,7 @@ __attribute__((noinline)) static void check_reply_answer(const struct words *w) * input: add-default * mutant: src/verbs.c:90 `return rc;` -> `return SUCCESS;` * input: plugin-hostd-pin_set-refused - * mutant: src/verbs.c:247 `return reply_text;` -> `return sup_resp(SUCCESS);` + * mutant: src/verbs.c:251 `return reply_text;` -> `return sup_resp(SUCCESS);` * input: mod-host-set_bpm-refused */ if (strcmp(g_rec.reply, g_rec.answer)) abort(); @@ -229,7 +229,7 @@ __attribute__((noinline)) static void check_reply(const struct words *w) int code; /* one reply, and nothing sent empty - * mutant: src/verbs.c:265 `reply(msg->sender_id, handle(line, tok, ntok));` -> `free(handle(line, tok, ntok));` + * mutant: src/verbs.c:269 `reply(msg->sender_id, handle(line, tok, ntok));` -> `free(handle(line, tok, ntok));` * input: add-default */ if (g_rec.nreplies != 1 || g_rec.reply_size == 0) abort(); @@ -252,7 +252,7 @@ __attribute__((noinline)) static void check_reply(const struct words *w) } shape = w->n ? rec_shape_of(w->w[0], strlen(w->w[0])) : REC_MOD_HOST_SHAPE; /* anything else has the shape its verb declares, or mod-host's - * mutant: src/verbs.c:236 `sup_resp(SUCCESS)` -> `sup_resp(1)` + * mutant: src/verbs.c:240 `sup_resp(SUCCESS)` -> `sup_resp(1)` * input: mod-host-quit */ if (!syntax_reply_matches(shape, g_rec.reply) && !syntax_reply_matches(REC_MOD_HOST_SHAPE, g_rec.reply)) abort(); @@ -490,17 +490,17 @@ __attribute__((noinline)) static void check_round_trip(const struct words *w, ui free(line); /* the same calls with the same arguments - * mutant: src/verbs.c:259 `strtok_r(copy, " \t", &save)` -> `strtok_r(copy, " ", &save)` + * mutant: src/verbs.c:263 `strtok_r(copy, " \t", &save)` -> `strtok_r(copy, " ", &save)` * input: mod-host-add-tabs */ if (!rec_calls_equal(first, nfirst, g_rec.calls, g_rec.ncalls)) abort(); /* the same reply - * mutant: src/verbs.c:233 `ntok != 1` -> `ntok != 1 || line[0] != ' '` + * mutant: src/verbs.c:237 `ntok != 1` -> `ntok != 1 || line[0] != ' '` * input: quit-leading-space */ if (g_rec.reply == NULL || strcmp(g_rec.reply, reply_first)) abort(); /* the same state, the same host settings, and quitting the same - * mutant: src/verbs.c:235 `g_quitting = 1;` -> `g_quitting = line[0] == ' ';` + * mutant: src/verbs.c:239 `g_quitting = 1;` -> `g_quitting = line[0] == ' ';` * input: quit-leading-space */ if (g_quitting != quit_first || g_rec.state != state_first || g_rec.host_settings != host_first) abort(); diff --git a/fuzz/sabotage/command-refusal.patch b/fuzz/sabotage/command-refusal.patch index 0b7db0f..ad3ec39 100644 --- a/fuzz/sabotage/command-refusal.patch +++ b/fuzz/sabotage/command-refusal.patch @@ -1,4 +1,4 @@ -Breaks the check of fuzz/fuzz_command.c:455 (a refused command leaves the daemon's state as it was). +Breaks the check of fuzz/fuzz_command.c:460 (a refused command leaves the daemon's state as it was). Caught by the seed input "add-instance-past-the-table" of fuzz_command. diff --git a/src/verbs.c b/src/verbs.c diff --git a/fuzz/sabotage/command-round-trip.patch b/fuzz/sabotage/command-round-trip.patch index a83cfe3..903e5e2 100644 --- a/fuzz/sabotage/command-round-trip.patch +++ b/fuzz/sabotage/command-round-trip.patch @@ -1,4 +1,4 @@ -Breaks the check of fuzz/fuzz_command.c:490 (a command's words joined by one space make the same calls). +Breaks the check of fuzz/fuzz_command.c:495 (a command's words joined by one space make the same calls). Caught by the seed input "mod-host-add-tabs" of fuzz_command. diff --git a/src/verbs.c b/src/verbs.c diff --git a/fuzz/sabotage/command-words-add.patch b/fuzz/sabotage/command-words-add.patch index 6a196a3..2e9f8fe 100644 --- a/fuzz/sabotage/command-words-add.patch +++ b/fuzz/sabotage/command-words-add.patch @@ -1,4 +1,4 @@ -Breaks the check of fuzz/fuzz_command.c:351 (the placement and the client, each where the message has it). +Breaks the check of fuzz/fuzz_command.c:356 (the placement and the client, each where the message has it). Caught by the seed input "add-pool-p-fuzz_client" of fuzz_command. diff --git a/src/verbs.c b/src/verbs.c diff --git a/fuzz/sabotage/command-words-chain.patch b/fuzz/sabotage/command-words-chain.patch index 47a1d05..2768a7c 100644 --- a/fuzz/sabotage/command-words-chain.patch +++ b/fuzz/sabotage/command-words-chain.patch @@ -1,4 +1,4 @@ -Breaks the check of fuzz/fuzz_command.c:370 (every member, in the order of the message, and their count). +Breaks the check of fuzz/fuzz_command.c:375 (every member, in the order of the message, and their count). Caught by the seed input "plugin-hostd-chain" of fuzz_command. diff --git a/src/verbs.c b/src/verbs.c diff --git a/fuzz/sabotage/command-words-pin-set.patch b/fuzz/sabotage/command-words-pin-set.patch index 4872217..ffdfeb6 100644 --- a/fuzz/sabotage/command-words-pin-set.patch +++ b/fuzz/sabotage/command-words-pin-set.patch @@ -1,4 +1,4 @@ -Breaks the check of fuzz/fuzz_command.c:386 (the verdict, the word the controller wrote after its key). +Breaks the check of fuzz/fuzz_command.c:391 (the verdict, the word the controller wrote after its key). Caught by the seed input "plugin-hostd-pin_set-full" of fuzz_command. diff --git a/src/verbs.c b/src/verbs.c From 2a9c1243ff8a78706d63fe064c1d82fb0a80338b Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Tue, 27 Oct 2026 12:12:21 +0100 Subject: [PATCH 15/16] tests: make the two chain sabotage texts unique, and catch the two mutants no test noticed The sabotage texts of the chain refusal (supervisor.c) and the chain grammar (verbs.c) now carry the lines around them, so each occurs once next to its plan twin. A failed history save is tested with a directory in the history file's place, which fails for root too; the old test skipped itself as root and so never saw the mutant that forgets the change. The first record of a plugin is tested with the larger figure listed second by the directory, so a reader that keeps the first record it meets gives the wrong figure whatever order the file system lists in. tests/planner-test and tests/qualcost-test join .gitignore. --- .gitignore | 2 ++ tests/history_test.c | 35 +++++++++++++++++++++++------------ tests/qualcost_test.c | 35 +++++++++++++++++++++++++++++++++++ tests/sabotage.py | 9 +++++---- 4 files changed, 65 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index 717af94..59e2d6f 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,5 @@ tests/lv2-alloc-lv2.o tests/transforms-test /build/ fuzz/findings/ +tests/planner-test +tests/qualcost-test diff --git a/tests/history_test.c b/tests/history_test.c index 4146bf2..3fbba35 100644 --- a/tests/history_test.c +++ b/tests/history_test.c @@ -508,16 +508,15 @@ static void test_the_file(void) group("the_history_file_is_mode_0600_in_a_private_directory_with_no_temporary_beside_it"); } +/* A save that fails has to fail for root too, so the directory is not made unwritable (root writes it anyway): the name + * of the history file is taken by a directory that is not empty, and renaming the finished temporary file over it is + * refused. */ static void test_a_failed_save(void) { - char *before, *after; + char *before, *after, stray[420], cmd[900], l[32] = ""; + struct stat st; + FILE *p; - if (geteuid() == 0) - { - printf("skip history: a failed save needs a directory root can't write to\n"); - printf("ok a_save_that_fails_leaves_the_old_file_whole_and_is_tried_again\n"); - return; - } fresh(); start(); history_tick(0); @@ -526,12 +525,24 @@ static void test_a_failed_save(void) before = file_text(); check(strlen(before) > strlen(HEADER), "the first save wrote the record"); history_window(URI, DIGEST, 0, 2.0, 2.0, 0, 70); - chmod(dir, 0500); + snprintf(stray, sizeof(stray), "%s/stray", file); + if (unlink(file) != 0 || mkdir(file, 0700) != 0) + { + printf("FAIL history: test can't put a directory in the history's place\n"); + exit(2); + } + put(stray, "x"); history_tick(1000); - chmod(dir, 0700); - after = file_text(); - check_str(after, before, "a save that can't write leaves the old file as it was"); - free(after); + check(stat(file, &st) == 0 && S_ISDIR(st.st_mode), "a save that can't write leaves what was there as it was"); + snprintf(cmd, sizeof(cmd), "ls -A '%s' | wc -l", dir); + p = popen(cmd, "r"); + if (p && fgets(l, sizeof(l), p)) + check(atoi(l) == 1, "a failed save leaves no temporary file behind"); + if (p) + pclose(p); + unlink(stray); + rmdir(file); + put(file, before); history_tick(1299); after = file_text(); check_str(after, before, "a failed save is tried again after the interval, not at every tick"); diff --git a/tests/qualcost_test.c b/tests/qualcost_test.c index 042a930..1c392f9 100644 --- a/tests/qualcost_test.c +++ b/tests/qualcost_test.c @@ -8,6 +8,7 @@ */ #include +#include #include #include #include @@ -79,6 +80,15 @@ static void the_larger_of_two_records_of_one_plugin_stands(void) CHECK(fabs(c - 0.48) < 1e-12, "10000 ns beats 2000 ns: %g", c); } +/* whichever of the two files the directory lists first holds the smaller figure, so a reader that keeps the first record + * it meets gives 2000 ns and not 10000 */ +static void the_larger_record_stands_whichever_the_directory_lists_first(void) +{ + double c = qualcost_read(g_dir, "org.example.d", 48000, 128); + + CHECK(fabs(c - 0.48) < 1e-12, "10000 ns beats 2000 ns, listed first or last: %g", c); +} + static void a_record_of_another_plugin_is_not_this_ones(void) { CHECK(qualcost_read(g_dir, "org.example.c", 48000, 128) == -1.0, "c has no record"); @@ -103,6 +113,7 @@ static const struct { const char *name; void (*fn)(void); } g_tests[] = { T(the_figure_is_the_p95_at_the_rate_and_quantum_as_a_fraction_of_a_core), T(a_rate_or_quantum_the_record_lacks_is_unknown), T(the_larger_of_two_records_of_one_plugin_stands), + T(the_larger_record_stands_whichever_the_directory_lists_first), T(a_record_of_another_plugin_is_not_this_ones), T(a_file_that_is_no_record_is_skipped), T(a_directory_that_is_not_there_gives_no_figure), @@ -122,6 +133,30 @@ int main(int argc, char **argv) record("org.example.b@1.0-bbbbbbbbbbbb.default.qualification.json", "org.example.b", 48000, 128, 2000); record("org.example.b@2.0-cccccccccccc.default.qualification.json", "org.example.b", 48000, 128, 10000); record("org.example.liar@1.0-dddddddddddd.default.qualification.json", "org.example.other", 48000, 128, 5000); + { + /* two records of one plugin: the one the directory lists first is made the smaller */ + const char *names[2] = { "org.example.d@1.0-111111111111.default.qualification.json", + "org.example.d@2.0-222222222222.default.qualification.json" }; + char first[300] = ""; + struct dirent *e; + DIR *d; + int k; + + record(names[0], "org.example.d", 48000, 128, 2000); + record(names[1], "org.example.d", 48000, 128, 2000); + if (!(d = opendir(g_dir))) + return 2; + while ((e = readdir(d)) != NULL) + if (!strncmp(e->d_name, "org.example.d@", 14)) + { + snprintf(first, sizeof(first), "%s", e->d_name); + break; + } + closedir(d); + for (k = 0; k < 2; k++) + if (strcmp(first, names[k])) + record(names[k], "org.example.d", 48000, 128, 10000); + } put("org.example.junk@1.0-eeeeeeeeeeee.default.qualification.json", "{not json"); put("org.example.empty@1.0-ffffffffffff.default.qualification.json", ""); for (n = 0; n < sizeof(g_tests) / sizeof(g_tests[0]); n++) diff --git a/tests/sabotage.py b/tests/sabotage.py index 426bc1f..df4d101 100644 --- a/tests/sabotage.py +++ b/tests/sabotage.py @@ -580,8 +580,8 @@ ' put(&buf, &used, &cap, "%s%d", any++ ? "," : "", c->ids[k]);', "chain_get_reads_a_chain_back_as_declared_and_as_it_runs"), ('an unknown member of a chain is refused without its named code', "src/supervisor.c", - ' return sup_resp(PHD_ERR_CHAIN_UNKNOWN_INSTANCE);', - ' return sup_resp(ERR_INSTANCE_NON_EXISTS);', + ' return sup_resp(PHD_ERR_CHAIN_UNKNOWN_INSTANCE);\n for (m = 0; m < k; m++)', + ' return sup_resp(ERR_INSTANCE_NON_EXISTS);\n for (m = 0; m < k; m++)', "a_chain_is_declared_over_existing_instances_and_each_refusal_is_named"), ('an instance named twice is a chain', "src/supervisor.c", ' if (ids[m] == ids[k])\n return sup_resp(PHD_ERR_CHAIN_DUPLICATE);', @@ -664,7 +664,8 @@ "#define PHD_CHAIN_MEMBERS_MIN 1\n", "#define PHD_CHAIN_MEMBERS_MIN 2\n", "a_chain_of_one_runs_in_a_worker_of_its_own_and_a_chain_of_none_is_refused"), ("the grammar of chain asks two members, not the declared floor", "src/verbs.c", - "if (ntok < 2 + PHD_CHAIN_MEMBERS_MIN || ", "if (ntok < 4 || ", + "if (ntok < 2 + PHD_CHAIN_MEMBERS_MIN || !number(tok[1], &id))\n return sup_resp(ERR_INVALID_OPERATION);\n for (n = 2; n < ntok; n++)\n if (!number(tok[n], &ids[n - 2]))\n return sup_resp(ERR_INVALID_OPERATION);\n return sup_chain(", + "if (ntok < 4 || !number(tok[1], &id))\n return sup_resp(ERR_INVALID_OPERATION);\n for (n = 2; n < ntok; n++)\n if (!number(tok[n], &ids[n - 2]))\n return sup_resp(ERR_INVALID_OPERATION);\n return sup_chain(", "a_chain_of_one_runs_in_a_worker_of_its_own_and_a_chain_of_none_is_refused"), ("the supervisor asks two members of a chain, not the declared floor", "src/supervisor.c", "|| n < PHD_CHAIN_MEMBERS_MIN)", "|| n < 2)", @@ -1111,7 +1112,7 @@ ('the first record of a plugin stands', 'src/qualcost.c', 'if (v > best)', 'if (best < 0)', - 'the_larger_of_two_records_of_one_plugin_stands', "qualcost"), + 'the_larger_record_stands_whichever_the_directory_lists_first', "qualcost"), ('the id inside the record is not read', 'src/qualcost.c', 'if (!got || strcmp(got, id) || !grid', 'if (!got || (strcmp(got, id) && 0) || !grid', From 893e1668a339263a4f25693c5dda8607d5c4038d Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Tue, 27 Oct 2026 12:44:21 +0100 Subject: [PATCH 16/16] CI: give build and test 60 minutes make sabotage runs a build per arm and, with the history and planner arms, passes 30 minutes: both runs of the batch were cancelled while it compiled. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7028ac7..fd3f2fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,7 +68,7 @@ jobs: test: name: build and test runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 60 container: image: fedora:44 # the jack test runs the daemon and its workers in a user, net, pid and mount namespace of their own