diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 91f8ab73..2e190ed0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,6 +51,7 @@ on: - '.editorconfig' - '.clang-format' - 'frama-c-stubs/**' + - '.github/workflows/conformance.yml' - '.github/workflows/lint.yml' - '.github/workflows/static-analysis.yml' - '.github/workflows/verify.yml' diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml new file mode 100644 index 00000000..8a5d72b4 --- /dev/null +++ b/.github/workflows/conformance.yml @@ -0,0 +1,122 @@ +name: Conformance + +on: + push: + branches: [main] + pull_request: + branches: [main] + merge_group: + schedule: + - cron: '17 3 * * *' + workflow_dispatch: + inputs: + scope: + type: choice + options: [pr, full] + default: pr + update_check: + type: boolean + default: false + +concurrency: + # A schedule event resolves github.ref to the default branch, so without the + # event split the nightly full run and a main push share one group and a + # second push cancels the queued nightly. + group: ${{ github.workflow }}-${{ github.event_name == 'schedule' && 'nightly' || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +env: + CONF_SCOPE: ${{ (github.event_name == 'schedule' || inputs.scope == 'full') && 'full' || 'pr' }} + CONF_REQUIRE: 1 + +jobs: + discover: + runs-on: ubuntu-24.04 + outputs: + suites: ${{ steps.suites.outputs.names }} + steps: + - uses: actions/checkout@v7 + - id: suites + run: | + names=$(python3 scripts/conformance suites --format json | + python3 -c 'import json,sys; print(json.dumps(json.load(sys.stdin)["suites"]))') + echo "names=$names" >> "$GITHUB_OUTPUT" + + harness: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + - run: python3 scripts/conformance selftest + + payload: + needs: discover + if: needs.discover.outputs.suites != '[]' + runs-on: ubuntu-24.04-arm + steps: + - uses: actions/checkout@v7 + - run: make conformance-payloads + - run: tar -C externals -cf conformance-payloads.tar payloads + - uses: actions/upload-artifact@v7 + with: + name: conformance-payloads + path: conformance-payloads.tar + if-no-files-found: error + + qemu: + needs: [discover, payload] + if: needs.discover.outputs.suites != '[]' + runs-on: [self-hosted, macOS, ARM64] + steps: + - uses: actions/checkout@v7 + - uses: actions/download-artifact@v8 + with: + name: conformance-payloads + - run: mkdir -p externals && tar -C externals -xf conformance-payloads.tar + - run: bash tests/fetch-fixtures.sh + - run: make test-conformance BACKEND=qemu + + elfuse: + needs: [discover, payload, qemu] + if: needs.discover.outputs.suites != '[]' + runs-on: [self-hosted, macOS, ARM64] + steps: + - uses: actions/checkout@v7 + - uses: actions/download-artifact@v8 + with: + name: conformance-payloads + - run: mkdir -p externals && tar -C externals -xf conformance-payloads.tar + - run: bash tests/fetch-fixtures.sh + - run: make elfuse + - run: make test-conformance BACKEND=elfuse + + conformance: + name: Conformance (make test-conformance) + needs: [discover, harness, payload, qemu, elfuse] + if: always() + runs-on: ubuntu-24.04 + steps: + - env: + SUITES: ${{ needs.discover.outputs.suites }} + DISCOVER: ${{ needs.discover.result }} + HARNESS: ${{ needs.harness.result }} + PAYLOAD: ${{ needs.payload.result }} + QEMU: ${{ needs.qemu.result }} + ELFUSE: ${{ needs.elfuse.result }} + run: | + [ "$DISCOVER" = success ] + [ "$HARNESS" = success ] + if [ "$SUITES" = '[]' ]; then + [ "$PAYLOAD $QEMU $ELFUSE" = 'skipped skipped skipped' ] + else + [ "$PAYLOAD $QEMU $ELFUSE" = 'success success success' ] + fi + + update-check: + if: github.event_name == 'workflow_dispatch' && inputs.update_check + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + - run: python3 scripts/conformance pins check diff --git a/Makefile b/Makefile index b63e70c3..0e8d472e 100644 --- a/Makefile +++ b/Makefile @@ -664,6 +664,7 @@ $(BUILD_DIR)/probe: tests/fixtures/sharun/probe.c \ endif include mk/tests.mk +include mk/conformance.mk include mk/lint.mk include mk/verify.mk include mk/format.mk diff --git a/README.md b/README.md index 8804f4b7..5d930116 100644 --- a/README.md +++ b/README.md @@ -141,8 +141,10 @@ The build signs `build/elfuse` before use. Override the signing identity with Rosetta, dynamic linking via `--sysroot`, and attaching `gdb` / `lldb` to the built-in stub. - [docs/testing.md](docs/testing.md): build prerequisites, the - `make check` flow, the QEMU and Rosetta cross-check matrices, and - fixture handling. + `make check` flow, the QEMU and Rosetta cross-check matrices, + fixture handling, and conformance commands. +- [docs/conformance.md](docs/conformance.md): the conformance harness, + expectations, payloads, and CI workflow. - [docs/filenames.md](docs/filenames.md): how a guest filename becomes a name on disk and back: case folding and normalization on the sysroot volume, the escape encoding, and the length limits both systems impose. diff --git a/docs/conformance.md b/docs/conformance.md new file mode 100644 index 00000000..51ca467c --- /dev/null +++ b/docs/conformance.md @@ -0,0 +1,102 @@ +# Conformance Harness + +The harness runs registered Linux test suites on elfuse and a QEMU reference. +It records suite status separately from the expectation verdict. The command +reference is in [testing.md](testing.md#conformance-tests). + +## Results + +`run` writes `results.json` below `///-/`. +That file is the canonical artifact: `schema_version: 1`, `kind: run`, run +metadata, derived counts and gate, and case records. Loading rejects a gate or +count that disagrees with the cases. An empty run is red. + +Each attempt records `normal`, `timeout`, `signal`, or `transport`, elapsed +microseconds, output paths, and an exit code or signal when applicable. Case +statuses are `PASS`, `FAIL`, `SKIP`, `CONF`, `WARN`, `BROK`, `TIMEOUT`, +`CRASH`, `INCONSISTENT`, and `ERROR`. Verdicts are `as_expected`, +`unexpected_failure`, `unexpected_pass`, `flaked`, `filtered`, and `error`. + +JSON list output also has `schema_version: 1` and a `kind` field. Requested +machine data uses stdout. Diagnostics use stderr. + +Exit codes are: + +- `0`: the operation succeeded or the run is green. +- `1`: a completed run or artifact check is red. +- `2`: the command, configuration, or operation is invalid. +- `3`: a non-writing pin or selection check found drift. +- `77`: an optional prerequisite is absent. `--require` and `CONF_REQUIRE=1` + promote it to `2`. + +## IDs and Selection + +Case IDs have one of these forms: + +```text +: +:/[/...] +``` + +Selectors and expectation matchers use shell globs across the complete ID. +A bare group selector also selects its cases. An unmatched selector is an +error. + +A selection file assigns each upstream launch group to `pr`, `full`, or a +declined group with a reason. PR groups run in both scopes. Enabled entries +may set `timeout_s` and suite-specific case filters. + +## Expectations + +Expectation files are JSONC and accept comments and trailing commas. A suite +has a base file, one leaf per backend, and optional `flaky.jsonc`. Files contain +ordered actions; the last matching non-quarantine action wins. The first +effective action is `expect_pass` for `*`. + +Actions are `expect_pass`, `expect_failure`, `expect_conf`, `skip`, and +`quarantine`. Every non-pass action needs a reason. `quarantine` is valid only +in `flaky.jsonc`; it runs the case alone for at most three attempts and reports +test mismatches as `flaked`. Harness errors remain red. A full run rejects +matchers that select no case. + +A skipped expectation prevents launch. `--bootstrap` launches skipped cases +and records status without applying expectations. `expectations seed` derives +actions from bootstrap statuses or red verdicts. It refuses harness errors. + +## Payloads and Pins + +Payloads live below `externals/payloads/` and are not committed. A fingerprint +hashes the pin and builder inputs. `manifest.json` records the fingerprint and +each staged file or symlink. Verification detects missing, extra, changed, and +stale content before a run starts. + +Pins are schema-checked JSON. `pins update` validates the new pin before +replacing the file. + +## Suite Interface + +`tests/conformance/providers/__init__.py` is the static suite registry; +`Provider` in `providers/base.py` declares what a suite supplies. + +The shared runner owns expectation loading, skip handling, unresolved batch +reruns, quarantine retries, result ordering, and judgment. Providers map +suite output to statuses. Backends return process invocations. A provider +translates host paths through `backend.guest_path()` before putting them in +argv; `Backend.run` forwards argv unchanged, because only the provider knows +which elements are paths. QEMU records non-timeout shell statuses as exit +codes. Providers interpret `128+n` through the suite contract because the +shell cannot distinguish it from a plain exit with the same value. + +The elfuse backend starts one `build/elfuse --timeout 0` process for each +command. The QEMU backend starts one VM through `tests/qemu-runner.sh`, shares +the repository read-only at `/mnt/host`, and executes commands over SSH. + +## Make and CI + +The Make targets take their suite list from the registry through +`scripts/conformance suites`. An empty registry makes suite targets print +`SKIP`; harness selftests still run. + +`.github/workflows/conformance.yml` runs QEMU before elfuse and gates on the +required `Conformance (make test-conformance)` job. Pull requests use the PR +scope. Schedules and `scope=full` dispatches use the full scope. diff --git a/docs/testing.md b/docs/testing.md index 01713ce8..29bbcfd4 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -423,6 +423,8 @@ The repository contains several layers of validation: - shell integration suites such as BusyBox, coreutils, and dynamic-loader tests - debugger integration tests for the GDB stub - native macOS HVF checks such as multi-vCPU and RWX validation +- conformance lanes that judge test suites against elfuse and a QEMU + reference; see `docs/conformance.md` The quick suite is driven by `tests/driver.sh`, which supports: @@ -436,6 +438,42 @@ Example: bash tests/driver.sh -f test-proc ``` +## Conformance Tests + +`scripts/conformance` runs registered suites on elfuse or QEMU. These are its +public commands: + +| Command | Result | +|---------|--------| +| `scripts/conformance suites [--format text\|json]` | List registered suites | +| `scripts/conformance list SUITE [--scope pr\|full] [--backend elfuse\|qemu\|all] [--format text\|json] [--require]` | List canonical case IDs; the default scope is `full` | +| `scripts/conformance run SUITE [--scope pr\|full] [--case ID_OR_GLOB] [--backend elfuse\|qemu\|all] [--jobs N] [--results DIR] [--bootstrap] [--require] [--no-retry] [--dry-run] [-v]` | Run cases; the defaults are the `pr` scope, elfuse, one job, and `build/conformance` | +| `scripts/conformance payload fingerprint SUITE` | Print the payload fingerprint | +| `scripts/conformance payload build SUITE [--force]` | Build the payload | +| `scripts/conformance payload verify SUITE [--fingerprint HASH]` | Verify the payload manifest and files | +| `scripts/conformance selection check SUITE` | Compare selection with the pinned inventory | +| `scripts/conformance selection update SUITE` | Rewrite generated selection | +| `scripts/conformance expectations check [SUITE]` | Validate expectation files | +| `scripts/conformance expectations seed SUITE RESULTS [--reason TEXT] [--write]` | Derive expectation actions from results | +| `scripts/conformance pins check [SUITE] [--ref REF]` | Report pin drift without writing | +| `scripts/conformance pins update SUITE [--ref REF]` | Update a pin | +| `scripts/conformance report RESULTS [--format text\|markdown\|json]` | Read canonical results | +| `scripts/conformance selftest` | Run harness selftests | + +Examples: + +```sh +scripts/conformance run SUITE --scope full --backend all +scripts/conformance run SUITE --case 'SUITE:GROUP/*' --backend qemu +scripts/conformance report RESULTS --format markdown +``` + +The Make aliases are `test-conformance-harness`, `test-conformance`, +`test-conformance-full`, `conformance-payloads`, `clean-payloads`, and +`update-pins`. `BACKEND`, `CONF_SCOPE`, `TEST`, `CONF_JOBS`, and +`CONF_RESULTS` configure the run targets. See [conformance.md](conformance.md) +for result, expectation, payload, and suite interfaces. + ## Validation Strategy By Change Type Suggested minimum validation: diff --git a/docs/usage.md b/docs/usage.md index 2ec20943..52a6a474 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -299,3 +299,8 @@ That has a few direct implications: work entirely inside the VM. Programs that link against `libfuse` (sshfs, ntfs-3g, AppImage runtimes) run without macFUSE, FUSE-T, or FSKit on the host. + +## Conformance testing + +See [testing.md](testing.md#conformance-tests) for the command reference and +[conformance.md](conformance.md) for the file and extension interfaces. diff --git a/mk/conformance.mk b/mk/conformance.mk new file mode 100644 index 00000000..af30828c --- /dev/null +++ b/mk/conformance.mk @@ -0,0 +1,53 @@ +.PHONY: test-conformance-harness test-conformance test-conformance-full \ + conformance-payloads clean-payloads update-pins + +CONFORMANCE := python3 scripts/conformance +# The suite registry lives in tests/conformance/providers/__init__.py. On a +# failed discovery the marker fails every consumer instead of skipping. +CONF_SUITES ?= $(shell $(CONFORMANCE) suites || echo suite-discovery-failed) +BACKEND ?= elfuse +TEST ?= +CONF_JOBS ?= 4 +CONF_RESULTS ?= $(BUILD_DIR)/conformance +CONF_RUN = $(CONFORMANCE) run +CONF_SCOPE ?= pr +CONF_SELECT = $(if $(TEST),$(foreach id,$(TEST),--case '$(id)'),--scope $(CONF_SCOPE)) +CONF_NO_SUITES = $(if $(CONF_SUITES),,@printf "$(YELLOW)SKIP$(RESET) no conformance suites registered\n") +# foreach inserts spaces, but RUN_OPTIONAL_SKIP77 expands as a recipe line. +define conf-newline + + +endef +define conf-lane +$(foreach s,$(CONF_SUITES),$(call RUN_OPTIONAL_SKIP77,$(CONF_RUN) $(s) $(1) --backend $(BACKEND) --jobs $(CONF_JOBS) --results $(CONF_RESULTS),test-$(s)$(2))$(conf-newline)) +endef + +## Run the conformance harness selftests (hermetic) +test-conformance-harness: + @$(CONFORMANCE) selftest + +## Run every suite's CONF_SCOPE subset, or TEST=ID... (BACKEND=elfuse|qemu|all) +test-conformance: + $(CONF_NO_SUITES) + $(call conf-lane,$(CONF_SELECT),) + +## Run every suite in full, the nightly shape +test-conformance-full: + $(CONF_NO_SUITES) + $(call conf-lane,--scope full,-full) + +## Build every conformance payload under externals/payloads/ +conformance-payloads: + $(CONF_NO_SUITES) + $(foreach s,$(CONF_SUITES),$(CONFORMANCE) payload build $(s) &&) true + +## Remove the conformance payloads (they survive clean and distclean) +clean-payloads: + rm -rf externals/payloads + +UPDATE_CHECK ?= + +## Refresh the conformance pins from upstream (UPDATE_CHECK=1 to report only) +update-pins: + $(CONF_NO_SUITES) + $(foreach s,$(CONF_SUITES),$(CONFORMANCE) pins $(if $(filter 1,$(UPDATE_CHECK)),check,update) $(s) $(if $(CONF_REF_$(s)),--ref $(CONF_REF_$(s))) &&) true diff --git a/mk/tests.mk b/mk/tests.mk index 3d2bbff7..1737bfd4 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -48,7 +48,7 @@ ELFUSE_HOST_NOFILE_MIN ?= $(shell bash "$(CURDIR)/tests/test-config.sh" --host-n test-sysroot-pathmax test-sysroot-corpus \ test-sysroot-name-soak check-soak \ check-name-caseexact test-sysroot-path-matrix \ - test-usage-synopsis \ + test-usage-synopsis test-qemu-runner-stop \ probe-volume-naming perf ## Build and run the assembly hello world test @@ -305,6 +305,8 @@ check: $(ELFUSE_BIN) $(TEST_DEPS) check-syscall-coverage check-eintr-contract ch $(call run-lane,test-rosetta-cli,rosetta CLI gating) $(call run-lane,test-bench-guardrail,hot-syscall guardrail) $(call run-lane,test-sharun,sharun launcher and probe) + $(call run-lane,test-conformance-harness,conformance harness selftests) + $(call run-lane,test-qemu-runner-stop,qemu-runner stop identity check) ## Hot-syscall performance guardrail: ensure getpid, libc clock_gettime, ## and 1-byte /dev/urandom reads stay under their TODO ns/op ceilings. @@ -1104,6 +1106,10 @@ test-launch-flags: $(ELFUSE_BIN) $(TEST_HELLO_DEP) $(TEST_ENV_DEPS) test-usage-synopsis: $(ELFUSE_BIN) @bash tests/test-usage-synopsis.sh $(ELFUSE_BIN) +## Check qemu-runner.sh stop against a recycled pid and the run's own process +test-qemu-runner-stop: + @bash tests/test-qemu-runner-stop.sh + ## Run GDB stub integration tests (LLDB <-> elfuse gdbstub) test-gdbstub: $(ELFUSE_BIN) $(TEST_DIR)/test-hello @bash tests/test-gdbstub.sh -e $(ELFUSE_BIN) -v diff --git a/scripts/conformance b/scripts/conformance new file mode 100755 index 00000000..073574a6 --- /dev/null +++ b/scripts/conformance @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "tests")) + +from conformance.cli import main # noqa: E402 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/proof-scope.py b/scripts/proof-scope.py index d7add432..e98bc5dd 100755 --- a/scripts/proof-scope.py +++ b/scripts/proof-scope.py @@ -349,6 +349,7 @@ def inert_name(name): MAKEFILE_INERT_INCLUDES = { "mk/shim.mk", "mk/tests.mk", + "mk/conformance.mk", "mk/lint.mk", "mk/format.mk", "mk/help.mk", diff --git a/tests/conformance/__init__.py b/tests/conformance/__init__.py new file mode 100644 index 00000000..62860527 --- /dev/null +++ b/tests/conformance/__init__.py @@ -0,0 +1,4 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +EXIT_OK, EXIT_RED, EXIT_USAGE, EXIT_DRIFT, EXIT_SKIP = 0, 1, 2, 3, 77 diff --git a/tests/conformance/backends/__init__.py b/tests/conformance/backends/__init__.py new file mode 100644 index 00000000..e5077194 --- /dev/null +++ b/tests/conformance/backends/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from conformance.backends.base import Backend, BackendError + +def make(name: str, repo_root: Path, **options: Any) -> Backend: + if name == "elfuse": + from conformance.backends.elfuse import ElfuseBackend as cls + elif name == "qemu": + from conformance.backends.qemu import QemuBackend as cls + else: + raise BackendError("unknown backend %r" % (name,)) + return cls(repo_root, **options) diff --git a/tests/conformance/backends/base.py b/tests/conformance/backends/base.py new file mode 100644 index 00000000..0db0c50b --- /dev/null +++ b/tests/conformance/backends/base.py @@ -0,0 +1,81 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import contextlib +import errno +import fcntl +import os +from pathlib import Path +from typing import Dict, Iterable, Iterator, List, Optional + +from conformance.model import Invocation + + +class BackendError(RuntimeError): + pass + + +FIXED_ENV = {"PATH": "/usr/bin:/bin", "LC_ALL": "C", "TZ": "UTC"} +SCRATCH_NAMES = ("HOME", "TMPDIR", "TEST_TMPDIR") +KILL_WAIT_S = 30 + + +def guest_environment(scratch: str, env: Optional[Dict[str, str]] = None) -> Dict[str, str]: + scratch = os.path.abspath(scratch) + return {**FIXED_ENV, **{name: scratch for name in SCRATCH_NAMES}, **(env or {})} + + +class Backend: + name = "" + max_jobs = 0 # 0: no cap on --jobs + lock_file: Optional[Path] = None # held by serialize() when set + + def prerequisites(self) -> Optional[str]: + """Return why the backend cannot run, or None when it can.""" + return None + + def start(self) -> None: + pass + + def stop(self) -> None: + pass + + def run( + self, + argv: List[str], + timeout_s: int, + scratch: Path, + env: Optional[Dict[str, str]] = None, + fetch: Iterable[str] = (), + ) -> Invocation: + """Run argv in a fresh guest cwd and return artifacts in scratch.""" + raise NotImplementedError + + def guest_path(self, host_path: Path) -> str: + return str(host_path) + + @contextlib.contextmanager + def serialize(self) -> Iterator[None]: + """Take a non-blocking flock when lock_file is set.""" + if self.lock_file is None: + yield + return + try: + # The lock lives in a shared namespace, so another uid may own it. + self.lock_file.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(self.lock_file, os.O_RDWR | os.O_CREAT, 0o600) + except OSError as e: + raise BackendError("cannot open %s: %s" % (self.lock_file, e)) from None + try: + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError as e: + if e.errno not in (errno.EWOULDBLOCK, errno.EAGAIN): + raise + raise BackendError("another %s conformance session holds %s" + % (self.name, self.lock_file)) from None + yield + finally: + os.close(fd) diff --git a/tests/conformance/backends/elfuse.py b/tests/conformance/backends/elfuse.py new file mode 100644 index 00000000..b9cfb20c --- /dev/null +++ b/tests/conformance/backends/elfuse.py @@ -0,0 +1,92 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import os +import signal +import subprocess +from pathlib import Path +from typing import Dict, Iterable, List, Optional + +from conformance.backends import base, proc +from conformance.model import Invocation + + +# Linux programs expect an 8 MiB initial stack. +WRAPPER = ["/bin/sh", "-c", 'ulimit -c 0; ulimit -s 8192; exec "$@"', "--"] + + +def orphan_pids(ps_output: str, binary: str, group: int) -> list: + """Limit orphan cleanup to fork children from one case process group.""" + out = [] + for line in ps_output.splitlines(): + fields = line.split(None, 3) + if len(fields) < 4: + continue + pid, ppid, pgid, command = fields + if (ppid == "1" and pgid == str(group) and command.startswith(binary + " ") + and "--fork-child" in command): + out.append(int(pid)) + return out + + +class ElfuseBackend(base.Backend): + name = "elfuse" + + def __init__(self, repo_root: Path, sysroot: Optional[Path] = None, + binary: Optional[Path] = None): + self.repo_root = repo_root + self.binary = binary or repo_root / "build" / "elfuse" + self.sysroot = sysroot + # One session per user: guest /dev/shm is a per-uid host directory. + self.lock_file = Path("/tmp/elfuse-conformance-%d.lock" % os.getuid()) + + def prerequisites(self) -> Optional[str]: + if not os.access(self.binary, os.X_OK): + return "%s is absent; run: make elfuse" % self.binary + if self.sysroot is not None and not self.sysroot.is_dir(): + return "sysroot %s is absent" % self.sysroot + return None + + def argv(self, guest_argv: List[str]) -> List[str]: + out = [str(self.binary), "--timeout", "0"] + if self.sysroot is not None: + out += ["--sysroot", str(self.sysroot)] + return out + list(guest_argv) + + def run( + self, + argv: List[str], + timeout_s: int, + scratch: Path, + env: Optional[Dict[str, str]] = None, + fetch: Iterable[str] = (), + ) -> Invocation: + full = base.guest_environment(str(scratch), env) + inv = proc.run_local(WRAPPER + self.argv(argv), timeout_s, scratch, env=full) + if inv.pid is not None: + self.reap_orphans(inv.pid) + return inv + + def reap_orphans(self, group: int) -> None: + """Release VMs held by fork children that outlived their case.""" + try: + # The case ran in its own session, so its pid is the pgid; an + # empty group means no fork child survived and ps can be skipped. + os.killpg(group, 0) + except (ProcessLookupError, PermissionError): + return + try: + listing = subprocess.run( + ["ps", "-eo", "pid=,ppid=,pgid=,command="], + capture_output=True, + text=True, + ) + except OSError: + return + for pid in orphan_pids(listing.stdout, str(self.binary), group): + try: + os.kill(pid, signal.SIGKILL) + except (ProcessLookupError, PermissionError): + pass diff --git a/tests/conformance/backends/proc.py b/tests/conformance/backends/proc.py new file mode 100644 index 00000000..ad79366a --- /dev/null +++ b/tests/conformance/backends/proc.py @@ -0,0 +1,73 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import os +import signal +import subprocess +import time +from pathlib import Path +from typing import Dict, List, Optional + +from conformance.backends.base import KILL_WAIT_S +from conformance.model import Invocation + + +def classify(rc: int, timed_out: bool, wall_us: int, stdout: str, stderr: str) -> Invocation: + """Interpret negative Popen return codes as signal deaths.""" + if timed_out: + return Invocation(execution="timeout", wall_us=wall_us, stdout=stdout, stderr=stderr) + if rc < 0: + return Invocation(execution="signal", wall_us=wall_us, signal=-rc, stdout=stdout, stderr=stderr) + return Invocation(execution="normal", wall_us=wall_us, exit_code=rc, stdout=stdout, stderr=stderr) + + +def _kill_and_reap(proc: subprocess.Popen) -> Optional[int]: + """Bound the wait for a guest stuck in an uninterruptible state.""" + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + return proc.wait(timeout=KILL_WAIT_S) + except subprocess.TimeoutExpired: + return None + + +def run_local( + argv: List[str], + timeout_s: int, + scratch: Path, + env: Optional[Dict[str, str]] = None, + stdout_name: str = "stdout", +) -> Invocation: + """Run argv in a new session so timeout cleanup reaches its group.""" + scratch.mkdir(parents=True, exist_ok=True) + out_path, err_path = scratch / stdout_name, scratch / "stderr" + started = time.monotonic() + with open(out_path, "wb") as out, open(err_path, "wb") as err, \ + open(os.devnull, "rb") as feed: + try: + proc = subprocess.Popen(argv, cwd=str(scratch), env=env, stdin=feed, + stdout=out, stderr=err, start_new_session=True) + except OSError as e: + err.write(("cannot spawn %s: %s\n" % (argv[0], e)).encode()) + return Invocation(execution="transport", wall_us=int((time.monotonic() - started) * 1_000_000), + stdout=str(out_path), stderr=str(err_path)) + timed_out = False + try: + rc = proc.wait(timeout=timeout_s) + except subprocess.TimeoutExpired: + timed_out = True + rc = _kill_and_reap(proc) + if rc is None: + err.write(("process group %d was not reaped after SIGKILL\n" % proc.pid).encode()) + wall_us = int((time.monotonic() - started) * 1_000_000) + if rc is None: + inv = Invocation(execution="transport", wall_us=wall_us, + stdout=str(out_path), stderr=str(err_path)) + else: + inv = classify(rc, timed_out, wall_us, str(out_path), str(err_path)) + inv.pid = proc.pid # start_new_session makes pid the process-group id + return inv diff --git a/tests/conformance/backends/qemu.py b/tests/conformance/backends/qemu.py new file mode 100644 index 00000000..b16423e6 --- /dev/null +++ b/tests/conformance/backends/qemu.py @@ -0,0 +1,121 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path +from typing import Dict, Iterable, List, Optional + +from conformance.backends import base +from conformance.backends.ssh import SshSession +from conformance.model import Invocation + +FIXTURES = ("kernel/vmlinuz-virt", "initramfs.cpio.gz", "keys/ssh_key") + +RUNNER_TIMEOUT_S = 600 +# Match the environment passed to qemu-runner.sh. +RUNNER_PATH = "/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin" + + +def parse_state(text: str) -> Dict[str, str]: + out = {} + for line in text.splitlines(): + key, sep, value = line.partition("=") + if sep: + out[key.strip()] = value.strip() + return out + + +class QemuBackend(base.Backend): + name = "qemu" + max_jobs = 1 # the reference accepts one command at a time + + def __init__(self, repo_root: Path, mem_mib: int = 2048, + runner: Optional[Path] = None, state_dir: Optional[Path] = None): + self.repo_root = repo_root + self.mem_mib = mem_mib + self.runner = runner or repo_root / "tests" / "qemu-runner.sh" + self.state_file = (state_dir or repo_root / "build" / "conformance") / "qemu.state" + # One session per checkout: start() reaps whatever the state file names. + self.lock_file = self.state_file.with_suffix(".lock") + self.session: Optional[SshSession] = None + + def prerequisites(self) -> Optional[str]: + fixtures = self.repo_root / "externals" / "test-fixtures" + missing = [f for f in FIXTURES if not (fixtures / f).is_file()] + if missing: + return "QEMU fixtures missing (%s); run: bash tests/fetch-fixtures.sh" % ", ".join(missing) + if shutil.which("qemu-system-aarch64", path=RUNNER_PATH) is None: + return ("qemu-system-aarch64 is not on the runner PATH (%s); " + "run: brew install qemu" % RUNNER_PATH) + return None + + def _runner(self, verb: str) -> subprocess.CompletedProcess: + self.state_file.parent.mkdir(parents=True, exist_ok=True) + with subprocess.Popen( + ["bash", str(self.runner), verb, "--state-file", str(self.state_file)], + cwd=str(self.repo_root), + env={**{k: v for k, v in os.environ.items() if k.startswith("QEMU_") or k == "TMPDIR"}, + "PATH": RUNNER_PATH, "QEMU_MEM": str(self.mem_mib), "HOME": str(Path.home())}, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + ) as run: + try: + out, _ = run.communicate(timeout=RUNNER_TIMEOUT_S) + except subprocess.TimeoutExpired: + # SIGTERM, not SIGKILL, so the runner's EXIT trap reaps the VM. + run.terminate() + try: + run.communicate(timeout=base.KILL_WAIT_S) + except subprocess.TimeoutExpired: + # Popen.__exit__ waits unconditionally; kill or hang here. + run.kill() + run.communicate() + raise base.BackendError("qemu-runner %s did not return in %ds" % (verb, RUNNER_TIMEOUT_S)) from None + return subprocess.CompletedProcess(run.args, run.returncode, out) + + def start(self) -> None: + # A state file from an interrupted run names a VM still up; reap it + # first or this start would overwrite the only record of it. + self.stop() + done = self._runner("start") + if done.returncode != 0: + raise base.BackendError("qemu-runner start failed:\n%s" % done.stdout) + try: + state = parse_state(self.state_file.read_text()) if self.state_file.exists() else {} + if "port" not in state or "key" not in state: + raise base.BackendError("qemu-runner wrote no port/key to %s" % self.state_file) + self.session = SshSession(int(state["port"]), Path(state["key"])) + except (base.BackendError, ValueError, OSError) as e: + self.stop() + raise base.BackendError(str(e)) from None + + def stop(self) -> None: + if self.session is not None or self.state_file.exists(): + done = self._runner("stop") + if done.returncode != 0: + raise base.BackendError("qemu-runner stop failed:\n%s" % done.stdout) + self.session = None + + def guest_path(self, host_path: Path) -> str: + try: + rel = Path(host_path).resolve().relative_to(self.repo_root.resolve()) + except ValueError: + raise base.BackendError( + "%s is outside the repo root, unreachable over the 9p share" % host_path + ) from None + return "/mnt/host/%s" % rel.as_posix() + + def run( + self, + argv: List[str], + timeout_s: int, + scratch: Path, + env: Optional[Dict[str, str]] = None, + fetch: Iterable[str] = (), + ) -> Invocation: + if self.session is None: + raise base.BackendError("qemu backend is not started") + return self.session.run(argv, timeout_s, scratch, env=env, fetch=fetch) diff --git a/tests/conformance/backends/ssh.py b/tests/conformance/backends/ssh.py new file mode 100644 index 00000000..65db5447 --- /dev/null +++ b/tests/conformance/backends/ssh.py @@ -0,0 +1,110 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 +# Alpine's initramfs has no sftp-server, so files return through ssh cat. + +from __future__ import annotations + +import os +import shlex +from pathlib import Path +from typing import Dict, Iterable, List, Optional + +from conformance.backends import base, proc +from conformance.model import Invocation + +SENTINEL = "__CONF_RC=" +DIR_MARK = " __CONF_DIR=" +TRANSPORT_SLACK_S = 15 + + +class SshSession: + def __init__(self, port: int, key: Path, host: str = "127.0.0.1", user: str = "root", + ssh: str = "ssh"): + self.port, self.key, self.host, self.user = port, key, host, user + self.ssh = ssh + + def options(self) -> List[str]: + # The shell lanes spell the same list in tests/lib/qemu-ssh.sh. + return [ + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "LogLevel=ERROR", + "-o", "BatchMode=yes", + "-o", "ConnectTimeout=10", + "-o", "ServerAliveInterval=10", + "-o", "ServerAliveCountMax=6", + "-i", str(self.key), + ] + + def ssh_argv(self, script: str) -> List[str]: + return [self.ssh] + self.options() + ["-p", str(self.port), + "%s@%s" % (self.user, self.host), script] + + @staticmethod + def remote_script(argv: List[str], timeout_s: int, env: Dict[str, str], + cwd: Optional[str], cleanup: bool = False) -> str: + """Build the guest command with an isolated cwd and fixed environment.""" + exports = "export %s;" % " ".join('%s="$PWD"' % n for n in base.SCRATCH_NAMES) + "".join( + " export %s=%s;" % (k, shlex.quote(v)) for k, v in sorted({**base.FIXED_ENV, **env}.items())) + enter = "cd %s" % shlex.quote(cwd) if cwd else 'd=$(mktemp -d /tmp/conf.XXXXXX) && cd "$d"' + if cleanup: + enter += " && trap 'cd / && rm -rf \"$d\"' EXIT" + return ( + "%s && { %s /usr/bin/timeout -s KILL %d %s; rc=$?; " + 'printf "\\n%s%%s%s%%s\\n" "$rc" "$PWD"; }' + % (enter, exports, timeout_s, shlex.join(argv), SENTINEL, DIR_MARK) + ) + + @staticmethod + def parse_sentinel(text: str) -> Optional[tuple]: + # DIR_MARK terminates rc and detects a partial sentinel write. + for line in reversed(text.splitlines()): + if line.startswith(SENTINEL): + head, mark, tail = line.partition(DIR_MARK) + field = head[len(SENTINEL):].split() + if not mark or not field or not field[0].lstrip("-").isdigit(): + return None + return int(field[0]), tail + return None + + def run(self, argv: List[str], timeout_s: int, scratch: Path, + env: Optional[Dict[str, str]] = None, cwd: Optional[str] = None, + fetch: Iterable[str] = ()) -> Invocation: + cleanup = not cwd and not fetch + script = self.remote_script(argv, timeout_s, env or {}, cwd, cleanup) + inv = proc.run_local(self.ssh_argv(script), timeout_s + TRANSPORT_SLACK_S, scratch, + stdout_name="stdout.raw") + raw_path, out_path = Path(inv.stdout), scratch / "stdout" + raw = raw_path.read_bytes() + parsed = self.parse_sentinel(raw.decode("utf-8", "replace")) + # A clean ssh exit vouches for the sentinel; a lost tail can leave a + # guest-printed lookalike as the last line. + if parsed is None or inv.execution != "normal" or inv.exit_code != 0: + raw_path.replace(out_path) + # An uninterruptible guest can outlive both timeout and SIGKILL. + return Invocation(execution="timeout" if inv.execution == "timeout" else "transport", + wall_us=inv.wall_us, + stdout=str(out_path), stderr=inv.stderr) + rc, guest_dir = parsed + os.truncate(raw_path, max(raw.rfind(SENTINEL.encode()) - 1, 0)) + raw_path.replace(out_path) + if guest_dir: + lost = [name for name in fetch + if not self.copy_from("%s/%s" % (guest_dir, name), scratch / name)] + if not cwd and fetch: + proc.run_local(self.ssh_argv("rm -rf %s" % shlex.quote(guest_dir)), 30, + scratch / ".cleanup") + if lost: + return Invocation(execution="transport", wall_us=inv.wall_us, + stdout=str(out_path), stderr=inv.stderr) + timed_out = rc == 137 and inv.wall_us >= timeout_s * 1_000_000 + # The shell status cannot distinguish signal death from exit(128+n). + return proc.classify(rc, timed_out, inv.wall_us, str(out_path), inv.stderr) + + def copy_from(self, remote: str, local: Path) -> bool: + inv = proc.run_local(self.ssh_argv("cat %s" % shlex.quote(remote)), 120, + local.parent / ".fetch", stdout_name=local.name + ".part") + if inv.exit_code != 0: + return False + Path(inv.stdout).replace(local) + return True diff --git a/tests/conformance/cli.py b/tests/conformance/cli.py new file mode 100644 index 00000000..89b8375e --- /dev/null +++ b/tests/conformance/cli.py @@ -0,0 +1,446 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import argparse +import contextlib +import datetime +import json +import os +import sys +import time +import unittest +from pathlib import Path +from typing import Callable, Iterator, List, Optional + +from conformance import EXIT_DRIFT, EXIT_OK, EXIT_RED, EXIT_SKIP, EXIT_USAGE +from conformance import backends, expectations, jsonc, payload, providers, report +from conformance import runner, seed, selection +from conformance.backends.base import BackendError +from conformance.model import Status +from conformance.providers.base import Provider, ProviderError + +REPO_ROOT = Path(__file__).resolve().parents[2] +BACKENDS = ("elfuse", "qemu", "all") + + +class Stop(Exception): + def __init__(self, code: int): + super().__init__(code) + self.code = code + + +class Cli: + def __init__(self, repo_root: Path, out: Callable[[str], None] = print, + err: Optional[Callable[[str], None]] = None): + self.repo_root = repo_root + self.out = out + self.err = err or out + + def fail(self, message: str) -> None: + self.err("conformance: " + message) + + def skip(self, args: argparse.Namespace, message: str) -> int: + self.fail(message) + required = args.require or os.environ.get("CONF_REQUIRE") == "1" + return EXIT_USAGE if required else EXIT_SKIP + + def provider(self, name: str) -> Provider: + return providers.make(name, self.repo_root) + + @staticmethod + def backend_names(name: str) -> List[str]: + return ["qemu", "elfuse"] if name == "all" else [name] + + def make_backend(self, args: argparse.Namespace, provider: Provider, name: str, + verify_payload: bool = False) -> backends.Backend: + absent = provider.prerequisites(name) + if absent: + raise Stop(self.skip(args, absent)) + if verify_payload: + try: + payload.verify(provider.payload_root(), provider.fingerprint()) + except payload.PayloadError as e: + self.fail(str(e)) + raise Stop(EXIT_USAGE) from None + try: + backend = backends.make( + name, self.repo_root, **provider.backend_options(name) + ) + except BackendError as e: + self.fail(str(e)) + raise Stop(EXIT_USAGE) from None + absent = backend.prerequisites() + if absent: + raise Stop(self.skip(args, absent)) + return backend + + @contextlib.contextmanager + def started(self, args: argparse.Namespace, + backend: backends.Backend) -> Iterator[None]: + with backend.serialize(): + try: + backend.start() + except BackendError as e: + raise Stop(self.skip(args, str(e))) from None + try: + yield + except BaseException: + # A failing stop must not replace the in-flight error. + try: + backend.stop() + except BackendError as e: + self.fail("backend stop failed: %s" % e) + raise + backend.stop() + + def emit_list(self, args: argparse.Namespace, kind: str, key: str, + items: List, lines: List[str], **extra) -> int: + if args.format == "json": + self.out(json.dumps({"schema_version": 1, "kind": kind, key: items, + **extra}, sort_keys=True)) + else: + for line in lines: + self.out(line) + return EXIT_OK + + def suites(self, args: argparse.Namespace) -> int: + names = sorted(providers.REGISTRY) + return self.emit_list(args, "suite-list", "suites", names, names) + + def list_cases(self, args: argparse.Namespace) -> int: + provider = self.provider(args.suite) + name = "elfuse" if args.backend == "all" else args.backend + try: + backend = self.make_backend(args, provider, name) + with self.started(args, backend): + cases = provider.enumerate( + backend, provider.selection.groups(args.scope) + ) + except Stop as e: + return e.code + except BackendError as e: + self.fail("backend error: %s" % e) + return EXIT_RED + return self.emit_list( + args, "case-list", "cases", + [{"id": c.id, "group": c.group, "scope": c.scope, + "timeout_s": c.timeout_s} for c in cases], + [c.id for c in cases], suite=args.suite, scope=args.scope) + + def run(self, args: argparse.Namespace) -> int: + provider = self.provider(args.suite) + codes = [self.run_one(args, provider, name) + for name in self.backend_names(args.backend)] + for code in (EXIT_RED, EXIT_USAGE, EXIT_SKIP): + if code in codes: + return code + return EXIT_OK + + def run_one(self, args: argparse.Namespace, provider: Provider, + backend_name: str) -> int: + try: + backend = self.make_backend( + args, provider, backend_name, verify_payload=True + ) + exps = expectations.load( + provider.name, backend_name, provider.expectations_dir + ) + except Stop as e: + return e.code + except (expectations.ExpectationError, jsonc.JsoncError) as e: + self.fail(str(e)) + return EXIT_USAGE + selected_scope = "full" if args.case else args.scope + result_scope = "cases" if args.case else args.scope + results_dir = self.results_dir(args, provider.name, backend_name) + started = time.monotonic() + try: + with self.started(args, backend): + cases = provider.enumerate( + backend, provider.selection.groups(selected_scope) + ) + if args.case: + chosen, errors = selection.resolve_ids( + args.case, [c.id for c in cases], provider.name + ) + for error in errors: + self.fail(error) + if errors: + return EXIT_USAGE + wanted = set(chosen) + cases = [case for case in cases if case.id in wanted] + if args.dry_run: + for case in cases: + self.out(case.id) + return EXIT_OK + log = self.err if args.verbose else (lambda _: None) + results = runner.run_lane( + provider, backend, cases, exps, results_dir, args.jobs, + not args.no_retry, args.bootstrap, log + ) + meta = { + "suite": provider.name, + "backend": backend_name, + "scope": result_scope, + "cases": list(args.case), + "started": datetime.datetime.now( + datetime.timezone.utc + ).isoformat(timespec="seconds"), + "elapsed_s": round(time.monotonic() - started, 3), + "bootstrap": args.bootstrap, + "argv": sys.argv[1:], + } + # Written before stop(), so a failing teardown cannot lose the lane. + report.write(results_dir, meta, results) + for line in report.summary_lines(meta, results, results_dir): + self.out(line) + except Stop as e: + return e.code + except BackendError as e: + self.fail("backend error: %s" % e) + return EXIT_RED + if result_scope == "full" and not args.bootstrap: + stale = exps.stale([case.id for case in cases]) + for problem in stale: + self.fail("stale expectation, " + problem) + if stale: + return EXIT_USAGE + if args.bootstrap: + return EXIT_RED if any( + case.status is Status.ERROR for case in results + ) else EXIT_OK + return EXIT_OK if report.gate(results) == "green" else EXIT_RED + + @staticmethod + def results_dir(args: argparse.Namespace, suite: str, backend: str) -> Path: + stamp = datetime.datetime.now(datetime.timezone.utc).strftime( + "%Y%m%dT%H%M%SZ" + ) + return Path(args.results) / suite / backend / ( + "%s-%d" % (stamp, os.getpid()) + ) + + def payload_fingerprint(self, args: argparse.Namespace) -> int: + self.out(self.provider(args.suite).fingerprint()) + return EXIT_OK + + def payload_verify(self, args: argparse.Namespace) -> int: + provider = self.provider(args.suite) + try: + payload.verify( + provider.payload_root(), args.fingerprint or provider.fingerprint() + ) + except payload.PayloadError as e: + self.fail(str(e)) + return EXIT_RED + self.out("%s: verified" % provider.payload_root()) + return EXIT_OK + + def payload_build(self, args: argparse.Namespace) -> int: + try: + self.provider(args.suite).build_payload(force=args.force) + except (payload.PayloadError, ProviderError) as e: + self.fail(str(e)) + if isinstance(e, payload.PayloadError) and e.kind != "config": + return EXIT_RED + return EXIT_USAGE + return EXIT_OK + + def selection_sync(self, args: argparse.Namespace) -> int: + lines = self.provider(args.suite).regen_selection(check=args.check) + for line in lines: + (self.fail if args.check else self.out)(line) + if not lines: + self.out("%s: selection is current" % args.suite) + return EXIT_DRIFT if args.check and lines else EXIT_OK + + def expectations_check(self, args: argparse.Namespace) -> int: + names = [args.suite] if args.suite else sorted(providers.REGISTRY) + problems: List[str] = [] + for name in names: + root = self.provider(name).expectations_dir + if root.is_dir(): + problems.extend(expectations.lint(root)) + for problem in problems: + self.fail(problem) + if not problems: + self.out("expectations: valid") + return EXIT_USAGE if problems else EXIT_OK + + def expectations_seed(self, args: argparse.Namespace) -> int: + provider = self.provider(args.suite) + meta, cases = report.load(Path(args.results)) + if meta.get("suite") != provider.name: + self.fail("%s does not contain %s results" % ( + args.results, provider.name + )) + return EXIT_USAGE + reason = args.reason or seed.default_reason( + Path(args.results), str(meta.get("started", ""))[:10] + ) + actions = seed.propose( + cases, reason, bool(meta.get("bootstrap")), + whole_groups=meta.get("scope") != "cases" + ) + if not actions: + self.out("expectations: no changes") + return EXIT_OK + if not args.write: + self.out(seed.format_actions(actions).rstrip("\n")) + return EXIT_OK + backend = meta.get("backend") + if not backend: + self.fail("results name no backend") + return EXIT_USAGE + leaf = expectations.leaf_path( + provider.expectations_dir, provider.name, backend + ) + seed.append(leaf, actions) + problems = expectations.lint(provider.expectations_dir, seeded_ok=True) + for problem in problems: + self.fail(problem) + self.out("%s: appended %d actions" % (leaf, len(actions))) + return EXIT_USAGE if problems else EXIT_OK + + def pins(self, args: argparse.Namespace) -> int: + names = [args.suite] if args.suite else sorted(providers.REGISTRY) + codes = [payload.refresh(self.provider(name), args.ref, args.check, + self.out, self.fail) for name in names] + for code in (EXIT_USAGE, EXIT_DRIFT): + if code in codes: + return code + return EXIT_OK + + def report(self, args: argparse.Namespace) -> int: + root = Path(args.results) + if args.format == "markdown": + self.out(report.markdown(root).rstrip("\n")) + return EXIT_OK + meta, cases = report.load(root) + if args.format == "json": + self.out(json.dumps(report.document(meta, cases), sort_keys=True)) + else: + for line in report.summary_lines(meta, cases, root): + self.out(line) + return EXIT_OK if report.gate(cases) == "green" else EXIT_RED + + def selftest(self, args: argparse.Namespace) -> int: + suite = unittest.defaultTestLoader.discover( + str(self.repo_root / "tests" / "conformance" / "selftest"), + top_level_dir=str(self.repo_root / "tests"), + ) + result = unittest.TextTestRunner(verbosity=1).run(suite) + return EXIT_OK if result.wasSuccessful() else EXIT_RED + + +def common_run(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--backend", choices=BACKENDS, default="elfuse") + parser.add_argument("--results", default="build/conformance") + parser.add_argument("--jobs", type=int, default=1) + parser.add_argument("--bootstrap", action="store_true") + parser.add_argument("--require", action="store_true") + parser.add_argument("--no-retry", action="store_true") + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("-v", "--verbose", action="store_true") + + +def command(parent: argparse._SubParsersAction, name: str, handler: str): + parser = parent.add_parser(name) + parser.set_defaults(handler=handler) + return parser + + +def family(parent: argparse._SubParsersAction, name: str): + parser = parent.add_parser(name) + return parser.add_subparsers(dest=name + "_command", required=True) + + +def suite_arg(parser: argparse.ArgumentParser, suites: List[str]) -> None: + parser.add_argument("suite", choices=suites) + + +def build_parser(suites: List[str]) -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="conformance", + description="Run registered Linux test suites on elfuse or QEMU.", + ) + top = parser.add_subparsers(dest="command", required=True) + p = command(top, "suites", "suites") + p.add_argument("--format", choices=("text", "json"), default="text") + + p = command(top, "list", "list_cases") + suite_arg(p, suites) + p.add_argument("--scope", choices=selection.SCOPES, default="full") + p.add_argument("--format", choices=("text", "json"), default="text") + p.add_argument("--backend", choices=BACKENDS, default="elfuse") + p.add_argument("--require", action="store_true") + + p = command(top, "run", "run") + suite_arg(p, suites) + p.add_argument("--scope", choices=selection.SCOPES, default="pr") + p.add_argument("--case", action="append", default=[]) + common_run(p) + + sub = family(top, "payload") + p = command(sub, "fingerprint", "payload_fingerprint") + suite_arg(p, suites) + p = command(sub, "build", "payload_build") + suite_arg(p, suites) + p.add_argument("--force", action="store_true") + p = command(sub, "verify", "payload_verify") + suite_arg(p, suites) + p.add_argument("--fingerprint") + + sub = family(top, "selection") + for name, check in (("check", True), ("update", False)): + p = command(sub, name, "selection_sync") + p.set_defaults(check=check) + suite_arg(p, suites) + + sub = family(top, "expectations") + p = command(sub, "check", "expectations_check") + p.add_argument("suite", nargs="?", choices=suites) + p = command(sub, "seed", "expectations_seed") + suite_arg(p, suites) + p.add_argument("results") + p.add_argument("--reason") + p.add_argument("--write", action="store_true") + + sub = family(top, "pins") + p = command(sub, "check", "pins") + p.set_defaults(check=True) + p.add_argument("suite", nargs="?", choices=suites) + p.add_argument("--ref") + p = command(sub, "update", "pins") + p.set_defaults(check=False) + suite_arg(p, suites) + p.add_argument("--ref") + + p = command(top, "report", "report") + p.add_argument("results") + p.add_argument("--format", choices=("text", "markdown", "json"), + default="text") + command(top, "selftest", "selftest") + return parser + + +def main(argv: Optional[List[str]] = None, repo_root: Path = REPO_ROOT, + out: Callable[[str], None] = print, + err: Optional[Callable[[str], None]] = None) -> int: + parser = build_parser(sorted(providers.REGISTRY)) + args = parser.parse_args(argv) + if err is None: + err = lambda message: print(message, file=sys.stderr) + cli = Cli(repo_root, out, err) + try: + return getattr(cli, args.handler)(args) + except (ProviderError, selection.SelectionError, payload.PinError, + jsonc.JsoncError, expectations.ExpectationError, report.ReportError, + seed.SeedError) as e: + cli.fail(str(e)) + return EXIT_USAGE + except NotImplementedError as e: + cli.fail(str(e) or "operation is not supported") + return EXIT_USAGE diff --git a/tests/conformance/elfcheck.py b/tests/conformance/elfcheck.py new file mode 100644 index 00000000..3c1ff2e3 --- /dev/null +++ b/tests/conformance/elfcheck.py @@ -0,0 +1,90 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import struct +from dataclasses import dataclass, field +from pathlib import Path +from typing import List, Optional, Tuple + +EM_AARCH64 = 183 +PT_LOAD, PT_DYNAMIC, PT_INTERP = 1, 2, 3 +DT_NULL, DT_NEEDED, DT_STRTAB = 0, 1, 5 + + +class ElfError(ValueError): + pass + + +@dataclass +class DynamicInfo: + machine: int + interp: Optional[str] = None + needed: List[str] = field(default_factory=list) + has_load: bool = False + + +def _headers(data: bytes, path: Path) -> Tuple[int, list]: + if len(data) < 64 or data[:4] != b"\x7fELF": + raise ElfError("%s: not an ELF file" % path) + if data[4] != 2 or data[5] != 1: + raise ElfError("%s: not ELF64 little-endian" % path) + machine = struct.unpack_from(" len(data): + raise ElfError("%s: program header table out of range" % path) + phdrs = [struct.unpack_from(" Optional[int]: + for p_type, _, p_offset, p_vaddr, _, p_filesz, _, _ in phdrs: + if p_type == PT_LOAD and p_vaddr <= vaddr < p_vaddr + p_filesz: + return p_offset + (vaddr - p_vaddr) + return None + + +def _cstring(data: bytes, offset: int, path: Path) -> str: + end = data.find(b"\0", offset) + if end < 0: + raise ElfError("%s: string out of range" % path) + return data[offset:end].decode("ascii", "replace") + + +def read_dynamic(path: Path) -> DynamicInfo: + data = path.read_bytes() + machine, phdrs = _headers(data, path) + info = DynamicInfo(machine) + for p_type, _, p_offset, _, _, p_filesz, _, _ in phdrs: + if p_type == PT_LOAD: + info.has_load = True + elif p_type == PT_INTERP: + info.interp = _cstring(data, p_offset, path) + elif p_type == PT_DYNAMIC: + if p_offset + p_filesz > len(data): + raise ElfError("%s: dynamic segment out of range" % path) + entries = [struct.unpack_from(" None: + info = read_dynamic(path) + if info.machine != EM_AARCH64: + raise ElfError("%s: machine %d is not AArch64" % (path, info.machine)) + if not info.has_load: + raise ElfError("%s: no PT_LOAD segment" % path) + if info.interp is not None: + raise ElfError("%s: has PT_INTERP %s, not static" % (path, info.interp)) + if info.needed: + raise ElfError("%s: needs %s, not static" % (path, ", ".join(info.needed))) diff --git a/tests/conformance/expectations.py b/tests/conformance/expectations.py new file mode 100644 index 00000000..d6f4d9d4 --- /dev/null +++ b/tests/conformance/expectations.py @@ -0,0 +1,214 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Tuple + +from conformance import ids, jsonc + +ACTION_TYPES = ("expect_pass", "expect_failure", "expect_conf", "skip", "quarantine") +_ACTION_KEYS = {"type", "matchers", "reason", "since", "tracking"} +_TRACKING_RE = re.compile(r"^(#\d+|https?://\S+)$") +_SINCE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") +SEEDED_PREFIX = "seeded from " +FLAKY = "flaky.jsonc" + + +class ExpectationError(ValueError): + pass + + +@dataclass(frozen=True) +class Action: + type: str + matchers: tuple + reason: str + source: str + since: str = "" + tracking: str = "" + + +@dataclass(frozen=True) +class Resolution: + type: str + reason: str + source: str + matcher: str + quarantined: bool = False + + def to_dict(self) -> Dict[str, Any]: + return { + "type": self.type, + "reason": self.reason, + "source": self.source, + "matcher": self.matcher, + "quarantined": self.quarantined, + } + + +class Expectations: + def __init__(self, suite: str, actions: List[Action]): + self.suite = suite + self.actions = list(actions) + first = next((a for a in actions if a.type != "quarantine"), None) + if first is None or first.type != "expect_pass" or first.matchers != ("*",): + raise ExpectationError( + 'the first effective action must be expect_pass on "*"' + ) + + def resolve(self, test_id: str) -> Resolution: + chosen: Optional[Action] = None + chosen_matcher = "" + quarantined = False + for action in self.actions: + for m in action.matchers: + if ids.matches(m, test_id): + if action.type == "quarantine": + quarantined = True + else: + chosen, chosen_matcher = action, m + assert chosen is not None + return Resolution( + type=chosen.type, + reason=chosen.reason, + source=chosen.source, + matcher=chosen_matcher, + quarantined=quarantined, + ) + + def stale(self, known: Iterable[str]) -> List[str]: + universe = list(known) + out = [] + for action in self.actions: + for m in action.matchers: + if m == "*" or ids.suite_of(m) != self.suite: + continue + if not any(ids.matches(m, i) for i in universe): + out.append("%s: %r matches no test" % (action.source, m)) + return out + + +def _check_text(text: str, where: str) -> None: + if "\u2014" in text: + raise ExpectationError("%s: em dash in text" % where) + + +def _parse_action(doc: Any, where: str, suite: str) -> Action: + if not isinstance(doc, dict): + raise ExpectationError("%s: action is not an object" % where) + unknown = set(doc) - _ACTION_KEYS + if unknown: + raise ExpectationError("%s: unknown keys %s" % (where, sorted(unknown))) + kind = doc.get("type") + if kind not in ACTION_TYPES: + raise ExpectationError("%s: unknown action type %r" % (where, kind)) + matchers = doc.get("matchers") + if not isinstance(matchers, list) or not matchers: + raise ExpectationError("%s: matchers must be a non-empty list" % where) + if any(not isinstance(m, str) for m in matchers): + raise ExpectationError("%s: matchers must be strings" % where) + if matchers != sorted(matchers): + raise ExpectationError("%s: matchers are not sorted" % where) + if len(set(matchers)) != len(matchers): + raise ExpectationError("%s: duplicate matcher" % where) + for m in matchers: + if m == "*": + if kind != "expect_pass": + raise ExpectationError('%s: "*" is legal only on expect_pass' % where) + continue + if ids.suite_of(m) != suite: + raise ExpectationError("%s: matcher %r is not in suite %s" % (where, m, suite)) + if not ids.is_valid(m.replace("*", "x").replace("?", "x")): + raise ExpectationError("%s: matcher %r is not an id pattern" % (where, m)) + reason = doc.get("reason", "") + if not isinstance(reason, str): + raise ExpectationError("%s: reason must be a string" % where) + if kind != "expect_pass" and not reason.strip(): + raise ExpectationError("%s: %s needs a reason" % (where, kind)) + _check_text(reason, where) + since = doc.get("since", "") + if since and not (isinstance(since, str) and _SINCE_RE.match(since)): + raise ExpectationError("%s: since must be YYYY-MM-DD" % where) + tracking = doc.get("tracking", "") + if tracking and not (isinstance(tracking, str) and _TRACKING_RE.match(tracking)): + raise ExpectationError("%s: tracking must be #N or a URL" % where) + return Action(kind, tuple(matchers), reason, where, since, tracking) + + +def read_file(path: Path, suite: Optional[str] = None, + seen: Optional[List[Path]] = None) -> List[Action]: + """With no suite, each action's suite comes from its first matcher.""" + seen = list(seen or []) + if path in seen: + raise ExpectationError("%s: include cycle" % path) + if len(seen) > 8: + raise ExpectationError("%s: include chain too deep" % path) + seen.append(path) + out: List[Action] = [] + for where, entry in _entries(path): + if isinstance(entry, dict) and "include" in entry: + if suite is None: + raise ExpectationError("%s: include is not legal in %s" % (where, FLAKY)) + if set(entry) != {"include"} or not isinstance(entry["include"], str): + raise ExpectationError("%s: include takes only a file name" % where) + out.extend(read_file(path.parent / entry["include"], suite, seen)) + continue + if suite is None: + matchers = entry.get("matchers") if isinstance(entry, dict) else None + first = matchers[0] if isinstance(matchers, list) and matchers else "" + entry_suite = ids.suite_of(first) if isinstance(first, str) else "" + else: + entry_suite = suite + action = _parse_action(entry, where, entry_suite) + if action.type == "quarantine" and path.name != FLAKY: + raise ExpectationError("%s: quarantine is legal only in %s" % (where, FLAKY)) + if path.name == FLAKY and action.type != "quarantine": + raise ExpectationError("%s: %s holds only quarantine actions" % (where, FLAKY)) + out.append(action) + return out + + +def _entries(path: Path) -> List[Tuple[str, Any]]: + if not path.exists(): + raise ExpectationError("%s: no such file" % path) + doc = jsonc.load(path) + if not isinstance(doc, dict) or set(doc) != {"actions"}: + raise ExpectationError('%s: expected an object with only "actions"' % path) + if not isinstance(doc["actions"], list): + raise ExpectationError("%s: actions must be a list" % path) + return [("%s:%d" % (path.name, index), entry) for index, entry in enumerate(doc["actions"])] + + +def leaf_path(root: Path, suite: str, backend: str) -> Path: + return root / ("%s_%s.jsonc" % (suite, backend)) + + +def load(suite: str, backend: str, root: Path) -> Expectations: + actions = read_file(leaf_path(root, suite, backend), suite) + flaky = root / FLAKY + if flaky.exists(): + actions.extend(a for a in read_file(flaky) if ids.suite_of(a.matchers[0]) == suite) + return Expectations(suite, actions) + + +def lint(root: Path, seeded_ok: bool = False) -> List[str]: + problems: List[str] = [] + seeded: List[str] = [] + for path in sorted(root.glob("*.jsonc")): + try: + suite = None if path.name == FLAKY else path.stem.split("_", 1)[0] + actions = read_file(path, suite) + if suite and "_" in path.stem: + Expectations(suite, actions) + except (ExpectationError, jsonc.JsoncError) as e: + problems.append(str(e)) + continue + seeded.extend(a.source for a in actions if a.reason.startswith(SEEDED_PREFIX)) + if not seeded_ok: + problems += ["%s: still carries a seeded reason; triage it" % s for s in seeded] + # A base file is read again through every leaf that includes it. + return list(dict.fromkeys(problems)) diff --git a/tests/conformance/ids.py b/tests/conformance/ids.py new file mode 100644 index 00000000..b8951a78 --- /dev/null +++ b/tests/conformance/ids.py @@ -0,0 +1,57 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import fnmatch +import hashlib +import re +from typing import Iterable, List, Optional, Tuple + +_ID_RE = re.compile( + r"^(?P[a-z][a-z0-9]*):(?P[A-Za-z0-9_][A-Za-z0-9_.-]*)" + r"(?P(/[A-Za-z0-9_.-]+)*)\Z" +) +_SLUG_BASE_MAX = 200 + + +class IdError(ValueError): + pass + + +def parse(test_id: str) -> Tuple[str, str, Optional[str]]: + m = _ID_RE.match(test_id) + if not m: + raise IdError("not a canonical test id: %r" % (test_id,)) + case = m.group("case") + return m.group("suite"), m.group("group"), case[1:] if case else None + + +def is_valid(test_id: str) -> bool: + return _ID_RE.match(test_id) is not None + + +def suite_of(text: str) -> str: + head, sep, _ = text.partition(":") + return head if sep else "" + + +def group_of(test_id: str) -> str: + return parse(test_id)[1] + + +def matches(pattern: str, test_id: str) -> bool: + """Match wildcards across the full canonical id, including slashes.""" + return fnmatch.fnmatchcase(test_id, pattern) + + +def expand(patterns: Iterable[str], ids: Iterable[str]) -> List[str]: + pats = list(patterns) + return [i for i in ids if any(matches(p, i) for p in pats)] + + +def slug(test_id: str) -> str: + """Append a digest to a bounded, sanitized id.""" + digest = hashlib.sha256(test_id.encode()).hexdigest()[:8] + base = re.sub(r"[^A-Za-z0-9_.-]", "_", test_id)[:_SLUG_BASE_MAX] + return base + "-" + digest diff --git a/tests/conformance/jsonc.py b/tests/conformance/jsonc.py new file mode 100644 index 00000000..4bd7ead6 --- /dev/null +++ b/tests/conformance/jsonc.py @@ -0,0 +1,66 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + + +class JsoncError(ValueError): + pass + + +def strip(text: str) -> str: + out = [] + comma = None + i, n = 0, len(text) + while i < n: + c = text[i] + if c == '"': + j = i + 1 + while j < n and text[j] != '"': + j += 2 if text[j] == "\\" else 1 + out.append(text[i : j + 1]) + comma = None + i = j + 1 + elif text.startswith("//", i): + j = text.find("\n", i) + i = n if j < 0 else j + elif text.startswith("/*", i): + j = text.find("*/", i + 2) + if j < 0: + raise JsoncError("unterminated block comment") + # Preserve token separation when a block comment is removed. + out.append(" " + "\n" * text.count("\n", i, j)) + i = j + 2 + else: + if c == ",": + comma = len(out) + elif c in "]}" and comma is not None: + out[comma] = "" + comma = None + elif not c.isspace(): + comma = None + out.append(c) + i += 1 + return "".join(out) + + +def _reject(literal: str) -> Any: + raise JsoncError("%s is not JSON" % literal) + + +def loads(text: str) -> Any: + try: + return json.loads(strip(text), parse_constant=_reject) + except json.JSONDecodeError as e: + raise JsoncError("line %d: %s" % (e.lineno, e.msg)) from None + + +def load(path: Path) -> Any: + try: + return loads(path.read_text()) + except (JsoncError, OSError, UnicodeDecodeError) as e: + raise JsoncError("%s: %s" % (path, e)) from None diff --git a/tests/conformance/judge.py b/tests/conformance/judge.py new file mode 100644 index 00000000..9bedcbdc --- /dev/null +++ b/tests/conformance/judge.py @@ -0,0 +1,50 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from typing import Tuple + +from conformance.expectations import Resolution +from conformance.model import Status, Verdict + +MAX_ATTEMPTS = 3 + +_SATISFIES = { + "expect_pass": (Status.PASS, Status.WARN), + "expect_failure": (Status.FAIL, Status.BROK), + "expect_conf": (Status.CONF,), +} + + +def decide(test_id: str, status: Status, resolution: Resolution) -> Tuple[Verdict, str]: + if status is Status.ERROR: + return Verdict.ERROR, "%s: HARNESS ERROR, the case did not run" % test_id + if status is Status.SKIP: + return Verdict.FILTERED, "" + if status in (Status.TIMEOUT, Status.CRASH, Status.INCONSISTENT): + return ( + Verdict.UNEXPECTED_FAILURE, + "%s: %s, which no expectation can satisfy" % (test_id, status.value), + ) + if status in _SATISFIES[resolution.type]: + return Verdict.AS_EXPECTED, "" + if status in _SATISFIES["expect_pass"]: + return ( + Verdict.UNEXPECTED_PASS, + "%s: %s but %s expects %s (matcher %r); narrow or delete that " + "matcher in this same change" % ( + test_id, status.value, resolution.source, resolution.type, + resolution.matcher), + ) + return ( + Verdict.UNEXPECTED_FAILURE, + "%s: %s but %s expects %s (matcher %r); fix the regression or record " + "the divergence in the backend leaf" % ( + test_id, status.value, resolution.source, resolution.type, + resolution.matcher), + ) + + +def may_retry(resolution: Resolution, attempts: int) -> bool: + return resolution.quarantined and attempts < MAX_ATTEMPTS diff --git a/tests/conformance/model.py b/tests/conformance/model.py new file mode 100644 index 00000000..54b2b9df --- /dev/null +++ b/tests/conformance/model.py @@ -0,0 +1,117 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import dataclasses +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Dict, List, Optional + + +class Status(str, Enum): + PASS = "PASS" + FAIL = "FAIL" + SKIP = "SKIP" + CONF = "CONF" + WARN = "WARN" + BROK = "BROK" + TIMEOUT = "TIMEOUT" + CRASH = "CRASH" + INCONSISTENT = "INCONSISTENT" + ERROR = "ERROR" + + +class Verdict(str, Enum): + AS_EXPECTED = "as_expected" + UNEXPECTED_FAILURE = "unexpected_failure" + UNEXPECTED_PASS = "unexpected_pass" + FLAKED = "flaked" + FILTERED = "filtered" + ERROR = "error" + + @property + def is_red(self) -> bool: + return self in ( + Verdict.UNEXPECTED_FAILURE, + Verdict.UNEXPECTED_PASS, + Verdict.ERROR, + ) + + +EXECUTIONS = ("normal", "timeout", "signal", "transport") + + +def _plain(value: Any) -> Any: + if isinstance(value, Enum): + return value.value + if dataclasses.is_dataclass(value): + return {f.name: _plain(getattr(value, f.name)) + for f in dataclasses.fields(value) if f.compare} + if isinstance(value, list): + return [_plain(v) for v in value] + if isinstance(value, dict): + return dict(value) + return value + + +class Doc: + """JSON mapping for the dataclasses below; compare=False stays out.""" + + def to_dict(self) -> Dict[str, Any]: + return _plain(self) + + @classmethod + def from_dict(cls, doc: Dict[str, Any]) -> Any: + kwargs = {f.name: _CONVERT.get(f.name, lambda v: v)(doc[f.name]) + for f in dataclasses.fields(cls) if f.compare and f.name in doc} + return cls(**kwargs) + + +@dataclass +class Invocation(Doc): + execution: str + wall_us: int + exit_code: Optional[int] = None + signal: Optional[int] = None + stdout: str = "" + stderr: str = "" + pid: Optional[int] = dataclasses.field(default=None, compare=False) + + def __post_init__(self) -> None: + if self.execution not in EXECUTIONS: + raise ValueError("unknown execution %r" % (self.execution,)) + if self.wall_us < 0: + raise ValueError("negative wall_us") + carries = {"normal": "exit_code", "signal": "signal"}.get(self.execution) + for name in ("exit_code", "signal"): + if (getattr(self, name) is not None) != (name == carries): + raise ValueError("%s execution %s carry %s" % ( + self.execution, "must" if name == carries else "cannot", name)) + + +@dataclass +class Attempt(Doc): + status: Status + invocation: Invocation + detail: str = "" + + +@dataclass +class CaseResult(Doc): + id: str + suite: str + backend: str + status: Status + verdict: Verdict + expectation: Dict[str, Any] = field(default_factory=dict) + attempts: List[Attempt] = field(default_factory=list) + detail: str = "" + + +_CONVERT = { + "status": Status, + "verdict": Verdict, + "invocation": Invocation.from_dict, + "attempts": lambda docs: [Attempt.from_dict(a) for a in docs], +} diff --git a/tests/conformance/payload.py b/tests/conformance/payload.py new file mode 100644 index 00000000..010e69e3 --- /dev/null +++ b/tests/conformance/payload.py @@ -0,0 +1,233 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import copy +import hashlib +import json +import os +import re +import tempfile +from pathlib import Path +from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence + +from conformance import EXIT_DRIFT, EXIT_OK, EXIT_USAGE + +MANIFEST = "manifest.json" +_HEX = {"hex40": re.compile(r"^[0-9a-f]{40}$"), "hex64": re.compile(r"^[0-9a-f]{64}$")} + + +class PayloadError(Exception): + def __init__(self, kind: str, message: str): + super().__init__(message) + self.kind = kind + + +class PinError(ValueError): + pass + + +class UpdateError(RuntimeError): + pass + + +def sha256_file(path: Path) -> str: + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def fingerprint(pin_section: Dict[str, Any], files: Sequence[Path], flavor: str = "") -> str: + """Hash by basename and in the given order, so inputs need distinct names.""" + h = hashlib.sha256() + h.update(json.dumps(pin_section, sort_keys=True).encode()) + for path in files: + h.update(path.name.encode() + b"\0") + h.update(sha256_file(path).encode() + b"\0") + h.update(flavor.encode()) + return h.hexdigest() + + +def _walk(root: Path) -> Dict[str, Dict[str, Any]]: + out: Dict[str, Dict[str, Any]] = {} + for dirpath, dirnames, filenames in os.walk(root, followlinks=False): + dirnames.sort() + for name in list(dirnames): + path = Path(dirpath) / name + if path.is_symlink(): + out[path.relative_to(root).as_posix()] = {"link": os.readlink(path)} + dirnames.remove(name) + for name in sorted(filenames): + path = Path(dirpath) / name + rel = path.relative_to(root).as_posix() + if rel == MANIFEST: + continue + if path.is_symlink(): + out[rel] = {"link": os.readlink(path)} + else: + st = path.stat() + out[rel] = {"sha256": sha256_file(path), "size": st.st_size, + "mode": "%o" % (st.st_mode & 0o777)} + return out + + +def write_manifest(root: Path, fp: str, extra: Optional[Dict[str, Any]] = None, + volatile: Iterable[str] = ()) -> Dict[str, Any]: + """Permit new files below volatile prefixes during verification.""" + doc = {"schema_version": 1, "fingerprint": fp, "files": _walk(root), "extra": extra or {}, + "volatile": sorted(volatile)} + atomic_write(root / MANIFEST, json.dumps(doc, indent=1, sort_keys=True) + "\n") + return doc + + +def read_manifest(root: Path) -> Dict[str, Any]: + path = root / MANIFEST + if not path.is_file(): + raise PayloadError("missing", "no %s under %s" % (MANIFEST, root)) + try: + doc = json.loads(path.read_text()) + except ValueError as e: + raise PayloadError("corrupt", "%s: %s" % (path, e)) from None + volatile = doc.get("volatile", []) if isinstance(doc, dict) else [] + if (not isinstance(doc, dict) or doc.get("schema_version") != 1 + or not isinstance(doc.get("files"), dict) + or not isinstance(volatile, list) or not all(isinstance(v, str) for v in volatile)): + raise PayloadError("corrupt", "%s: unexpected shape" % path) + return doc + + +def verify(root: Path, expected_fp: Optional[str] = None) -> Dict[str, Any]: + doc = read_manifest(root) + if expected_fp is not None and doc.get("fingerprint") != expected_fp: + raise PayloadError( + "stale", + "%s was built for fingerprint %s, the tree wants %s" + % (root, str(doc.get("fingerprint"))[:12], expected_fp[:12]), + ) + actual = _walk(root) + want = doc["files"] + # A trailing slash, so a volatile "tmp" does not also absorb "tmplog/". + volatile = tuple(v.rstrip("/") + "/" for v in doc.get("volatile", [])) + missing = sorted(set(want) - set(actual)) + extra = sorted(k for k in set(actual) - set(want) if not k.startswith(volatile)) + changed = sorted(k for k in set(want) & set(actual) if want[k] != actual[k]) + if missing or extra or changed: + parts = [] + for label, items in (("missing", missing), ("extra", extra), ("changed", changed)): + if items: + parts.append("%s: %s" % (label, ", ".join(items[:5]) + (" ..." if len(items) > 5 else ""))) + raise PayloadError("corrupt", "%s does not match its manifest (%s)" % (root, "; ".join(parts))) + return doc + + +def status(root: Path, expected_fp: str) -> str: + try: + doc = read_manifest(root) + except PayloadError as e: + return e.kind + return "ok" if doc.get("fingerprint") == expected_fp else "stale" + + +def absent_message(suite: str, root: Path, state: str, build_hint: str) -> str: + return ("%s payload %s (%s); run: %s, see docs/conformance.md" + % (suite, state, root, build_hint)) + + +def atomic_write(path: Path, text: str) -> None: + fd, tmp = tempfile.mkstemp(dir=str(path.parent), prefix=path.name + ".") + try: + os.fchmod(fd, 0o644) # generated files should not retain mkstemp's 0600 + with os.fdopen(fd, "w") as f: + f.write(text) + os.replace(tmp, path) + except BaseException: + os.unlink(tmp) + raise + + +def check_pins(doc: Any, schema: Dict[str, Dict[str, str]]) -> Dict[str, Any]: + if not isinstance(doc, dict) or doc.get("schema_version") != 1: + raise PinError("pins: schema_version must be 1") + for section, fields in schema.items(): + body = doc.get(section) + if not isinstance(body, dict): + raise PinError("pins: missing section %r" % section) + for name, kind in fields.items(): + value = body.get(name) + where = "pins: %s.%s" % (section, name) + if kind not in ("int", "url", "str") and kind not in _HEX: + raise PinError("%s: unknown kind %r in schema" % (where, kind)) + if kind == "int": + if not isinstance(value, int) or isinstance(value, bool): + raise PinError("%s must be an integer" % where) + elif not isinstance(value, str) or not value: + raise PinError("%s must be a non-empty string" % where) + elif kind in _HEX and not _HEX[kind].match(value): + raise PinError("%s is not a %s digest" % (where, kind)) + elif kind == "url" and not value.startswith("https://"): + raise PinError("%s must be an https URL" % where) + return doc + + +def load_pins(path: Path, schema: Dict[str, Dict[str, str]]) -> Dict[str, Any]: + try: + doc = json.loads(path.read_text()) + except (OSError, ValueError) as e: + raise PinError("%s: %s" % (path, e)) from None + return check_pins(doc, schema) + + +def write_pins(path: Path, doc: Dict[str, Any], schema: Dict[str, Dict[str, str]]) -> None: + check_pins(doc, schema) + atomic_write(path, json.dumps(doc, indent=2, sort_keys=True) + "\n") + + +def diff_pins(old: Dict[str, Any], new: Dict[str, Any]) -> List[str]: + out = [] + for section in sorted(set(old) | set(new)): + a, b = old.get(section), new.get(section) + if not isinstance(a, dict) or not isinstance(b, dict): + if a != b: + out.append("%s: %r -> %r" % (section, a, b)) + continue + for key in sorted(set(a) | set(b)): + if a.get(key) != b.get(key): + out.append("%s.%s: %r -> %r" % (section, key, a.get(key), b.get(key))) + return out + + +def refresh(provider: Any, ref: Optional[str] = None, check: bool = False, + out: Callable[[str], None] = print, + fail: Callable[[str], None] = print) -> int: + """Refresh pins through the provider schema and latest_pin hook.""" + try: + current = load_pins(provider.pins_path, provider.pins_schema) + fresh = provider.latest_pin(copy.deepcopy(current), ref) + check_pins(fresh, provider.pins_schema) + except (UpdateError, OSError, KeyError, IndexError, ValueError) as e: + # urllib failures are OSError; a malformed upstream response raises + # Key/Index/ValueError out of latest_pin. + fail(str(e)) + return EXIT_USAGE + changes = diff_pins(current, fresh) + if not changes: + out("%s: pins are current" % provider.pins_path) + return EXIT_OK + for line in changes: + out(" " + line) + if check: + out("%s: upstream has moved; run: scripts/conformance pins update %s" + % (provider.pins_path, provider.name)) + return EXIT_DRIFT + try: + write_pins(provider.pins_path, fresh, provider.pins_schema) + except OSError as e: + fail(str(e)) + return EXIT_USAGE + out("%s: rewritten" % provider.pins_path) + for step in provider.update_next_steps(): + out(" next: " + step) + return EXIT_OK diff --git a/tests/conformance/providers/__init__.py b/tests/conformance/providers/__init__.py new file mode 100644 index 00000000..230351d5 --- /dev/null +++ b/tests/conformance/providers/__init__.py @@ -0,0 +1,29 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import importlib +from pathlib import Path +from typing import Dict, Type, Union + +from conformance.providers.base import Provider, ProviderError + +REGISTRY: Dict[str, Union[str, Type[Provider]]] = {} + + +def make(name: str, repo_root: Path) -> Provider: + if name not in REGISTRY: + raise ProviderError( + "unknown suite %r; registered: %s" + % (name, ", ".join(sorted(REGISTRY)) or "none") + ) + target = REGISTRY[name] + if isinstance(target, str): + module, _, cls = target.partition(":") + target = getattr(importlib.import_module(module), cls) + if target.name != name: + # A blank name would collapse suite_dir and payload_root onto the + # shared parents. + raise ProviderError("provider for %r declares name %r" % (name, target.name)) + return target(repo_root) diff --git a/tests/conformance/providers/base.py b/tests/conformance/providers/base.py new file mode 100644 index 00000000..879ed24e --- /dev/null +++ b/tests/conformance/providers/base.py @@ -0,0 +1,88 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional + +from conformance.backends.base import Backend +from conformance.model import Attempt +from conformance.selection import Entry, Selection + + +class ProviderError(RuntimeError): + pass + + +@dataclass +class Case: + id: str + group: str + scope: str + timeout_s: int + meta: Dict[str, Any] = field(default_factory=dict) + + +class Provider: + name = "" + default_timeout_s = 120 + pins_schema: Dict[str, Dict[str, str]] = {} + + def __init__(self, repo_root: Path): + self.repo_root = repo_root + self.suite_dir = repo_root / "tests" / "conformance" / self.name + + @property + def pins_path(self) -> Path: + return self.suite_dir / "pins.json" + + @property + def selection(self) -> Selection: + raise NotImplementedError + + @property + def expectations_dir(self) -> Path: + return self.suite_dir / "expectations" + + def backend_options(self, backend: str) -> Dict[str, Any]: + return {} + + def prerequisites(self, backend: str) -> Optional[str]: + """Return why the suite cannot run, or None when it can.""" + return None + + def enumerate(self, backend: Backend, entries: List[Entry]) -> List[Case]: + raise NotImplementedError + + def batch_key(self, case: Case) -> str: + return case.group + + def run_batch(self, backend: Backend, cases: List[Case], scratch: Path) -> Dict[str, Attempt]: + """Run cases that share a batch key; ids left out are rerun alone.""" + raise NotImplementedError + + def run_single(self, backend: Backend, case: Case, scratch: Path) -> Attempt: + raise NotImplementedError + + def payload_root(self) -> Path: + return self.repo_root / "externals" / "payloads" / self.name + + def fingerprint(self) -> str: + raise NotImplementedError + + def build_payload(self, force: bool = False) -> None: + raise NotImplementedError + + def build_hint(self) -> str: + return "make %s-payload" % self.name + + def latest_pin(self, doc: Dict[str, Any], ref: Optional[str]) -> Dict[str, Any]: + raise NotImplementedError + + def update_next_steps(self) -> List[str]: + return [self.build_hint()] + + def regen_selection(self, check: bool = False) -> List[str]: + return [] diff --git a/tests/conformance/report.py b/tests/conformance/report.py new file mode 100644 index 00000000..634e8a45 --- /dev/null +++ b/tests/conformance/report.py @@ -0,0 +1,116 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Dict, Iterable, List, Tuple + +from conformance import payload +from conformance.model import CaseResult, Verdict + +RESULTS = "results.json" + + +class ReportError(ValueError): + pass + + +def red_line(case: CaseResult) -> str: + return case.detail or "%s: %s" % (case.id, case.status.value) + + +def gate(cases: Iterable[CaseResult]) -> str: + cases = list(cases) + return "red" if not cases or any(c.verdict.is_red for c in cases) else "green" + + +def counts(cases: Iterable[CaseResult]) -> Dict[str, int]: + out = {v.value: 0 for v in Verdict} + for c in cases: + out[c.verdict.value] += 1 + return out + + +def write(results_dir: Path, meta: Dict[str, Any], cases: List[CaseResult]) -> Dict[str, Any]: + results_dir.mkdir(parents=True, exist_ok=True) + doc = document(meta, cases) + payload.atomic_write(results_dir / RESULTS, json.dumps(doc, indent=1, sort_keys=True) + "\n") + (results_dir / "summary.txt").write_text("\n".join(summary_lines(meta, cases, results_dir)) + "\n") + return doc + + +def document(meta: Dict[str, Any], cases: List[CaseResult]) -> Dict[str, Any]: + return {"schema_version": 1, "kind": "run", "run": dict(meta), + "gate": gate(cases), "counts": counts(cases), + "cases": [c.to_dict() for c in cases]} + + +def load(results_dir: Path) -> Tuple[Dict[str, Any], List[CaseResult]]: + path = results_dir / RESULTS + if not path.is_file(): + raise ReportError("no %s under %s" % (RESULTS, results_dir)) + try: + doc = json.loads(path.read_text()) + except (OSError, ValueError) as e: + raise ReportError("%s: %s" % (path, e)) from None + if (not isinstance(doc, dict) or doc.get("kind") != "run" + or not isinstance(doc.get("cases"), list) + or not isinstance(doc.get("run"), dict)): + raise ReportError("%s: unexpected shape" % path) + if doc.get("schema_version") != 1: + raise ReportError("%s: unknown schema" % path) + try: + cases = [CaseResult.from_dict(c) for c in doc["cases"]] + except (TypeError, ValueError, KeyError) as e: + raise ReportError("%s: malformed case record: %s" % (path, e)) from None + if doc.get("gate") != gate(cases) or doc.get("counts") != counts(cases): + raise ReportError("%s: stored gate or counts contradict the case records" % path) + return doc["run"], cases + + +def _duration(seconds: float) -> str: + seconds = int(seconds) + if seconds >= 3600: + return "%dh%02dm" % (seconds // 3600, seconds % 3600 // 60) + return "%dm%02ds" % (seconds // 60, seconds % 60) + + +def summary_lines(meta: Dict[str, Any], cases: List[CaseResult], results_dir: Path) -> List[str]: + n = counts(cases) + head = "conformance %s/%s %s: %d cases in %s" % ( + meta.get("suite", "?"), meta.get("backend", "?"), meta.get("scope", "?"), + len(cases), _duration(meta.get("elapsed_s", 0))) + lines = [head] + if meta.get("bootstrap"): + lines.append(" bootstrap: expectations not applied") + lines.append(" as_expected %d flaked %d filtered %d" % ( + n["as_expected"], n["flaked"], n["filtered"])) + lines.append(" unexpected_failure %d unexpected_pass %d error %d" % ( + n["unexpected_failure"], n["unexpected_pass"], n["error"])) + for c in cases: + if c.verdict.is_red: + lines.append(" RED " + red_line(c)) + if not cases: + lines.append(" RED no cases ran") + lines.append("RESULT: %s (results: %s)" % (gate(cases).upper(), results_dir)) + return lines + + +def markdown(root: Path) -> str: + rows = ["| lane | scope | gate | as_expected | flaked | filtered | red |", "|---|---|---|---|---|---|---|"] + found = False + for path in sorted(root.rglob(RESULTS)): + found = True + try: + meta, cases = load(path.parent) + except ValueError as e: + rows.append("| %s | | error | | | | %s |" % (path.parent, e)) + continue + n = counts(cases) + rows.append("| %s/%s | %s | %s | %d | %d | %d | %d |" % ( + meta.get("suite"), meta.get("backend"), meta.get("scope"), gate(cases), + n["as_expected"], n["flaked"], n["filtered"], + n["unexpected_failure"] + n["unexpected_pass"] + n["error"])) + return "\n".join(rows) + "\n" if found else "no conformance results under %s\n" % root diff --git a/tests/conformance/runner.py b/tests/conformance/runner.py new file mode 100644 index 00000000..e525fd93 --- /dev/null +++ b/tests/conformance/runner.py @@ -0,0 +1,114 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import concurrent.futures +from pathlib import Path +from typing import Callable, Dict, List, Optional + +from conformance import ids, judge +from conformance.backends.base import Backend +from conformance.expectations import Expectations, Resolution +from conformance.model import Attempt, CaseResult, Invocation, Status, Verdict +from conformance.providers.base import Case, Provider + +Log = Callable[[str], None] + + +def _relativize(inv: Invocation, root: Path) -> None: + for name in ("stdout", "stderr"): + value = getattr(inv, name) + if value: + try: + setattr(inv, name, str(Path(value).relative_to(root))) + except ValueError: + pass + + +def _finish(case: Case, attempts: List[Attempt], resolution: Resolution, + bootstrap: bool, results_dir: Path, backend: str) -> CaseResult: + last = attempts[-1] + if bootstrap: + verdict = Verdict.ERROR if last.status is Status.ERROR else Verdict.AS_EXPECTED + message = last.detail if verdict is Verdict.ERROR else "" + else: + verdict, message = judge.decide(case.id, last.status, resolution) + if verdict is Verdict.ERROR and last.detail: + message = "%s: %s" % (message, last.detail) + if (resolution.quarantined and verdict.is_red + and verdict is not Verdict.ERROR): + verdict, message = Verdict.FLAKED, "" + elif verdict is Verdict.AS_EXPECTED and len(attempts) > 1: + verdict = Verdict.FLAKED + for a in attempts: + _relativize(a.invocation, results_dir) + return CaseResult( + id=case.id, suite=ids.suite_of(case.id), backend=backend, status=last.status, + verdict=verdict, expectation=resolution.to_dict(), attempts=attempts, detail=message, + ) + + +def run_lane(provider: Provider, backend: Backend, cases: List[Case], + expectations: Expectations, results_dir: Path, jobs: int = 1, + retry: bool = True, bootstrap: bool = False, + log: Optional[Log] = None) -> List[CaseResult]: + log = log or (lambda _: None) + results: Dict[str, CaseResult] = {} + launch: List[Case] = [] + resolutions: Dict[str, Resolution] = {} + for case in cases: + resolution = expectations.resolve(case.id) + resolutions[case.id] = resolution + if resolution.type == "skip" and not bootstrap: + results[case.id] = CaseResult( + id=case.id, suite=ids.suite_of(case.id), backend=backend.name, + status=Status.SKIP, verdict=Verdict.FILTERED, expectation=resolution.to_dict(), + detail="skip: " + resolution.reason) + else: + launch.append(case) + + batches: Dict[str, List[Case]] = {} + for case in launch: + key = case.id if resolutions[case.id].quarantined else provider.batch_key(case) + batches.setdefault(key, []).append(case) + + def run_batch(key: str) -> List[CaseResult]: + members = batches[key] + quarantined = resolutions[members[0].id].quarantined + # A quarantined key holds one case; with no batch result it takes + # the single-case path below. + first = ({} if quarantined + else provider.run_batch(backend, members, + results_dir / "cases" / ("batch-" + ids.slug(key)))) + out = [] + for case in members: + case_dir = results_dir / "cases" / ids.slug(case.id) + if case.id in first: + attempts = [first[case.id]] + else: + if not quarantined: + log("%s: unresolved by the batch, rerunning alone" % case.id) + attempts = [provider.run_single(backend, case, case_dir / "attempt-1")] + resolution = resolutions[case.id] + while (retry and not bootstrap + and judge.decide(case.id, attempts[-1].status, resolution)[0].is_red + and attempts[-1].status is not Status.ERROR + and judge.may_retry(resolution, len(attempts))): + n = len(attempts) + 1 + log("%s: %s on attempt %d, quarantined, retrying" % (case.id, attempts[-1].status.value, n - 1)) + attempts.append(provider.run_single(backend, case, case_dir / ("attempt-%d" % n))) + out.append(_finish(case, attempts, resolution, bootstrap, results_dir, backend.name)) + return out + + workers = min(jobs, backend.max_jobs or jobs) + keys = list(batches) + if workers > 1 and len(keys) > 1: + with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool: + batched = list(pool.map(run_batch, keys)) + else: + batched = [run_batch(k) for k in keys] + for group in batched: + for r in group: + results[r.id] = r + return [results[c.id] for c in cases] diff --git a/tests/conformance/seed.py b/tests/conformance/seed.py new file mode 100644 index 00000000..36516279 --- /dev/null +++ b/tests/conformance/seed.py @@ -0,0 +1,111 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Dict, List + +from conformance import ids, jsonc, payload +from conformance.expectations import SEEDED_PREFIX +from conformance.model import CaseResult, Status, Verdict + +class SeedError(ValueError): + pass + + +_ACTION_FOR = { + Status.FAIL: "expect_failure", + Status.BROK: "expect_failure", + Status.CONF: "expect_conf", + Status.TIMEOUT: "skip", + Status.CRASH: "skip", + Status.INCONSISTENT: "skip", +} +_ORDER = ("expect_pass", "expect_failure", "expect_conf", "skip") + + +def default_reason(results_dir: Path, date: str) -> str: + return "%s%s on %s; untriaged" % (SEEDED_PREFIX, results_dir, date) + + +def propose(cases: List[CaseResult], reason: str, bootstrap: bool, + whole_groups: bool = True) -> List[Dict[str, Any]]: + """Collapse complete groups when whole_groups is set.""" + wanted: Dict[str, str] = {} + for c in cases: + if c.status is Status.ERROR: + raise SeedError("%s is a harness ERROR; seeding refuses it" % c.id) + if bootstrap: + # What an earlier leaf recorded is not evidence. + action = _ACTION_FOR.get(c.status) + if action: + wanted[c.id] = action + elif c.verdict is Verdict.UNEXPECTED_PASS: + wanted[c.id] = "expect_pass" + elif c.verdict is Verdict.UNEXPECTED_FAILURE: + wanted[c.id] = _ACTION_FOR.get(c.status, "expect_failure") + by_group: Dict[tuple, List[str]] = {} + for c in cases: + by_group.setdefault((ids.suite_of(c.id), ids.group_of(c.id)), []).append(c.id) + out: Dict[str, List[str]] = {} + for (suite, group), members in sorted(by_group.items()): + actions = {wanted.get(m) for m in members} + if whole_groups and len(actions) == 1 and None not in actions and len(members) > 1: + out.setdefault(actions.pop(), []).append("%s:%s/*" % (suite, group)) + continue + for m in members: + if m in wanted: + out.setdefault(wanted[m], []).append(m) + return [{"type": kind, "reason": reason, "matchers": sorted(out[kind])} + for kind in _ORDER if kind in out] + + +def format_actions(actions: List[Dict[str, Any]], header: str = "") -> str: + lines = [header.rstrip("\n")] if header else [] + lines += ["{", ' "actions": ['] + for a in actions: + if "include" in a: + lines.append(' { "include": %s },' % json.dumps(a["include"])) + continue + lines.append(' { "type": %s,' % json.dumps(a["type"])) + for key in ("reason", "since", "tracking"): + if a.get(key): + lines.append(' "%s": %s,' % (key, json.dumps(a[key]))) + matchers = a["matchers"] + if len(matchers) == 1: + lines.append(' "matchers": [%s] },' % json.dumps(matchers[0])) + else: + lines.append(' "matchers": [') + lines.extend(" %s," % json.dumps(m) for m in matchers) + lines.append(" ] },") + lines += [" ],", "}", ""] + return "\n".join(lines) + + +def append(leaf: Path, actions: List[Dict[str, Any]]) -> None: + """Append actions new to the leaf, preserving the leading comments.""" + text = leaf.read_text() if leaf.exists() else "" + header_lines = [] + for line in text.splitlines(): + if line.startswith("//") or not line.strip(): + header_lines.append(line) + else: + break + if text.strip(): + existing = jsonc.loads(text)["actions"] + else: + # Prefer the shared suite default when present. + base = leaf.parent / (leaf.stem.split("_", 1)[0] + ".jsonc") + existing = ([{"include": base.name}] if base.exists() + else [{"type": "expect_pass", "matchers": ["*"]}]) + present = {(a["type"], m) for a in existing if "type" in a for m in a["matchers"]} + fresh = [] + for a in actions: + matchers = [m for m in a["matchers"] if (a["type"], m) not in present] + if matchers: + fresh.append(dict(a, matchers=matchers)) + if not fresh: + return + payload.atomic_write(leaf, format_actions(existing + fresh, "\n".join(header_lines))) diff --git a/tests/conformance/selection.py b/tests/conformance/selection.py new file mode 100644 index 00000000..eba62847 --- /dev/null +++ b/tests/conformance/selection.py @@ -0,0 +1,130 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import difflib +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Tuple + +from conformance import ids, jsonc + +SCOPES = ("pr", "full") + + +class SelectionError(ValueError): + pass + + +@dataclass(frozen=True) +class Entry: + group: str + scope: str + timeout_s: Optional[int] = None + only: Tuple[str, ...] = () + + +@dataclass +class Selection: + enabled: List[Entry] + declined: List[Tuple[str, Tuple[str, ...]]] + extra: Dict[str, Any] = field(default_factory=dict) + source: str = "" + + def groups(self, scope: str) -> List[Entry]: + if scope not in SCOPES: + raise SelectionError("unknown scope %r" % (scope,)) + return [e for e in self.enabled if scope == "full" or e.scope == "pr"] + + def entry(self, group: str) -> Optional[Entry]: + return next((e for e in self.enabled if e.group == group), None) + + def lint(self) -> List[str]: + problems = [] + seen: Dict[str, str] = {} + for e in self.enabled: + if e.group in seen: + problems.append("%s: %s is enabled twice" % (self.source, e.group)) + seen[e.group] = "enabled" + for reason, groups in self.declined: + for g in groups: + if seen.get(g) == "enabled": + problems.append("%s: %s is both enabled and declined" % (self.source, g)) + elif g in seen: + problems.append("%s: %s is declined twice" % (self.source, g)) + seen[g] = "declined" + return problems + + +def _entry(doc: Any, where: str) -> Entry: + if not isinstance(doc, dict) or not isinstance(doc.get("group"), str): + raise SelectionError("%s: enabled entry needs a group" % where) + unknown = set(doc) - {"group", "scope", "timeout_s", "only"} + if unknown: + raise SelectionError("%s: unknown keys %s" % (where, sorted(unknown))) + scope = doc.get("scope") + if scope not in SCOPES: + raise SelectionError("%s: %s has scope %r, want pr or full" % (where, doc["group"], scope)) + timeout = doc.get("timeout_s") + if timeout is not None and (isinstance(timeout, bool) or not isinstance(timeout, int) + or timeout <= 0): + raise SelectionError("%s: timeout_s must be a positive integer" % where) + only = doc.get("only", []) + if not isinstance(only, list) or any(not isinstance(o, str) for o in only): + raise SelectionError("%s: only must be a list of case globs" % where) + return Entry(doc["group"], scope, timeout, tuple(only)) + + +def parse(doc: Any, source: str) -> Selection: + if not isinstance(doc, dict) or doc.get("schema_version") != 1: + raise SelectionError("%s: schema_version must be 1" % source) + enabled = doc.get("enabled") + if not isinstance(enabled, list): + raise SelectionError("%s: enabled must be a list" % source) + entries = [_entry(e, "%s:enabled[%d]" % (source, i)) for i, e in enumerate(enabled)] + declined_doc = doc.get("declined", []) + if not isinstance(declined_doc, list): + raise SelectionError("%s: declined must be a list" % source) + declined = [] + for i, d in enumerate(declined_doc): + where = "%s:declined[%d]" % (source, i) + if (not isinstance(d, dict) or set(d) != {"reason", "groups"} + or not isinstance(d["reason"], str) or not d["reason"].strip() + or not isinstance(d["groups"], list) or not d["groups"] + or any(not isinstance(g, str) for g in d["groups"])): + raise SelectionError("%s: a declined entry is a reason and a group list" % where) + declined.append((d["reason"], tuple(d["groups"]))) + extra = {k: v for k, v in doc.items() if k not in ("schema_version", "enabled", "declined")} + sel = Selection(entries, declined, extra, source) + problems = sel.lint() + if problems: + raise SelectionError("; ".join(problems)) + return sel + + +def load(path: Path) -> Selection: + return parse(jsonc.load(path), path.name) + + +def resolve_ids(patterns: Iterable[str], universe: Iterable[str], + suite: str) -> Tuple[List[str], List[str]]: + """Report unmatched patterns with nearby canonical ids.""" + known = list(universe) + chosen: List[str] = [] + seen = set() + errors: List[str] = [] + for pattern in patterns: + if ids.suite_of(pattern) != suite: + errors.append("%s: not a %s id (want %s:[/])" % (pattern, suite, suite)) + continue + # A bare group id also selects its cases. + pats = [pattern] if "/" in pattern else [pattern, pattern + "/*"] + hits = ids.expand(pats, known) + if not hits: + near = difflib.get_close_matches(pattern, known, n=3, cutoff=0.6) + errors.append("%s: no such test%s" % ( + pattern, ("; near: " + ", ".join(near)) if near else "")) + chosen.extend(h for h in hits if h not in seen) + seen.update(hits) + return chosen, errors diff --git a/tests/conformance/selftest/__init__.py b/tests/conformance/selftest/__init__.py new file mode 100644 index 00000000..13b8e032 --- /dev/null +++ b/tests/conformance/selftest/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 diff --git a/tests/conformance/selftest/fixture.py b/tests/conformance/selftest/fixture.py new file mode 100644 index 00000000..81c8bf90 --- /dev/null +++ b/tests/conformance/selftest/fixture.py @@ -0,0 +1,123 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path +from typing import Dict, List + +from conformance import ids, payload, selection +from conformance.backends import proc +from conformance.backends.base import Backend +from conformance.model import Attempt, Status +from conformance.providers.base import Case, Provider + + +class TempDirTest(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.dir = self.root = Path(tmp.name) + +DATA = { + "schema_version": 1, + "enabled": [ + {"group": "basic", "scope": "pr"}, + {"group": "slow", "scope": "full", "timeout_s": 1}, + ], +} +CASES = { + "basic": { + "pass": "exit 0", "fail": "exit 1", "flaky": "flaky", + "quarantined": "exit 1", + }, + "slow": {"timeout": "sleep 30", "unresolved": "unresolved"}, +} + + +def setup(root: Path) -> None: + expectations = root / "fixture" / "expectations" + expectations.mkdir(parents=True) + (expectations / "fixture.jsonc").write_text( + '{"actions":[{"type":"expect_pass","matchers":["*"]}]}\n' + ) + (expectations / "fixture_elfuse.jsonc").write_text( + '{"actions":[{"include":"fixture.jsonc"},' + '{"type":"expect_failure","reason":"fixture exit",' + '"matchers":["fixture:basic/fail"]}]}\n' + ) + (expectations / "flaky.jsonc").write_text( + '{"actions":[{"type":"quarantine","reason":"fixture retry",' + '"matchers":["fixture:basic/flaky",' + '"fixture:basic/quarantined"]}]}\n' + ) + + +class LocalBackend(Backend): + name = "elfuse" + + def run(self, argv, timeout_s, scratch, env=None, fetch=()): + return proc.run_local(argv, timeout_s, scratch, env=env) + + +class FixtureProvider(Provider): + name = "fixture" + + def __init__(self, repo_root: Path): + super().__init__(repo_root) + self.suite_dir = repo_root / "fixture" + self._selection = selection.parse(DATA, "fixture") + self.runs: Dict[str, int] = {} + + @property + def selection(self): + return self._selection + + def fingerprint(self) -> str: + return "0" * 64 + + def build_payload(self, force: bool = False) -> None: + root = self.payload_root() + root.mkdir(parents=True, exist_ok=True) + (root / "fixture").write_text("fixture\n") + payload.write_manifest(root, self.fingerprint()) + + def enumerate(self, backend: Backend, + entries: List[selection.Entry]) -> List[Case]: + return [ + Case("fixture:%s/%s" % (entry.group, name), entry.group, + entry.scope, entry.timeout_s or 5, {"script": script}) + for entry in entries + for name, script in CASES[entry.group].items() + ] + + def invoke(self, backend: Backend, case: Case, scratch: Path) -> Attempt: + self.runs[case.id] = self.runs.get(case.id, 0) + 1 + script = case.meta["script"] + if script == "flaky": + script = "exit %d" % (1 if self.runs[case.id] == 1 else 0) + inv = backend.run(["sh", "-c", script], case.timeout_s, scratch) + if inv.execution == "timeout": + status = Status.TIMEOUT + elif inv.execution != "normal": + status = Status.ERROR + else: + status = Status.PASS if inv.exit_code == 0 else Status.FAIL + return Attempt(status, inv, inv.execution) + + def run_batch(self, backend: Backend, cases: List[Case], + scratch: Path) -> Dict[str, Attempt]: + return { + case.id: self.invoke(backend, case, scratch / ids.slug(case.id)) + for case in cases + if case.meta["script"] != "unresolved" + } + + def run_single(self, backend: Backend, case: Case, + scratch: Path) -> Attempt: + if case.meta["script"] == "unresolved": + case = Case(case.id, case.group, case.scope, case.timeout_s, + {"script": "exit 0"}) + return self.invoke(backend, case, scratch) diff --git a/tests/conformance/selftest/test_backends.py b/tests/conformance/selftest/test_backends.py new file mode 100644 index 00000000..4cdf06c5 --- /dev/null +++ b/tests/conformance/selftest/test_backends.py @@ -0,0 +1,297 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +import os +import stat +import time +import unittest +import unittest.mock +from pathlib import Path + +from conformance.backends import BackendError, elfuse, proc, qemu, ssh +from conformance.selftest.fixture import TempDirTest + + +def script(path, body): + path.write_text("#!/bin/sh\n" + body) + path.chmod(path.stat().st_mode | stat.S_IXUSR) + return path + + +class ProcTest(TempDirTest): + def test_normal(self): + inv = proc.run_local(["sh", "-c", "echo out; echo err >&2; exit 3"], 10, self.dir) + self.assertEqual((inv.execution, inv.exit_code, inv.signal), ("normal", 3, None)) + self.assertEqual(Path(inv.stdout).read_text(), "out\n") + self.assertEqual(Path(inv.stderr).read_text(), "err\n") + self.assertGreater(inv.wall_us, 0) + + def test_unspawnable(self): + inv = proc.run_local([str(self.dir / "absent")], 10, self.dir) + self.assertEqual(inv.execution, "transport") + self.assertIn("cannot spawn", Path(inv.stderr).read_text()) + + def test_signal(self): + inv = proc.run_local(["sh", "-c", "kill -SEGV $$"], 10, self.dir) + self.assertEqual((inv.execution, inv.exit_code, inv.signal), ("signal", None, 11)) + + def test_timeout_kills_the_group(self): + started = time.monotonic() + inv = proc.run_local(["sh", "-c", "sleep 30 & echo $! > pid; wait"], 1, self.dir) + self.assertEqual(inv.execution, "timeout") + self.assertLess(time.monotonic() - started, 5) + self.assertIsNone(inv.exit_code) + child = int((self.dir / "pid").read_text()) + for _ in range(50): # SIGKILL reaches the orphan asynchronously + try: + os.kill(child, 0) + except ProcessLookupError: + break + time.sleep(0.1) + else: + self.fail("background child %d survived the group kill" % child) + + def test_timeout_that_cannot_reap_is_transport(self): + kill_and_reap = proc._kill_and_reap + + def report_unreaped(child): + kill_and_reap(child) + return None + + with unittest.mock.patch.object(proc, "_kill_and_reap", report_unreaped): + inv = proc.run_local(["sleep", "30"], 0.01, self.dir) + self.assertEqual(inv.execution, "transport") + self.assertIn("not reaped after SIGKILL", Path(inv.stderr).read_text()) + + +class SshTest(TempDirTest): + def setUp(self): + super().setUp() + self.args = self.dir / "args" + self.stub = script( + self.dir / "ssh", + 'printf "%%s\\n" "$@" >> "%s"\n' % self.args + + 'case "$*" in *"cat /tmp/conf.abc/result.xml") printf ""; exit 0 ;; esac\n' + + 'case "$*" in *"cat /tmp/conf.abc/missing.bin") exit 1 ;; esac\n' + + 'case "$*" in *"rm -rf /tmp/conf"*) exit 0 ;; esac\n' + # ssh forwards the remote script's status, 0 once the sentinel + # printf ran; the guest rc travels inside the sentinel. + + 'case "$STUB" in\n' + ' ok) printf "hello\\n\\n__CONF_RC=3 __CONF_DIR=/tmp/conf.abc\\n"; exit 0 ;;\n' + ' sig) printf "__CONF_RC=139 __CONF_DIR=/tmp/x\\n"; exit 0 ;;\n' + ' bigrc) printf "__CONF_RC=255 __CONF_DIR=/tmp/x\\n"; exit 0 ;;\n' + ' spoof) printf "__CONF_RC=0 __CONF_DIR=/\\n"; exit 255 ;;\n' + " lost) exit 255 ;;\n" + "esac\n", + ) + self.session = ssh.SshSession(2222, Path("/k"), ssh=str(self.stub)) + + def run_stub(self, mode, **kw): + os.environ["STUB"] = mode + try: + return self.session.run(["true", "a b"], 5, self.dir / mode, **kw) + finally: + del os.environ["STUB"] + + def test_script(self): + text = ssh.SshSession.remote_script(["cmd", "a b"], 7, {"X": "y z"}, None) + self.assertIn("mktemp -d /tmp/conf.XXXXXX", text) + self.assertNotIn("rm -rf", text) + self.assertIn("export LC_ALL=C;", text) + self.assertIn("trap 'cd / && rm -rf \"$d\"' EXIT", ssh.SshSession.remote_script(["x"], 1, {}, None, cleanup=True)) + self.assertIn("/usr/bin/timeout -s KILL 7 cmd 'a b'", text) + self.assertIn("export X='y z';", text) + self.assertIn('export HOME="$PWD"', text) + self.assertIn(ssh.SENTINEL, text) + self.assertIn("cd /opt/ltp", ssh.SshSession.remote_script(["x"], 1, {}, "/opt/ltp")) + + def test_ok(self): + inv = self.run_stub("ok", fetch=["result.xml"]) + self.assertEqual((inv.execution, inv.exit_code), ("normal", 3)) + self.assertEqual(Path(inv.stdout).read_text(), "hello\n") + args = self.args.read_text().splitlines() + self.assertIn("BatchMode=yes", args) + self.assertEqual(args[args.index("-p") + 1], "2222") + self.assertEqual(args[-2], "root@127.0.0.1") + self.assertEqual((self.dir / "ok" / "result.xml").read_text(), "") + self.assertIn("rm -rf /tmp/conf.abc", self.args.read_text()) + + def test_signal_like_status_stays_an_exit_code(self): + inv = self.run_stub("sig") + self.assertEqual( + (inv.execution, inv.exit_code, inv.signal), ("normal", 139, None) + ) + + def test_an_exit_past_192_stays_an_exit_code(self): + inv = self.run_stub("bigrc") + self.assertEqual((inv.execution, inv.exit_code, inv.signal), ("normal", 255, None)) + + def test_a_sentinel_cut_mid_write_is_a_transport_loss(self): + self.assertIsNone(ssh.SshSession.parse_sentinel("x\n" + ssh.SENTINEL + "1")) + self.assertEqual(ssh.SshSession.parse_sentinel(ssh.SENTINEL + "12" + ssh.DIR_MARK + "/t"), (12, "/t")) + + def test_a_garbled_sentinel_is_a_transport_loss(self): + # A partial sentinel is a transport loss, not a parser error. + for line in ("__CONF_RC=", "__CONF_RC=xx __CONF_DIR=/tmp/a", "__CONF_RC=-"): + self.assertIsNone(ssh.SshSession.parse_sentinel(line), line) + self.assertEqual(ssh.SshSession.parse_sentinel("__CONF_RC=3 __CONF_DIR=/tmp/a b"), + (3, "/tmp/a b")) + + def test_transport(self): + inv = self.run_stub("lost") + self.assertEqual(inv.execution, "transport") + self.assertIsNone(inv.exit_code) + + def test_a_lookalike_sentinel_on_a_lost_connection_is_transport(self): + inv = self.run_stub("spoof") + self.assertEqual(inv.execution, "transport") + self.assertNotIn("rm -rf /", self.args.read_text()) + + def test_a_failed_fetch_is_a_transport_loss(self): + inv = self.run_stub("ok", fetch=["missing.bin"]) + self.assertEqual(inv.execution, "transport") + + +class ElfuseTest(TempDirTest): + def test_argv_and_prerequisites(self): + b = elfuse.ElfuseBackend(self.dir, sysroot=self.dir / "root") + self.assertIn("absent; run: make elfuse", b.prerequisites()) + script(self.dir / "elfuse", "exit 0\n") + (self.dir / "build").mkdir() + os.rename(self.dir / "elfuse", self.dir / "build" / "elfuse") + self.assertIn("sysroot", b.prerequisites()) + (self.dir / "root").mkdir() + self.assertIsNone(b.prerequisites()) + self.assertEqual( + b.argv(["/bin/true", "x"]), + [str(self.dir / "build" / "elfuse"), "--timeout", "0", "--sysroot", + str(self.dir / "root"), "/bin/true", "x"], + ) + + def test_run_scrubs_the_environment(self): + (self.dir / "build").mkdir() + script(self.dir / "build" / "elfuse", 'shift 2; echo "$TZ $HOME"; exec "$@"\n') + b = elfuse.ElfuseBackend(self.dir) + with unittest.mock.patch.dict(os.environ, {"LEAK": "1"}): + inv = b.run(["sh", "-c", 'echo "${LEAK:-clean}"'], 5, self.dir / "s") + self.assertEqual(inv.exit_code, 0) + self.assertEqual(Path(inv.stdout).read_text(), "UTC %s\nclean\n" % (self.dir / "s")) + + def test_orphan_pids(self): + listing = (" 12 1 40 /repo/build/elfuse --fork-child 8\n" + " 13 500 40 /repo/build/elfuse --fork-child 8\n" + " 14 1 40 /repo/build/elfuse --timeout 0 /bin/true\n" + " 15 1 40 /other/build/elfuse --fork-child 8\n" + " 16 1 41 /repo/build/elfuse --fork-child 9\n") + self.assertEqual(elfuse.orphan_pids(listing, "/repo/build/elfuse", 40), [12]) + + def test_serialize_excludes_a_second_session(self): + a, b = elfuse.ElfuseBackend(self.dir), elfuse.ElfuseBackend(self.dir) + a.lock_file = b.lock_file = self.dir / "lock" + with a.serialize(): + with self.assertRaises(BackendError): + with b.serialize(): + pass + with b.serialize(): + pass + + @unittest.skipIf(os.geteuid() == 0, "root ignores directory permissions") + def test_an_unopenable_lock_is_a_backend_error(self): + a = elfuse.ElfuseBackend(self.dir) + a.lock_file = self.dir / "locked" / "lock" + a.lock_file.parent.mkdir() + a.lock_file.parent.chmod(0o500) + try: + with self.assertRaises(BackendError): + with a.serialize(): + pass + finally: + a.lock_file.parent.chmod(0o700) + + +class QemuTest(TempDirTest): + def test_prerequisites_report_the_tree_before_the_host(self): + # Missing fixtures produce the same answer on every host. + b = qemu.QemuBackend(self.dir) + with unittest.mock.patch("shutil.which", return_value=None): + self.assertIn("QEMU fixtures missing", b.prerequisites()) + + def test_guest_path(self): + b = qemu.QemuBackend(self.dir) + (self.dir / "tests").mkdir() + self.assertEqual(b.guest_path(self.dir / "tests" / "x"), "/mnt/host/tests/x") + with self.assertRaises(BackendError): + b.guest_path(Path("/etc/passwd")) + + def test_start_reads_the_state_file(self): + runner = script( + self.dir / "runner.sh", + '[ "$1" = start ] && printf "port=2200\\nkey=/k\\npidfile=/p\\n" > "$3"\n' + 'echo "$QEMU_MEM $QEMU_BOOT_TIMEOUT" > "%s/mem"\n' % self.dir, + ) + b = qemu.QemuBackend(self.dir, mem_mib=4096, runner=runner, state_dir=self.dir) + with unittest.mock.patch.dict(os.environ, {"QEMU_BOOT_TIMEOUT": "300", "QEMU_MEM": "1"}): + b.start() + self.assertEqual((b.session.port, b.session.key), (2200, Path("/k"))) + self.assertEqual((self.dir / "mem").read_text().split(), ["4096", "300"]) + b.stop() + self.assertIsNone(b.session) + + def test_start_reaps_the_vm_a_stale_state_file_names(self): + runner = script( + self.dir / "runner.sh", + 'echo "$1" >> "%s/verbs"\n' % self.dir + + '[ "$1" = start ] && printf "port=2200\\nkey=/k\\n" > "$3"\nexit 0\n', + ) + (self.dir / "qemu.state").write_text("port=2199\nkey=/k\npidfile=/p\n") + b = qemu.QemuBackend(self.dir, runner=runner, state_dir=self.dir) + b.start() + self.assertEqual((self.dir / "verbs").read_text().split(), ["stop", "start"]) + self.assertEqual(b.session.port, 2200) + + def test_start_after_a_bad_state_stops_the_vm(self): + # ":" writes nothing, so there is no VM record left to stop. + for writer, verbs in (('printf "key=/k\\n" > "$3"', ["start", "stop"]), + ('printf "port=abc\\nkey=/k\\n" > "$3"', ["start", "stop"]), + (":", ["start"]), + ('mkdir "$3"', ["start", "stop"])): + runner = script( + self.dir / "runner.sh", + 'echo "$1" >> "%s/verbs"\n' % self.dir + + '[ "$1" = stop ] && rm -rf "$3"\n' + '[ "$1" = start ] && %s\nexit 0\n' % writer, + ) + b = qemu.QemuBackend(self.dir, runner=runner, state_dir=self.dir) + with self.assertRaises(BackendError, msg=writer): + b.start() + self.assertEqual((self.dir / "verbs").read_text().split(), verbs, writer) + (self.dir / "verbs").unlink() + + def test_a_failed_stop_is_an_error(self): + runner = script( + self.dir / "runner.sh", + '[ "$1" = start ] && printf "port=2200\\nkey=/k\\n" > "$3"\n' + '[ "$1" = stop ] && { echo "no such vm"; exit 1; }\nexit 0\n', + ) + b = qemu.QemuBackend(self.dir, runner=runner, state_dir=self.dir) + b.start() + with self.assertRaises(BackendError) as cm: + b.stop() + self.assertIn("no such vm", str(cm.exception)) + + def test_a_relative_scratch_still_names_an_absolute_home(self): + env = elfuse.base.guest_environment("rel/x") + self.assertTrue(Path(env["HOME"]).is_absolute()) + self.assertEqual(env["HOME"], env["TMPDIR"]) + + def test_serialize_excludes_a_second_session_on_the_state_dir(self): + a = qemu.QemuBackend(self.dir, state_dir=self.dir) + b = qemu.QemuBackend(self.dir, state_dir=self.dir) + with a.serialize(): + with self.assertRaises(BackendError): + with b.serialize(): + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/conformance/selftest/test_cli.py b/tests/conformance/selftest/test_cli.py new file mode 100644 index 00000000..9d21bb6c --- /dev/null +++ b/tests/conformance/selftest/test_cli.py @@ -0,0 +1,132 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import json +import os +import unittest +import unittest.mock +from pathlib import Path + +from conformance import cli, providers, report +from conformance.backends.base import BackendError +from conformance.selftest.fixture import FixtureProvider, LocalBackend, TempDirTest, setup + + +class CliTest(TempDirTest): + def setUp(self): + super().setUp() + setup(self.root) + self.out = [] + self.err = [] + self.registry = unittest.mock.patch.dict( + providers.REGISTRY, {"fixture": FixtureProvider}, clear=True + ) + self.backend = unittest.mock.patch.object( + cli.backends, "make", return_value=LocalBackend() + ) + self.registry.start() + self.backend.start() + + def tearDown(self): + self.backend.stop() + self.registry.stop() + + def invoke(self, *args): + self.out.clear() + self.err.clear() + return cli.main(list(args), self.root, self.out.append, self.err.append) + + def test_suite_json_uses_a_versioned_envelope(self): + self.assertEqual(self.invoke("suites", "--format", "json"), 0) + doc = json.loads(self.out[0]) + self.assertEqual(doc, { + "kind": "suite-list", "schema_version": 1, + "suites": ["fixture"], + }) + self.assertEqual(self.err, []) + + def test_list_all_uses_one_backend(self): + self.assertEqual( + self.invoke("list", "fixture", "--scope", "pr", "--backend", "all"), + 0, + ) + self.assertEqual(self.out, [ + "fixture:basic/pass", "fixture:basic/fail", "fixture:basic/flaky", + "fixture:basic/quarantined", + ]) + self.assertEqual(cli.backends.make.call_count, 1) + + def test_run_writes_results_and_report_reads_them(self): + self.assertEqual(self.invoke("payload", "build", "fixture"), 0) + results = self.root / "results" + self.assertEqual(self.invoke( + "run", "fixture", "--scope", "pr", "--results", str(results) + ), 0, self.err) + paths = list(results.rglob(report.RESULTS)) + self.assertEqual(len(paths), 1) + doc = json.loads(paths[0].read_text()) + self.assertEqual((doc["kind"], doc["gate"]), ("run", "green")) + flaky = next(c for c in doc["cases"] if c["id"].endswith("/flaky")) + self.assertEqual(len(flaky["attempts"]), 2) + self.assertEqual(self.invoke( + "report", str(paths[0].parent), "--format", "json" + ), 0) + self.assertEqual(json.loads(self.out[0])["gate"], "green") + + def test_results_survive_a_failing_stop(self): + self.assertEqual(self.invoke("payload", "build", "fixture"), 0) + results = self.root / "results" + with unittest.mock.patch.object(LocalBackend, "stop", + side_effect=BackendError("vm stuck")): + self.assertEqual(self.invoke( + "run", "fixture", "--scope", "pr", "--results", str(results) + ), 1) + self.assertEqual(len(list(results.rglob(report.RESULTS))), 1) + self.assertEqual(self.err[-1], "conformance: backend error: vm stuck") + + def test_case_selector_and_dry_run(self): + self.assertEqual(self.invoke("payload", "build", "fixture"), 0) + self.assertEqual(self.invoke( + "run", "fixture", "--case", "fixture:basic/pa*", "--dry-run" + ), 0) + self.assertEqual(self.out, ["fixture:basic/pass"]) + self.assertEqual(self.invoke( + "run", "fixture", "--case", "fixture:basic/pas", "--dry-run" + ), 2) + self.assertIn("near: fixture:basic/pass", self.err[0]) + + def test_skip_and_require_are_stable(self): + args = argparse.Namespace(require=False) + instance = cli.Cli(self.root, self.out.append, self.err.append) + with unittest.mock.patch.dict(os.environ, {"CONF_REQUIRE": "0"}): + self.assertEqual(instance.skip(args, "missing"), 77) + args.require = True + self.assertEqual(instance.skip(args, "missing"), 2) + args.require = False + os.environ["CONF_REQUIRE"] = "1" + self.assertEqual(instance.skip(args, "missing"), 2) + self.assertEqual(self.out, []) + self.assertEqual(self.err, ["conformance: missing"] * 3) + + def test_expectation_errors_use_stderr(self): + leaf = self.root / "fixture" / "expectations" / "fixture_elfuse.jsonc" + leaf.write_text("{") + self.assertEqual(self.invoke("expectations", "check", "fixture"), 2) + self.assertEqual(self.out, []) + self.assertIn("line 1", self.err[0]) + + def test_a_provider_must_declare_its_registry_name(self): + with unittest.mock.patch.dict(providers.REGISTRY, {"other": FixtureProvider}): + with self.assertRaisesRegex(providers.ProviderError, "declares name"): + providers.make("other", self.root) + + def test_parser_has_no_suite_owned_commands(self): + help_text = cli.build_parser(["fixture"]).format_help() + self.assertIn("{suites,list,run,payload,selection,expectations,pins,report,selftest}", + help_text) + self.assertNotIn("fixture}", help_text) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/conformance/selftest/test_elfcheck.py b/tests/conformance/selftest/test_elfcheck.py new file mode 100644 index 00000000..e8ad7869 --- /dev/null +++ b/tests/conformance/selftest/test_elfcheck.py @@ -0,0 +1,89 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +import struct +import unittest +from pathlib import Path + +from conformance import elfcheck +from conformance.selftest.fixture import TempDirTest + + +def elf(machine=elfcheck.EM_AARCH64, interp=None, needed=(), load=True): + phdrs = [] + body = bytearray() + strtab = b"\0" + b"\0".join(n.encode() for n in needed) + b"\0" + offsets, pos = [], 1 + for n in needed: + offsets.append(pos) + pos += len(n) + 1 + interp_bytes = (interp.encode() + b"\0") if interp else b"" + dyn = b"".join(struct.pack(" %r" % ("a" * 40, "b" * 40), self.lines[0]) + self.assertIn("scripts/conformance pins update tool", self.lines[-1]) + + def test_write(self): + rc = payload.refresh(Stub(self.path, "b" * 40), out=self.lines.append) + self.assertEqual(rc, EXIT_OK) + self.assertEqual(json.loads(self.path.read_text())["tool"]["commit"], "b" * 40) + self.assertIn("next: make tool-payload", self.lines[-1]) + + def test_invalid_never_replaces(self): + rc = payload.refresh(Stub(self.path, "not a digest"), fail=self.errors.append) + self.assertEqual(rc, EXIT_USAGE) + self.assertEqual(self.path.read_text(), self.before) + + def test_upstream_error(self): + rc = payload.refresh(Stub(self.path, "b" * 40, fail=True), fail=self.errors.append) + self.assertEqual(rc, EXIT_USAGE) + self.assertIn("api unreachable", self.errors[0]) + + def test_diff_sections(self): + self.assertEqual(payload.diff_pins({"a": {"x": 1}}, {"a": {"x": 1}, "b": {"y": 2}}), + ["b: None -> {'y': 2}"]) + self.assertEqual(payload.diff_pins({"a": {"x": 1, "y": 1}}, {"a": {"x": 2, "y": 1}}), + ["a.x: 1 -> 2"]) + + def test_a_bad_upstream_is_an_error_not_a_traceback(self): + for error in (OSError("unreachable"), KeyError("tag_name"), IndexError("x")): + out = [] + rc = payload.refresh(Stub(self.path, "b" * 40, error=error), + None, False, out.append, out.append) + self.assertEqual(rc, EXIT_USAGE, error) + self.assertEqual(self.path.read_text(), self.before) + + @unittest.skipIf(os.geteuid() == 0, "root ignores directory permissions") + def test_an_unwritable_pins_directory_is_an_error(self): + os.chmod(self.dir, 0o500) + try: + rc = payload.refresh(Stub(self.path, "b" * 40), out=self.lines.append, + fail=self.errors.append) + finally: + os.chmod(self.dir, 0o700) + self.assertEqual(rc, EXIT_USAGE) + self.assertTrue(self.errors) + self.assertEqual(self.path.read_text(), self.before) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/conformance/selftest/test_report.py b/tests/conformance/selftest/test_report.py new file mode 100644 index 00000000..db6c25bb --- /dev/null +++ b/tests/conformance/selftest/test_report.py @@ -0,0 +1,80 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +import json +import unittest +from pathlib import Path + +from conformance import report +from conformance.selftest.fixture import TempDirTest +from conformance.model import Attempt, CaseResult, Invocation, Status, Verdict + + +def case(cid, status, verdict, detail=""): + inv = Invocation("normal", 1500, exit_code=0 if status is Status.PASS else 1) + return CaseResult(cid, "fake", "host", status, verdict, {"type": "expect_pass"}, + [Attempt(status, inv)], detail) + + +class ReportTest(TempDirTest): + def setUp(self): + super().setUp() + self.meta = {"suite": "fake", "backend": "host", "scope": "pr", "elapsed_s": 61.4} + + def test_empty_is_red(self): + self.assertEqual(report.gate([]), "red") + lines = report.summary_lines(self.meta, [], self.dir) + self.assertIn(" RED no cases ran", lines) + self.assertTrue(lines[-1].startswith("RESULT: RED")) + + def test_round_trip_and_summary(self): + cases = [case("fake:b/p", Status.PASS, Verdict.AS_EXPECTED), + case("fake:b/f", Status.FAIL, Verdict.UNEXPECTED_FAILURE, "fake:b/f: FAIL \x01 raw")] + report.write(self.dir, self.meta, cases) + meta, again = report.load(self.dir) + self.assertEqual(again, cases) + self.assertEqual(meta["scope"], "pr") + text = (self.dir / "summary.txt").read_text() + self.assertIn("conformance fake/host pr: 2 cases in 1m01s", text) + self.assertIn(" unexpected_failure 1 unexpected_pass 0 error 0", text) + self.assertIn(" RED fake:b/f: FAIL", text) + self.assertIn("RESULT: RED", text) + doc = json.loads((self.dir / report.RESULTS).read_text()) + self.assertEqual(doc["kind"], "run") + + def test_contradiction_is_rejected(self): + report.write(self.dir, self.meta, [case("fake:b/p", Status.PASS, Verdict.AS_EXPECTED)]) + doc = json.loads((self.dir / report.RESULTS).read_text()) + doc["gate"] = "red" + (self.dir / report.RESULTS).write_text(json.dumps(doc)) + with self.assertRaises(report.ReportError): + report.load(self.dir) + + def test_a_wrong_shape_is_rejected(self): + for text in ("[]", '{"schema_version": 1, "kind": "run", ' + '"cases": null, "run": {}}', + '{"schema_version": 1, "kind": "run", ' + '"cases": [], "run": []}', + '{"schema_version": 1, "kind": "run", ' + '"cases": ["x"], "run": {}}'): + (self.dir / report.RESULTS).write_text(text) + with self.assertRaises(report.ReportError, msg=text): + report.load(self.dir) + + def test_markdown(self): + report.write(self.dir / "fake" / "host" / "1", self.meta, + [case("fake:b/p", Status.PASS, Verdict.AS_EXPECTED)]) + bad = self.dir / "fake" / "host" / "2" + bad.mkdir(parents=True) + (bad / report.RESULTS).write_text('{"schema_version": 1}') + worse = self.dir / "fake" / "host" / "3" + worse.mkdir(parents=True) + (worse / report.RESULTS).write_text("[]") + text = report.markdown(self.dir) + self.assertIn("| fake/host | pr | green | 1 | 0 | 0 | 0 |", text) + self.assertEqual(text.count("| error |"), 2) + self.assertIn("no conformance results", report.markdown(self.dir / "none")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/conformance/selftest/test_runner.py b/tests/conformance/selftest/test_runner.py new file mode 100644 index 00000000..81df02ee --- /dev/null +++ b/tests/conformance/selftest/test_runner.py @@ -0,0 +1,72 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +import unittest +from pathlib import Path + +from conformance import expectations, report, runner +from conformance.model import Status, Verdict +from conformance.selftest.fixture import FixtureProvider, LocalBackend, TempDirTest, setup + + +class RunnerTest(TempDirTest): + def setUp(self): + super().setUp() + setup(self.root) + self.provider = FixtureProvider(self.root) + self.backend = LocalBackend() + self.exps = expectations.load( + "fixture", "elfuse", self.provider.expectations_dir + ) + self.log = [] + + def lane(self, scope, **kwargs): + cases = self.provider.enumerate( + self.backend, self.provider.selection.groups(scope) + ) + results = runner.run_lane( + self.provider, self.backend, cases, self.exps, self.root / "results", + log=self.log.append, **kwargs + ) + return {result.id: result for result in results} + + def test_pr_scope_is_green(self): + results = self.lane("pr") + self.assertEqual(report.gate(results.values()), "green") + self.assertEqual( + results["fixture:basic/fail"].verdict, Verdict.AS_EXPECTED + ) + flaky = results["fixture:basic/flaky"] + self.assertEqual(flaky.verdict, Verdict.FLAKED) + self.assertEqual( + [attempt.status for attempt in flaky.attempts], + [Status.FAIL, Status.PASS], + ) + quarantined = results["fixture:basic/quarantined"] + self.assertEqual(quarantined.verdict, Verdict.FLAKED) + self.assertEqual( + [attempt.status for attempt in quarantined.attempts], + [Status.FAIL, Status.FAIL, Status.FAIL], + ) + + def test_no_retry_keeps_quarantine_non_gating(self): + result = self.lane("pr", retry=False)["fixture:basic/flaky"] + self.assertEqual(result.verdict, Verdict.FLAKED) + self.assertEqual(len(result.attempts), 1) + + def test_unresolved_batch_case_runs_alone(self): + result = self.lane("full")["fixture:slow/unresolved"] + self.assertEqual(result.verdict, Verdict.AS_EXPECTED) + self.assertTrue(any("rerunning alone" in line for line in self.log)) + + def test_bootstrap_records_without_expectations(self): + results = self.lane("pr", bootstrap=True) + self.assertEqual(report.gate(results.values()), "green") + self.assertEqual( + results["fixture:basic/fail"].verdict, Verdict.AS_EXPECTED + ) + self.assertEqual(len(results["fixture:basic/flaky"].attempts), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/conformance/selftest/test_seed.py b/tests/conformance/selftest/test_seed.py new file mode 100644 index 00000000..6fc112d4 --- /dev/null +++ b/tests/conformance/selftest/test_seed.py @@ -0,0 +1,98 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +import tempfile +import unittest +from pathlib import Path + +from conformance import expectations, jsonc, seed +from conformance.model import Attempt, CaseResult, Invocation, Status, Verdict + + +def case(cid, status, verdict=Verdict.AS_EXPECTED, expectation="expect_pass"): + inv = Invocation("normal", 1, exit_code=0) + return CaseResult(cid, "s", "b", status, verdict, {"type": expectation}, [Attempt(status, inv)]) + + +class SeedTest(unittest.TestCase): + def test_bootstrap_collapses_whole_groups(self): + cases = [case("s:a/1", Status.FAIL), case("s:a/2", Status.BROK), + case("s:b/1", Status.FAIL), case("s:b/2", Status.PASS), + case("s:c/1", Status.CONF), case("s:d/1", Status.TIMEOUT), case("s:d/2", Status.CRASH), + case("s:e/1", Status.PASS), case("s:e/2", Status.SKIP)] + actions = seed.propose(cases, "r", bootstrap=True) + self.assertEqual(actions, [ + {"type": "expect_failure", "reason": "r", "matchers": ["s:a/*", "s:b/1"]}, + {"type": "expect_conf", "reason": "r", "matchers": ["s:c/1"]}, + {"type": "skip", "reason": "r", "matchers": ["s:d/*"]}, + ]) + + def test_a_partial_group_never_collapses(self): + cases = [case("s:a/1", Status.FAIL, Verdict.UNEXPECTED_FAILURE), + case("s:a/2", Status.PASS, Verdict.AS_EXPECTED)] + self.assertEqual(seed.propose(cases, "r", False)[0]["matchers"], ["s:a/1"]) + + def test_whole_groups_disabled_lists_members(self): + cases = [case("s:a/1", Status.FAIL, Verdict.UNEXPECTED_FAILURE), + case("s:a/2", Status.FAIL, Verdict.UNEXPECTED_FAILURE)] + self.assertEqual(seed.propose(cases, "r", False, whole_groups=False)[0]["matchers"], + ["s:a/1", "s:a/2"]) + + def test_bootstrap_ignores_what_an_earlier_leaf_recorded(self): + cases = [case("s:a/1", Status.FAIL, expectation="expect_failure"), case("s:a/2", Status.FAIL)] + self.assertEqual(seed.propose(cases, "r", True), + [{"type": "expect_failure", "reason": "r", "matchers": ["s:a/*"]}]) + + def test_gated_seeds_only_reds(self): + cases = [case("s:a/1", Status.PASS, Verdict.UNEXPECTED_PASS, "expect_failure"), + case("s:a/2", Status.FAIL, Verdict.AS_EXPECTED, "expect_failure"), + case("s:b/1", Status.FAIL, Verdict.UNEXPECTED_FAILURE)] + self.assertEqual(seed.propose(cases, "r", False), [ + {"type": "expect_pass", "reason": "r", "matchers": ["s:a/1"]}, + {"type": "expect_failure", "reason": "r", "matchers": ["s:b/1"]}, + ]) + + def test_error_refused(self): + with self.assertRaises(ValueError): + seed.propose([case("s:a/1", Status.ERROR, Verdict.ERROR)], "r", True) + + def test_append_keeps_header_and_loads(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "s.jsonc").write_text('{"actions":[{"type":"expect_pass","matchers":["*"]}]}') + leaf = root / "s_b.jsonc" + leaf.write_text('// head\n// two\n{\n "actions": [\n { "include": "s.jsonc" }, // inline\n ],\n}\n') + seed.append(leaf, [{"type": "skip", "reason": "r", "since": "2026-08-25", "matchers": ["s:a/1", "s:a/2"]}]) + text = leaf.read_text() + self.assertTrue(text.startswith("// head\n// two\n{")) + self.assertNotIn("inline", text) + self.assertEqual(jsonc.loads(text)["actions"][0], {"include": "s.jsonc"}) + e = expectations.load("s", "b", root) + self.assertEqual(e.resolve("s:a/2").type, "skip") + self.assertEqual(expectations.lint(root), []) + + def test_append_creates_a_loadable_leaf(self): + action = {"type": "skip", "reason": "r", "matchers": ["s:a/1"]} + for base in (True, False): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + if base: + (root / "s.jsonc").write_text('{"actions":[{"type":"expect_pass","matchers":["*"]}]}') + seed.append(root / "s_b.jsonc", [action]) + first = jsonc.loads((root / "s_b.jsonc").read_text())["actions"][0] + self.assertEqual(first, {"include": "s.jsonc"} if base + else {"type": "expect_pass", "matchers": ["*"]}) + self.assertEqual(expectations.load("s", "b", root).resolve("s:a/1").type, "skip") + + def test_append_skips_matchers_already_recorded(self): + with tempfile.TemporaryDirectory() as tmp: + leaf = Path(tmp) / "s_b.jsonc" + action = {"type": "skip", "reason": "r", "matchers": ["s:a/1"]} + seed.append(leaf, [action]) + before = leaf.read_text() + seed.append(leaf, [action]) + self.assertEqual(leaf.read_text(), before) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/conformance/selftest/test_selection.py b/tests/conformance/selftest/test_selection.py new file mode 100644 index 00000000..d555e781 --- /dev/null +++ b/tests/conformance/selftest/test_selection.py @@ -0,0 +1,68 @@ +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 + +import unittest + +from conformance import selection + +DATA = { + "schema_version": 1, + "enabled": [ + {"group": "basic", "scope": "pr"}, + {"group": "slow", "scope": "full", "timeout_s": 1}, + {"group": "narrow", "scope": "full", "only": ["keep*"]}, + ], + "declined": [{"reason": "unsupported", "groups": ["kernel"]}], +} + + +class SelectionTest(unittest.TestCase): + def test_parse(self): + sel = selection.parse(DATA, "fixture") + self.assertEqual([e.group for e in sel.groups("pr")], ["basic"]) + self.assertEqual([e.group for e in sel.groups("full")], + ["basic", "slow", "narrow"]) + self.assertEqual(sel.entry("slow").timeout_s, 1) + self.assertEqual(sel.entry("narrow").only, ("keep*",)) + self.assertEqual(sel.declined, [("unsupported", ("kernel",))]) + + def test_full_is_a_superset(self): + sel = selection.parse(DATA, "fixture") + pr = {e.group for e in sel.groups("pr")} + self.assertTrue(pr <= {e.group for e in sel.groups("full")}) + with self.assertRaises(selection.SelectionError): + sel.groups("nightly") + + def test_rejections(self): + base = {"schema_version": 1, "enabled": [{"group": "a", "scope": "pr"}]} + cases = [ + ({**base, "enabled": [{"group": "a", "scope": "pr"}, {"group": "a", "scope": "full"}]}, + "enabled twice"), + ({**base, "declined": [{"reason": "r", "groups": ["a"]}]}, "both enabled and declined"), + ({**base, "enabled": [{"group": "a", "scope": "nightly"}]}, "want pr or full"), + ({**base, "enabled": [{"group": "a", "scope": "pr", "tier": 1}]}, "unknown keys"), + ({**base, "enabled": [{"group": "a", "scope": "pr", "timeout_s": 0}]}, "positive"), + # isinstance(True, int) holds. + ({**base, "enabled": [{"group": "a", "scope": "pr", "timeout_s": True}]}, "positive"), + ({**base, "declined": [{"reason": "", "groups": ["b"]}]}, "reason and a group list"), + ({"schema_version": 2, "enabled": []}, "schema_version"), + ] + for doc, fragment in cases: + with self.assertRaises(selection.SelectionError) as cm: + selection.parse(doc, "t.jsonc") + self.assertIn(fragment, str(cm.exception)) + + def test_resolve_ids(self): + universe = ["fake:basic/pass", "fake:basic/fail", "fake:slow/crash"] + chosen, errors = selection.resolve_ids( + ["fake:basic/*", "fake:basic/pass", "fake:basic/pas", "ltp:x"], universe, "fake") + self.assertEqual(chosen, ["fake:basic/pass", "fake:basic/fail"]) + self.assertEqual(len(errors), 2) + self.assertIn("near: fake:basic/pass", errors[0]) + self.assertIn("not a fake id", errors[1]) + chosen, errors = selection.resolve_ids(["fake:basic"], universe, "fake") + self.assertEqual((chosen, errors), (["fake:basic/pass", "fake:basic/fail"], [])) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/lib/qemu-ssh.sh b/tests/lib/qemu-ssh.sh new file mode 100644 index 00000000..3d1a9b2e --- /dev/null +++ b/tests/lib/qemu-ssh.sh @@ -0,0 +1,25 @@ +# Shared ssh argv for the qemu test VM. +# +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 +# shellcheck shell=bash +# timeout(1) cannot wrap a shell function, so what the callers share is the +# argv: qemu_ssh_opts fills QEMU_SSH_OPTS from QEMU_SSH_KEY and QEMU_PORT at +# call time, and each caller builds its own ssh command line around it. The +# conformance backend spells the same list in tests/conformance/backends/ssh.py. + +# shellcheck disable=SC2034 # Consumed by the sourcing script. +qemu_ssh_opts() +{ + QEMU_SSH_OPTS=( + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null + -o LogLevel=ERROR + -o BatchMode=yes + -o ConnectTimeout=10 + -o ServerAliveInterval=10 + -o ServerAliveCountMax=6 + -i "$QEMU_SSH_KEY" + -p "$QEMU_PORT" + ) +} diff --git a/tests/qemu-runner.sh b/tests/qemu-runner.sh index e5837fe4..7832f6ec 100755 --- a/tests/qemu-runner.sh +++ b/tests/qemu-runner.sh @@ -19,6 +19,9 @@ _QR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")/.." && pwd)" _QR_FIX="${_QR_DIR}/externals/test-fixtures" +# shellcheck source=tests/lib/qemu-ssh.sh +source "${_QR_DIR}/tests/lib/qemu-ssh.sh" + QEMU_BIN="${QEMU_BIN:-qemu-system-aarch64}" QEMU_PORT="${QEMU_PORT:-2222}" QEMU_MEM="${QEMU_MEM:-2048}" @@ -165,7 +168,11 @@ qemu_start() # a dedicated tmpfs, as any regular system has, so paths under /tmp map to a # resolvable st_dev. Guarded so a repeated qemu_start against a running VM # does not stack mounts. - _qemu_ssh_raw 'grep -q " /tmp tmpfs " /proc/mounts || mount -t tmpfs tmpfs /tmp' + if ! _qemu_ssh_raw 'grep -q " /tmp tmpfs " /proc/mounts || mount -t tmpfs tmpfs /tmp'; then + echo "qemu-runner: could not prepare /tmp in the guest" >&2 + qemu_stop + return 1 + fi } # Each call opens a fresh ssh connection. Avoids ControlMaster pitfalls (master @@ -174,16 +181,8 @@ qemu_start() # the suite's tolerance. _qemu_ssh_raw() { - ssh -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - -o BatchMode=yes \ - -o ConnectTimeout=10 \ - -o ServerAliveInterval=10 \ - -o ServerAliveCountMax=6 \ - -i "$QEMU_SSH_KEY" \ - -p "$QEMU_PORT" \ - root@127.0.0.1 "$@" + qemu_ssh_opts + ssh "${QEMU_SSH_OPTS[@]}" root@127.0.0.1 "$@" } # Run a command in the VM. Any argument that is an absolute path under the host @@ -210,6 +209,14 @@ qemu_stop() if [ -n "$_QR_PIDFILE" ] && [ -s "$_QR_PIDFILE" ]; then local pid pid=$(cat "$_QR_PIDFILE" 2> /dev/null) + + # A state file outlives its VM, so the pid it names may since have been + # recycled. qemu_start gives qemu this pidfile, and mktemp makes the + # path unique, so the argv is what proves the process is ours. + case " $(ps -o command= -p "${pid:-0}" 2> /dev/null) " in + *" -pidfile $_QR_PIDFILE "*) ;; + *) pid="" ;; + esac if [ -n "$pid" ] && kill -0 "$pid" 2> /dev/null; then kill "$pid" 2> /dev/null # give qemu time to exit cleanly; force-kill if it lingers @@ -218,7 +225,15 @@ qemu_stop() kill -0 "$pid" 2> /dev/null || break sleep 1 done - kill -0 "$pid" 2> /dev/null && kill -9 "$pid" 2> /dev/null + if kill -0 "$pid" 2> /dev/null; then + kill -9 "$pid" 2> /dev/null + sleep 1 + # Keep the pidfile and state so a later stop can retry. + if kill -0 "$pid" 2> /dev/null; then + echo "qemu-runner: pid $pid survived SIGKILL" >&2 + return 1 + fi + fi fi fi if [ -n "$_QR_PIDFILE" ]; then @@ -229,30 +244,67 @@ qemu_stop() _QR_CTL="" } -# When sourced, register a cleanup trap that does not clobber the caller's -# existing trap chain. When executed directly, the EXIT trap fires on script -# exit. -trap 'qemu_stop' EXIT +qemu_write_state() +{ + printf 'port=%s\nkey=%s\npidfile=%s\n' \ + "$QEMU_PORT" "$QEMU_SSH_KEY" "$_QR_PIDFILE" > "$1.tmp" && mv -f "$1.tmp" "$1" +} + +qemu_read_state() +{ + [ -s "$1" ] || { + echo "qemu-runner: no state file $1" >&2 + return 1 + } + QEMU_PORT="$(sed -n 's/^port=//p' "$1")" + QEMU_SSH_KEY="$(sed -n 's/^key=//p' "$1")" + _QR_PIDFILE="$(sed -n 's/^pidfile=//p' "$1")" + + # Restrict cleanup to the directory shape created by mktemp. + case "$_QR_PIDFILE" in + */elfuse-qemu.*/qemu.pid) ;; + *) + echo "qemu-runner: $1 names no qemu-runner pidfile: $_QR_PIDFILE; remove the file once the VM is gone" >&2 + return 1 + ;; + esac +} -# CLI driver: when run directly, support 'qemu-runner.sh start|exec|stop'. if [ "${BASH_SOURCE[0]:-$0}" = "$0" ]; then cmd="${1:-help}" shift || true + state_file="" + if [ "$cmd" != exec ] && [ "${1:-}" = "--state-file" ]; then + state_file="${2:?--state-file needs a path}" + shift 2 + fi case "$cmd" in start) - qemu_start + if [ -n "$state_file" ] && [ -e "$state_file" ]; then + echo "qemu-runner: $state_file names a live VM; run stop first" >&2 + exit 1 + fi + trap 'qemu_stop' EXIT + qemu_start || exit 1 + if [ -n "$state_file" ]; then + qemu_write_state "$state_file" || exit 1 + trap - EXIT + fi echo "PORT=$QEMU_PORT KEY=$QEMU_SSH_KEY" ;; exec) + trap 'qemu_stop' EXIT qemu_start qemu_exec "$@" ;; stop) - qemu_stop + [ -z "$state_file" ] || qemu_read_state "$state_file" || exit 1 + qemu_stop || exit 1 + [ -z "$state_file" ] || rm -f "$state_file" ;; *) cat << EOF -Usage: $0 +Usage: $0 Boots qemu-system-aarch64 with the test fixtures and exposes ssh. The host repo is shared into the VM at /mnt/host (read-only). diff --git a/tests/test-matrix.sh b/tests/test-matrix.sh index 038dfeec..8cd7d35d 100755 --- a/tests/test-matrix.sh +++ b/tests/test-matrix.sh @@ -73,6 +73,8 @@ source "${REPO_ROOT}/tests/test-config.sh" TEST_LABEL_WIDTH=45 # shellcheck source=tests/lib/test-runner.sh source "${REPO_ROOT}/tests/lib/test-runner.sh" +# shellcheck source=tests/lib/qemu-ssh.sh +source "${REPO_ROOT}/tests/lib/qemu-ssh.sh" # Globals (test-runner.sh seeds pass/fail/skip; test-matrix.sh resets them per # mode and tracks no extra counters). @@ -192,15 +194,8 @@ run_qemu() if [ "${#args[@]}" -gt 0 ]; then printf -v quoted '%q ' "${args[@]}" fi - timeout 60 ssh \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - -o BatchMode=yes \ - -o ConnectTimeout=10 \ - -o ServerAliveInterval=15 \ - -o ServerAliveCountMax=4 \ - -i "$QEMU_SSH_KEY" -p "$QEMU_PORT" \ + qemu_ssh_opts + timeout 60 ssh "${QEMU_SSH_OPTS[@]}" \ root@127.0.0.1 "cd /mnt/host && ${quoted}" 2> /dev/null } diff --git a/tests/test-qemu-runner-stop.sh b/tests/test-qemu-runner-stop.sh new file mode 100755 index 00000000..4bef013d --- /dev/null +++ b/tests/test-qemu-runner-stop.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash + +# test-qemu-runner-stop.sh -- Pin qemu-runner.sh stop against a state file +# +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Usage: tests/test-qemu-runner-stop.sh +# +# A state file outlives its VM, so the pid it records may since have been +# recycled by an unrelated process. stop --state-file has to leave such a +# process alone and still remove the record, and has to terminate a process +# whose argv names the run's own pidfile, as qemu's does. Both halves run +# against stand-ins, so no VM boots. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +RUNNER="$SCRIPT_DIR/qemu-runner.sh" +# shellcheck source=tests/lib/report.sh +. "$SCRIPT_DIR/lib/report.sh" + +work="$(mktemp -d)" +victims=() +cleanup() +{ + local p + for p in ${victims[@]+"${victims[@]}"}; do + kill "$p" 2> /dev/null || true + done + rm -rf "$work" +} +trap cleanup EXIT + +# qemu_read_state accepts only the directory shape mktemp gives a run. +rundir="$work/elfuse-qemu.test" +pidfile="$rundir/qemu.pid" +state="$work/qemu.state" + +arm() +{ + mkdir -p "$rundir" + printf '%s\n' "$1" > "$pidfile" + printf 'port=1\nkey=/dev/null\npidfile=%s\n' "$pidfile" > "$state" +} + +check() +{ + local label="$1" want="$2" got="$3" + if [ "$want" = "$got" ]; then + report_pass "$label" + else + report_fail "$label (got $got, want $want)" + fi +} + +alive() +{ + kill -0 "$1" 2> /dev/null && echo alive || echo dead +} + +present() +{ + [ -e "$1" ] && echo present || echo gone +} + +# A recycled pid: a sleep whose argv never mentions the pidfile. +sleep 300 & +bystander=$! +victims+=("$bystander") +arm "$bystander" +rc=0 +bash "$RUNNER" stop --state-file "$state" > /dev/null 2>&1 || rc=$? +check "stop returns 0 for a recycled pid" 0 "$rc" +check "a recycled pid survives stop" alive "$(alive "$bystander")" +check "stop removes the stale state file" gone "$(present "$state")" +check "stop removes the stale run directory" gone "$(present "$rundir")" + +# The run's own process: its argv carries the pidfile, and it exits on TERM. +loop='trap "exit 0" TERM; while :; do sleep 1; done' +bash -c "$loop" bash -pidfile "$pidfile" & +own=$! +victims+=("$own") +arm "$own" +rc=0 +bash "$RUNNER" stop --state-file "$state" > /dev/null 2>&1 || rc=$? +check "stop returns 0 for the run's own process" 0 "$rc" +check "the run's own process is terminated" dead "$(alive "$own")" +check "stop removes the state file after a kill" gone "$(present "$state")" + +report_summary +[ "$fail" -eq 0 ]