test: enforce native coverage validation

.github/workflows/ci.yml

@@ -65,11 +65,25 @@ jobs:
       - run: make install-ci
 
       - name: Test suite
+        if: matrix.python-version != '3.12'
         # Durations are reported because a slow test is invisible until
         # someone profiles the suite, and the runner is the only place that
         # measures the hardware CI actually runs on.
         run: make test PYTEST_ARGS="--durations=25"
 
+      - name: Test suite with coverage
+        if: matrix.python-version == '3.12'
+        run: make coverage PYTEST_ARGS="--durations=25"
+
+      - name: Coverage report
+        if: always() && matrix.python-version == '3.12'
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: coverage-python-3.12
+          path: coverage.xml
+          if-no-files-found: error
+          retention-days: 14
+
       - name: Offline self-check
         run: make smoke
 

CONTRIBUTING.md

@@ -116,6 +116,20 @@ assuming `.git` is a directory.
 model catalog, every probe's assets, and the plugin entry points without
 making a single model call.
 
+### Reproducing CI and coverage
+
+Run `make validate-ci` for the native checks and clean distribution validation.
+`make validate-native` runs static checks, dependency license policy, coverage,
+the offline self-check, and installed extension checks without building the
+distribution archives. CI runs the same test suite across supported Python
+versions and enforces the 84% coverage floor on Python 3.12.
+
+Use `make coverage PYTEST_WORKERS=2` locally. The XML report is `coverage.xml`;
+change its path with `COV_XML`, or add pytest options through `PYTEST_ARGS`, for
+example `make coverage PYTEST_ARGS="--junitxml=junit.xml"`. See
+[Local and CI validation](docs/engineering/ci.md) for bundle membership, report
+artifacts, and failure diagnosis.
+
 ### Choosing models and providers
 
 Three providers are built in, each needing only a credential. Pick the shipped

Makefile

@@ -42,6 +42,7 @@ PYTEST_WORKERS ?= auto
 
 # Importable package names for coverage (not paths -- src/
 # installs as `usersim.engine`).
+COV_XML ?= coverage.xml
 COV_PKGS ?= usersim
 COV_ARGS := $(foreach pkg,$(COV_PKGS),--cov=$(pkg))
 
@@ -59,7 +60,7 @@ COV_FAIL_UNDER ?= 84
 .DEFAULT_GOAL := help
 
 .PHONY: help bootstrap check-tools check-lock install-dev install-ci install-kernel health-check check-notebooks-clean lint lint-fix format format-check check-all \
-        check-hooks check-all-fix test test-fast coverage smoke dist check-dist check-release-tag check-extensions check-doc-links clean-notebooks check-license-headers update-license-headers check-dependency-licenses clean
+        check-hooks check-all-fix validate-native validate-ci test test-fast coverage smoke dist check-dist check-release-tag check-extensions check-doc-links clean-notebooks check-license-headers update-license-headers check-dependency-licenses clean
 
 
 help:  ## Show the available targets
@@ -138,8 +139,12 @@ test-fast:  ## Stop at the first failure, serially (tight debugging loop)
 coverage:  ## Run tests with coverage and enforce the floor
 	$(UV) run pytest -n $(PYTEST_WORKERS) $(COV_ARGS) \
 		--cov-report=term-missing \
-		--cov-report=xml \
-		--cov-fail-under=$(COV_FAIL_UNDER)
+		--cov-report="xml:$(COV_XML)" \
+		--cov-fail-under=$(COV_FAIL_UNDER) $(PYTEST_ARGS)
+
+validate-native: check-all check-dependency-licenses coverage smoke check-extensions  ## Check source and installed extension behavior
+
+validate-ci: validate-native check-dist  ## Run the native CI checks, including distribution validation
 
 smoke:  ## Offline self-check of config, registries, and report builders
 	$(UV) run usersim smoke

docs/engineering/ci.md

@@ -0,0 +1,46 @@
+# Local and CI validation
+
+CI calls repository Make targets so failures can be reproduced locally. After
+`make install-dev`, run `make validate-ci` to execute the native validation
+bundle, including distribution checks.
+
+| Target | Checks |
+| --- | --- |
+| `make check-all` | Static checks defined by the repository. |
+| `make check-dependency-licenses` | Dependency license policy. |
+| `make coverage` | Ordinary test suite with an 84% minimum total coverage. |
+| `make smoke` | Offline configuration, registry, asset, and reporting self-check. |
+| `make check-extensions` | Out-of-tree installed extension behavior. |
+| `make validate-native` | The five targets above. |
+| `make check-dist` | Build and validate archives, then check a clean wheel installation. |
+| `make validate-ci` | `validate-native` plus `check-dist`. |
+
+The distribution archives are produced by `check-dist`; extension checks build
+a separate temporary wheel. CI also runs the test suite across Python 3.11
+through 3.14; a local bundle checks the active
+Python environment. Provider health checks run separately and need credentials.
+
+## Coverage
+
+The Python 3.12 CI matrix job runs `make coverage`; the other Python versions
+run `make test`. Both test paths use the same ordinary suite. Coverage failures
+fail the existing `Tests (Python 3.12)` check. The 84% floor lives in the Makefile
+and applies to local runs as well as CI.
+
+```bash
+make coverage PYTEST_WORKERS=2
+make coverage PYTEST_ARGS="--durations=25 --junitxml=junit.xml"
+make coverage COV_XML=coverage-custom.xml
+```
+
+`coverage.xml` is the default XML output. `COV_XML` changes its location, and
+`PYTEST_ARGS` passes additional pytest options through to the coverage command.
+Create a custom output directory before writing there. A focused pytest
+selection also limits measured execution; run the complete suite to reproduce
+the CI coverage gate. `COV_FAIL_UNDER` is configurable for experiments, but
+maintainers must not lower the committed floor to make a change pass.
+
+CI uploads `coverage.xml` as `coverage-python-3.12` for 14 days, even when the
+coverage command fails. Missing XML fails the upload step, and uploading a
+report does not suppress a preceding test or coverage failure. If collection or
+setup stops before XML is produced, inspect the failed step's logs.

tests/conftest.py

@@ -22,6 +22,35 @@
 import pytest
 
 
+@pytest.fixture
+def command_test_env() -> Callable[..., dict[str, str]]:
+    """Build an isolated environment for command-default contract tests.
+
+    Opt in only when testing defaults: inheritance tests should keep their
+    ambient environment. Explicit overrides are applied after removing outer
+    Make state and test selectors; tool and environment selection still inherit.
+    """
+
+    def environment(**overrides: str) -> dict[str, str]:
+        env = os.environ.copy()
+        for name in (
+            "MAKEFLAGS",
+            "MFLAGS",
+            "MAKEOVERRIDES",
+            "MAKELEVEL",
+            "PYTEST_ARGS",
+            "PYTEST_WORKERS",
+            "COV_PKGS",
+            "COV_XML",
+            "COV_FAIL_UNDER",
+        ):
+            env.pop(name, None)
+        env.update(overrides)
+        return env
+
+    return environment
+
+
 class _WordSplitEncoder:
     """Deterministic stand-in for a ``tiktoken`` encoding."""
 

tests/test_coverage_gate.py

@@ -0,0 +1,114 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Coverage enforcement uses the same overridable Make command locally and in CI."""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+import yaml
+
+_ROOT = Path(__file__).resolve().parents[1]
+
+
+@pytest.mark.parametrize("status", [0, 9])
+@pytest.mark.parametrize("overrides", [False, True])
+def test_coverage_arguments_and_exit_status(tmp_path, status, overrides, monkeypatch, command_test_env):
+    # A caller may itself be make coverage with report overrides. Test defaults
+    # independently of its exported variables and recursive-Make argument state.
+    for name, value in {
+        "MAKEFLAGS": " -- COV_XML=ambient.xml PYTEST_ARGS=-k=ambient",
+        "MFLAGS": "--no-print-directory",
+        "MAKEOVERRIDES": "COV_XML=ambient.xml",
+        "MAKELEVEL": "4",
+        "COV_XML": "ambient.xml",
+        "PYTEST_ARGS": "-k ambient",
+        "PYTEST_WORKERS": "99",
+        "COV_PKGS": "ambient",
+        "COV_FAIL_UNDER": "100",
+    }.items():
+        monkeypatch.setenv(name, value)
+    executable = tmp_path / "uv"
+    recorded = tmp_path / "arguments.json"
+    executable.write_text(
+        f"#!{sys.executable}\n"
+        "import json, os, pathlib, sys\n"
+        "pathlib.Path(os.environ['COVERAGE_TEST_ARGS']).write_text(json.dumps(sys.argv[1:]))\n"
+        "sys.exit(int(os.environ['COVERAGE_TEST_STATUS']))\n"
+    )
+    executable.chmod(0o755)
+    options = ["PYTEST_WORKERS=0"]
+    if overrides:
+        options += [
+            "COV_PKGS=example another",
+            "COV_XML=custom report.xml",
+            "COV_FAIL_UNDER=88",
+            "PYTEST_ARGS=--junitxml=junit.xml -k 'selected or other'",
+        ]
+    result = subprocess.run(
+        ["make", "coverage", f"UV={executable}", *options],
+        cwd=_ROOT,
+        env=command_test_env(COVERAGE_TEST_ARGS=str(recorded), COVERAGE_TEST_STATUS=str(status)),
+        capture_output=True,
+        text=True,
+        check=False,
+        timeout=30,
+    )
+    assert (result.returncode == 0) is (status == 0)
+    arguments = recorded.read_text()
+    args = json.loads(arguments)
+    assert args[:5] == ["run", "pytest", "-n", "0", "--cov=example" if overrides else "--cov=usersim"]
+    assert f"--cov-fail-under={88 if overrides else 84}" in args
+    assert f"--cov-report=xml:{'custom report.xml' if overrides else 'coverage.xml'}" in args
+    if overrides:
+        assert "--cov=another" in args
+        assert args[-3:] == ["--junitxml=junit.xml", "-k", "selected or other"]
+
+
+def test_ci_enforces_coverage_once_and_retains_failure_artifacts():
+    workflow = yaml.safe_load((_ROOT / ".github/workflows/ci.yml").read_text())
+    job = workflow["jobs"]["test"]
+    assert job["name"] == "Tests (Python ${{ matrix.python-version }})"
+    assert "3.12" in job["strategy"]["matrix"]["python-version"]
+    tests = [step for step in job["steps"] if step.get("run", "").startswith("make test ")]
+    coverage = [step for step in job["steps"] if step.get("run", "").startswith("make coverage ")]
+    assert len(tests) == len(coverage) == 1
+    assert tests[0]["if"] == "matrix.python-version != '3.12'"
+    assert coverage[0]["if"] == "matrix.python-version == '3.12'"
+    assert not coverage[0].get("continue-on-error", False)
+    upload = next(step for step in job["steps"] if step.get("uses", "").startswith("actions/upload-artifact@"))
+    assert upload["if"] == "always() && matrix.python-version == '3.12'"
+    assert upload["with"]["path"] == "coverage.xml"
+    assert upload["with"]["if-no-files-found"] == "error"
+    assert not upload.get("continue-on-error", False)
+
+
+def test_native_and_ci_bundle_membership():
+    result = subprocess.run(
+        ["make", "--question", "--print-data-base", "validate-ci"],
+        cwd=_ROOT,
+        capture_output=True,
+        text=True,
+        check=False,
+        timeout=30,
+    )
+    # --question returns 1 when any prerequisite would need executing.
+    assert result.returncode in (0, 1), result.stderr
+    graph = dict(
+        line.split(":", 1)
+        for line in result.stdout.splitlines()
+        if line.startswith(("validate-native:", "validate-ci:"))
+    )
+    assert set(graph["validate-native"].split()) == {
+        "check-all",
+        "check-dependency-licenses",
+        "coverage",
+        "smoke",
+        "check-extensions",
+    }
+    assert set(graph["validate-ci"].split()) == {"validate-native", "check-dist"}