diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2aaa0eb..3e6bf15 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,11 +65,25 @@ jobs: - run: make install-ci - name: Test suite + if: matrix.python-version != '3.12' # Durations are reported because a slow test is invisible until # someone profiles the suite, and the runner is the only place that # measures the hardware CI actually runs on. run: make test PYTEST_ARGS="--durations=25" + - name: Test suite with coverage + if: matrix.python-version == '3.12' + run: make coverage PYTEST_ARGS="--durations=25" + + - name: Coverage report + if: always() && matrix.python-version == '3.12' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: coverage-python-3.12 + path: coverage.xml + if-no-files-found: error + retention-days: 14 + - name: Offline self-check run: make smoke diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c5ff069..56f9a64 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -116,6 +116,20 @@ assuming `.git` is a directory. model catalog, every probe's assets, and the plugin entry points without making a single model call. +### Reproducing CI and coverage + +Run `make validate-ci` for the native checks and clean distribution validation. +`make validate-native` runs static checks, dependency license policy, coverage, +the offline self-check, and installed extension checks without building the +distribution archives. CI runs the same test suite across supported Python +versions and enforces the 84% coverage floor on Python 3.12. + +Use `make coverage PYTEST_WORKERS=2` locally. The XML report is `coverage.xml`; +change its path with `COV_XML`, or add pytest options through `PYTEST_ARGS`, for +example `make coverage PYTEST_ARGS="--junitxml=junit.xml"`. See +[Local and CI validation](docs/engineering/ci.md) for bundle membership, report +artifacts, and failure diagnosis. + ### Choosing models and providers Three providers are built in, each needing only a credential. Pick the shipped diff --git a/Makefile b/Makefile index 2f12a66..3ac8263 100644 --- a/Makefile +++ b/Makefile @@ -42,6 +42,7 @@ PYTEST_WORKERS ?= auto # Importable package names for coverage (not paths -- src/ # installs as `usersim.engine`). +COV_XML ?= coverage.xml COV_PKGS ?= usersim COV_ARGS := $(foreach pkg,$(COV_PKGS),--cov=$(pkg)) @@ -59,7 +60,7 @@ COV_FAIL_UNDER ?= 84 .DEFAULT_GOAL := help .PHONY: help bootstrap check-tools check-lock install-dev install-ci install-kernel health-check check-notebooks-clean lint lint-fix format format-check check-all \ - check-hooks check-all-fix test test-fast coverage smoke dist check-dist check-release-tag check-extensions check-doc-links clean-notebooks check-license-headers update-license-headers check-dependency-licenses clean + check-hooks check-all-fix validate-native validate-ci test test-fast coverage smoke dist check-dist check-release-tag check-extensions check-doc-links clean-notebooks check-license-headers update-license-headers check-dependency-licenses clean help: ## Show the available targets @@ -138,8 +139,12 @@ test-fast: ## Stop at the first failure, serially (tight debugging loop) coverage: ## Run tests with coverage and enforce the floor $(UV) run pytest -n $(PYTEST_WORKERS) $(COV_ARGS) \ --cov-report=term-missing \ - --cov-report=xml \ - --cov-fail-under=$(COV_FAIL_UNDER) + --cov-report="xml:$(COV_XML)" \ + --cov-fail-under=$(COV_FAIL_UNDER) $(PYTEST_ARGS) + +validate-native: check-all check-dependency-licenses coverage smoke check-extensions ## Check source and installed extension behavior + +validate-ci: validate-native check-dist ## Run the native CI checks, including distribution validation smoke: ## Offline self-check of config, registries, and report builders $(UV) run usersim smoke diff --git a/docs/engineering/ci.md b/docs/engineering/ci.md new file mode 100644 index 0000000..75e72d0 --- /dev/null +++ b/docs/engineering/ci.md @@ -0,0 +1,46 @@ +# Local and CI validation + +CI calls repository Make targets so failures can be reproduced locally. After +`make install-dev`, run `make validate-ci` to execute the native validation +bundle, including distribution checks. + +| Target | Checks | +| --- | --- | +| `make check-all` | Static checks defined by the repository. | +| `make check-dependency-licenses` | Dependency license policy. | +| `make coverage` | Ordinary test suite with an 84% minimum total coverage. | +| `make smoke` | Offline configuration, registry, asset, and reporting self-check. | +| `make check-extensions` | Out-of-tree installed extension behavior. | +| `make validate-native` | The five targets above. | +| `make check-dist` | Build and validate archives, then check a clean wheel installation. | +| `make validate-ci` | `validate-native` plus `check-dist`. | + +The distribution archives are produced by `check-dist`; extension checks build +a separate temporary wheel. CI also runs the test suite across Python 3.11 +through 3.14; a local bundle checks the active +Python environment. Provider health checks run separately and need credentials. + +## Coverage + +The Python 3.12 CI matrix job runs `make coverage`; the other Python versions +run `make test`. Both test paths use the same ordinary suite. Coverage failures +fail the existing `Tests (Python 3.12)` check. The 84% floor lives in the Makefile +and applies to local runs as well as CI. + +```bash +make coverage PYTEST_WORKERS=2 +make coverage PYTEST_ARGS="--durations=25 --junitxml=junit.xml" +make coverage COV_XML=coverage-custom.xml +``` + +`coverage.xml` is the default XML output. `COV_XML` changes its location, and +`PYTEST_ARGS` passes additional pytest options through to the coverage command. +Create a custom output directory before writing there. A focused pytest +selection also limits measured execution; run the complete suite to reproduce +the CI coverage gate. `COV_FAIL_UNDER` is configurable for experiments, but +maintainers must not lower the committed floor to make a change pass. + +CI uploads `coverage.xml` as `coverage-python-3.12` for 14 days, even when the +coverage command fails. Missing XML fails the upload step, and uploading a +report does not suppress a preceding test or coverage failure. If collection or +setup stops before XML is produced, inspect the failed step's logs. diff --git a/tests/conftest.py b/tests/conftest.py index a494ad1..17e23d1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,6 +22,35 @@ import pytest +@pytest.fixture +def command_test_env() -> Callable[..., dict[str, str]]: + """Build an isolated environment for command-default contract tests. + + Opt in only when testing defaults: inheritance tests should keep their + ambient environment. Explicit overrides are applied after removing outer + Make state and test selectors; tool and environment selection still inherit. + """ + + def environment(**overrides: str) -> dict[str, str]: + env = os.environ.copy() + for name in ( + "MAKEFLAGS", + "MFLAGS", + "MAKEOVERRIDES", + "MAKELEVEL", + "PYTEST_ARGS", + "PYTEST_WORKERS", + "COV_PKGS", + "COV_XML", + "COV_FAIL_UNDER", + ): + env.pop(name, None) + env.update(overrides) + return env + + return environment + + class _WordSplitEncoder: """Deterministic stand-in for a ``tiktoken`` encoding.""" diff --git a/tests/test_coverage_gate.py b/tests/test_coverage_gate.py new file mode 100644 index 0000000..35dd7e9 --- /dev/null +++ b/tests/test_coverage_gate.py @@ -0,0 +1,114 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Coverage enforcement uses the same overridable Make command locally and in CI.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +_ROOT = Path(__file__).resolve().parents[1] + + +@pytest.mark.parametrize("status", [0, 9]) +@pytest.mark.parametrize("overrides", [False, True]) +def test_coverage_arguments_and_exit_status(tmp_path, status, overrides, monkeypatch, command_test_env): + # A caller may itself be make coverage with report overrides. Test defaults + # independently of its exported variables and recursive-Make argument state. + for name, value in { + "MAKEFLAGS": " -- COV_XML=ambient.xml PYTEST_ARGS=-k=ambient", + "MFLAGS": "--no-print-directory", + "MAKEOVERRIDES": "COV_XML=ambient.xml", + "MAKELEVEL": "4", + "COV_XML": "ambient.xml", + "PYTEST_ARGS": "-k ambient", + "PYTEST_WORKERS": "99", + "COV_PKGS": "ambient", + "COV_FAIL_UNDER": "100", + }.items(): + monkeypatch.setenv(name, value) + executable = tmp_path / "uv" + recorded = tmp_path / "arguments.json" + executable.write_text( + f"#!{sys.executable}\n" + "import json, os, pathlib, sys\n" + "pathlib.Path(os.environ['COVERAGE_TEST_ARGS']).write_text(json.dumps(sys.argv[1:]))\n" + "sys.exit(int(os.environ['COVERAGE_TEST_STATUS']))\n" + ) + executable.chmod(0o755) + options = ["PYTEST_WORKERS=0"] + if overrides: + options += [ + "COV_PKGS=example another", + "COV_XML=custom report.xml", + "COV_FAIL_UNDER=88", + "PYTEST_ARGS=--junitxml=junit.xml -k 'selected or other'", + ] + result = subprocess.run( + ["make", "coverage", f"UV={executable}", *options], + cwd=_ROOT, + env=command_test_env(COVERAGE_TEST_ARGS=str(recorded), COVERAGE_TEST_STATUS=str(status)), + capture_output=True, + text=True, + check=False, + timeout=30, + ) + assert (result.returncode == 0) is (status == 0) + arguments = recorded.read_text() + args = json.loads(arguments) + assert args[:5] == ["run", "pytest", "-n", "0", "--cov=example" if overrides else "--cov=usersim"] + assert f"--cov-fail-under={88 if overrides else 84}" in args + assert f"--cov-report=xml:{'custom report.xml' if overrides else 'coverage.xml'}" in args + if overrides: + assert "--cov=another" in args + assert args[-3:] == ["--junitxml=junit.xml", "-k", "selected or other"] + + +def test_ci_enforces_coverage_once_and_retains_failure_artifacts(): + workflow = yaml.safe_load((_ROOT / ".github/workflows/ci.yml").read_text()) + job = workflow["jobs"]["test"] + assert job["name"] == "Tests (Python ${{ matrix.python-version }})" + assert "3.12" in job["strategy"]["matrix"]["python-version"] + tests = [step for step in job["steps"] if step.get("run", "").startswith("make test ")] + coverage = [step for step in job["steps"] if step.get("run", "").startswith("make coverage ")] + assert len(tests) == len(coverage) == 1 + assert tests[0]["if"] == "matrix.python-version != '3.12'" + assert coverage[0]["if"] == "matrix.python-version == '3.12'" + assert not coverage[0].get("continue-on-error", False) + upload = next(step for step in job["steps"] if step.get("uses", "").startswith("actions/upload-artifact@")) + assert upload["if"] == "always() && matrix.python-version == '3.12'" + assert upload["with"]["path"] == "coverage.xml" + assert upload["with"]["if-no-files-found"] == "error" + assert not upload.get("continue-on-error", False) + + +def test_native_and_ci_bundle_membership(): + result = subprocess.run( + ["make", "--question", "--print-data-base", "validate-ci"], + cwd=_ROOT, + capture_output=True, + text=True, + check=False, + timeout=30, + ) + # --question returns 1 when any prerequisite would need executing. + assert result.returncode in (0, 1), result.stderr + graph = dict( + line.split(":", 1) + for line in result.stdout.splitlines() + if line.startswith(("validate-native:", "validate-ci:")) + ) + assert set(graph["validate-native"].split()) == { + "check-all", + "check-dependency-licenses", + "coverage", + "smoke", + "check-extensions", + } + assert set(graph["validate-ci"].split()) == {"validate-native", "check-dist"}