Makefile

# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

# usersim developer and CI entry points.
#
# CI is a thin wrapper over these targets: every check
# runs identically on a laptop and on a runner. Keep the logic HERE rather than
# in pipeline YAML -- that is what keeps a move to a different CI provider a
# translation instead of a rewrite, and what stops "works locally" drift.

SHELL := /bin/bash

UV ?= uv

# A top-level `_`-prefixed directory holds material that is not part of the
# distribution and may carry its own targets. Absent one, this expands to
# nothing.
-include $(wildcard _*/targets.mk)

# Overridable so CI can scope a job without editing recipes.
#
# scripts/ and templates/ are in scope deliberately. The gate scripts run in
# CI, and templates/probe/ is example code a contributor copies, so both
# should meet the same bar as src/. Leaving them out meant a later scope
# change would trigger a second repo-wide reformat.
PY_SRC   ?= src scripts templates
PY_TESTS ?= tests conftest.py
PY_ALL   := $(PY_SRC) $(PY_TESTS)

# Worker count for the parallel suite. `auto` follows the local core count,
# which suits a CPU-bound suite with every LLM call mocked (~2x on a laptop).
#
# Override it in containers. `auto` resolves to the HOST's core count inside
# Docker, not the cgroup CPU limit, so on a shared many-core runner it spawns
# far more workers than there are usable CPUs and the run wedges during worker
# startup, so CI pins an explicit worker count.
#
# `test-fast` stays serial deliberately: interleaved worker output makes a
# single failure much harder to read, and that target exists for debugging.
PYTEST_WORKERS ?= auto

# Importable package names for coverage (not paths -- src/
# installs as `usersim.engine`).
COV_PKGS ?= usersim
COV_ARGS := $(foreach pkg,$(COV_PKGS),--cov=$(pkg))

# The coverage floor lives here, not in CI, so `make coverage` is the same gate
# everywhere. Measured at 85% (2,644 of 17,686 statements uncovered); 84 leaves
# a point of headroom for platform differences between a laptop and the Linux
# runner while still catching roughly 180 newly uncovered statements. Ratchet it
# upward as coverage improves; do not lower it to make a branch pass.
#
# Read it as a regression tripwire, not a quality score: reporting/ emits a
# React app from Python string literals, and those lines execute during tests
# while only being asserted on by substring match.
COV_FAIL_UNDER ?= 84

.DEFAULT_GOAL := help

.PHONY: help install-dev install-ci install-kernel health-check check-notebooks-clean lint lint-fix format format-check check-all \
        check-all-fix test test-fast coverage smoke dist check-dist check-extensions check-doc-links clean-notebooks check-license-headers update-license-headers check-dependency-licenses clean

help:  ## Show the available targets
	@echo "usersim make targets:"
	@grep -hE '^[a-zA-Z_-]+:.*?## ' $(MAKEFILE_LIST) \
		| awk 'BEGIN {FS = ":.*?## "}; {printf "  \033[36m%-26s\033[0m %s\n", $$1, $$2}'

# --- setup -----------------------------------------------------------------

# Reinstalling the project restores any file that removing another distribution
# took with it, such as the `usersim` command an earlier name also installed.
install-dev:  ## Sync the workspace with dev tooling and install pre-commit hooks
	$(UV) sync --reinstall-package usersim
	@if [ ! -f .git/hooks/pre-commit ]; then \
		echo "Installing pre-commit hooks..."; \
		$(UV) run pre-commit install; \
	else \
		echo "Pre-commit hooks already installed"; \
	fi

# --locked installs strictly from uv.lock and refuses to re-resolve, so
# dependency drift becomes a build failure rather than a silent
# re-resolution. No pre-commit hooks -- CI has no working tree to guard.
install-ci:  ## Sync exactly what CI needs, enforcing the lockfile
	$(UV) sync --locked

# --- lint / format ---------------------------------------------------------

lint:  ## Check lint rules (no changes written)
	$(UV) run ruff check --output-format=full $(PY_ALL)
	@echo "Lint clean. Run 'make lint-fix' to auto-fix issues."

lint-fix:  ## Fix what ruff can fix automatically
	$(UV) run ruff check --fix $(PY_ALL)

format-check:  ## Check formatting (no changes written)
	$(UV) run ruff format --check $(PY_ALL)
	@echo "Formatting clean. Run 'make format' to auto-fix issues."

format:  ## Reformat in place
	$(UV) run ruff format $(PY_ALL)

check-all: lint format-check  ## Run every static check CI enforces

check-all-fix: lint-fix format  ## Fix everything fixable, then you re-review

# --- test ------------------------------------------------------------------

# PYTEST_ARGS is a passthrough, so a one-off run does not need a new target:
#   make test PYTEST_ARGS="--durations=25 -k financial"
PYTEST_ARGS ?=

test:  ## Run the full test suite (parallel)
	$(UV) run pytest -n $(PYTEST_WORKERS) $(PYTEST_ARGS)

test-fast:  ## Stop at the first failure, serially (tight debugging loop)
	$(UV) run pytest -x -q

coverage:  ## Run tests with coverage and enforce the floor
	$(UV) run pytest -n $(PYTEST_WORKERS) $(COV_ARGS) \
		--cov-report=term-missing \
		--cov-report=xml \
		--cov-fail-under=$(COV_FAIL_UNDER)

smoke:  ## Offline self-check of config, registries, and report builders
	$(UV) run usersim smoke

dist:  ## Build the sdist, then the wheel from it, into a fresh dist/
	rm -rf dist
	$(UV) build -q -o dist
	@ls -1 dist

# The editable install resolves imports from the source tree and finds assets
# next to them, so `make test` and `make smoke` stay green even when the built
# artifact is unusable. This target checks what a user actually installs: it
# builds both archives, checks them (scripts/check_dist.py), installs the wheel
# into a throwaway environment with no access to this checkout, and runs it
# from an empty directory.
#
# Run it before any release, and after any change to packaging metadata, asset
# locations, or resource loading.
check-dist: dist  ## Build dist/, check both archives, clean-install the wheel and smoke it from an empty dir
	@set -eu; \
	$(UV) run python scripts/check_dist.py dist; \
	WORK=$$(mktemp -d); \
	EMPTY=$$(mktemp -d); \
	trap 'rm -rf "$$WORK" "$$EMPTY"' EXIT; \
	echo "→ installing the wheel into a throwaway venv"; \
	$(UV) venv "$$WORK/venv" >/dev/null; \
	test -x "$$WORK/venv/bin/python" || { echo "venv was not created"; exit 1; }; \
	$(UV) pip install --quiet --python "$$WORK/venv/bin/python" dist/*.whl; \
	test -x "$$WORK/venv/bin/usersim" \
		|| { echo "   ✗ the wheel installed but shipped no usersim entry point"; exit 1; }; \
	echo "→ running from $$EMPTY (no repo in sight)"; \
	cd "$$EMPTY" && "$$WORK/venv/bin/usersim" smoke; \
	cd "$$EMPTY" && "$$WORK/venv/bin/usersim" simulate --dry-run \
		--locale en_US --num-rows 1 --out ./out >/dev/null; \
	echo "✓ installed package runs from a clean environment"

check-dependency-licenses:  ## Check runtime dependency licenses against the policy
	@$(UV) run --isolated --no-dev --group license-check --locked \
		python scripts/check_dependency_licenses.py

check-license-headers:  ## Verify every source file carries the SPDX header
	@$(UV) run python scripts/update_license_headers.py --check

update-license-headers:  ## Add or refresh the SPDX header on every source file
	@$(UV) run python scripts/update_license_headers.py

clean-notebooks:  ## Strip outputs from every notebook (source only)
	@$(UV) run --with nbstripout nbstripout $$(git ls-files '*.ipynb' | grep -v '^_')
	@echo "✓ notebook outputs stripped"

# The pre-commit hook that strips outputs only exists if someone ran
# `make install-dev`; a fresh clone has none. This is what actually keeps
# outputs, and the absolute paths and endpoint URLs inside them, out of the
# repository.
check-notebooks-clean:  ## Fail if any committed notebook carries outputs
	@$(UV) run python -c "$$CHECK_NOTEBOOKS_PY"

define CHECK_NOTEBOOKS_PY
import json, subprocess, sys
files = subprocess.run(
    ["git", "ls-files", "*.ipynb"], capture_output=True, text=True, check=True
).stdout.split()
dirty = []
for path in files:
    nb = json.load(open(path))
    n = sum(len(c.get("outputs", [])) for c in nb["cells"])
    n += sum(1 for c in nb["cells"] if c.get("execution_count"))
    if n:
        dirty.append(f"{path} ({n} output/execution entries)")
if dirty:
    print("Committed notebooks carry outputs:", *dirty, sep="\n  ")
    print("\nRun `make clean-notebooks` and commit the result.")
    sys.exit(1)
print(f"OK {len(files)} notebook(s) carry no outputs")
endef
export CHECK_NOTEBOOKS_PY

# `uv run jupyter lab` already sees the kernel inside .venv. This registers a
# NAMED one so editors that pick kernels system-wide can find this project's
# environment rather than offering a bare "python3" per checkout.
install-kernel:  ## Register a Jupyter kernel named for this project's venv
	@$(UV) run python -m ipykernel install --user \
		--name usersim --display-name "User Sim (.venv)"

check-doc-links:  ## Fail on broken relative links in tracked markdown
	@$(UV) run python scripts/check_doc_links.py

# --- live checks (need credentials; never run in PR CI) --------------------

# The offline suite proves the code is self-consistent. This proves a real
# provider accepts what we send it, which is a different question: the
# parameters, the provider routing and the engine's token budget are only
# exercised by an actual run. One row, two turns, so the cost is negligible.
PROVIDER_CONFIG ?= src/usersim/cli/models_default.toml
HEALTH_LOCALE   ?= ja_JP
HEALTH_PROBE    ?= sov_ai_dynamic

# A provider you have no key for is skipped rather than failed. Without this
# the target is red for everyone who has one key, and the scheduled workflow
# is red until every secret is provisioned, which trains people to ignore it.
define REQUIRE_KEYS_PY
import sys
from usersim.cli._models import load_models_config, required_api_key_env_vars
config = sys.argv[1]
missing = sorted(v for v in required_api_key_env_vars(load_models_config(config)) if not __import__("os").environ.get(v))
if missing:
    print(f"SKIP  {config}: {', '.join(missing)} not set")
    sys.exit(1)
endef
export REQUIRE_KEYS_PY

health-check:  ## Live one-row simulate + eval + report against a provider
	@set -eu; \
	if ! $(UV) run python -c "$$REQUIRE_KEYS_PY" "$(PROVIDER_CONFIG)"; then exit 0; fi; \
	OUT=$$(mktemp -d); \
	trap 'rm -rf "$$OUT"' EXIT; \
	echo "==> simulate  ($(HEALTH_LOCALE), $(PROVIDER_CONFIG))"; \
	$(UV) run usersim simulate \
		--locale $(HEALTH_LOCALE) --num-rows 1 --max-turns 2 \
		--probe-mix "$(HEALTH_PROBE)=1.0" \
		--models $(PROVIDER_CONFIG) \
		--out "$$OUT/trajectories"; \
	RUN=$$(basename $$(ls -d "$$OUT"/trajectories/run=* | head -1) | cut -d= -f2); \
	echo "==> eval      (run $$RUN)"; \
	$(UV) run usersim eval --run "$$RUN" \
		--trajectories "$$OUT/trajectories" \
		--models $(PROVIDER_CONFIG) \
		--out "$$OUT/evaluations"; \
	echo "==> report    (run $$RUN)"; \
	$(UV) run usersim report --run "$$RUN" \
		--trajectories "$$OUT/trajectories" \
		--evaluations "$$OUT/evaluations" \
		--out "$$OUT/report"; \
	test -f "$$OUT/report/run=$$RUN/index.html" \
		|| { echo "report produced no index.html"; exit 1; }; \
	echo "✓ simulate, eval and report all completed"

check-extensions:  ## Install an out-of-tree package and prove all six seams work
	@set -eu; \
	WORK=$$(mktemp -d); \
	EMPTY=$$(mktemp -d); \
	trap 'rm -rf "$$WORK" "$$EMPTY"' EXIT; \
	echo "→ building wheels into $$WORK"; \
	$(UV) build --wheel -o "$$WORK" >/dev/null; \
	echo "→ installing them plus the fixture extension into a throwaway venv"; \
	$(UV) venv "$$WORK/venv" >/dev/null; \
	test -x "$$WORK/venv/bin/python" || { echo "venv was not created"; exit 1; }; \
	$(UV) pip install --quiet --python "$$WORK/venv/bin/python" "$$WORK"/*.whl; \
	cp -R tests/fixtures/extension_package "$$WORK/ext"; \
	$(UV) pip install --quiet --python "$$WORK/venv/bin/python" "$$WORK/ext"; \
	echo "→ asking the installed CLI what it can see, from $$EMPTY"; \
	cp tests/fixtures/extension_package/verify_seams.py "$$EMPTY/"; \
	cd "$$EMPTY" && "$$WORK/venv/bin/python" verify_seams.py

# --- housekeeping ----------------------------------------------------------

clean:  ## Remove caches, coverage output, compiled artifacts, build output and package metadata
	find . -type d -name __pycache__ -not -path './.venv*' -exec rm -rf {} + 2>/dev/null || true
	find . -type f -name '*.py[co]' -not -path './.venv*' -delete 2>/dev/null || true
	rm -rf .pytest_cache .ruff_cache .coverage coverage.xml htmlcov build dist src/*.egg-info
	@echo "Cleaned."