binaryaaron · Oct 15, 2026
@@ -111,3 +111,25 @@ jobs:
 
       - name: An out-of-tree package reaches every extension seam
         run: make check-extensions
+
+  e2e:
+    name: Installed CLI e2e
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
+        with:
+          version-file: "pyproject.toml"
+          python-version: "3.12"
+          enable-cache: true
+      - run: make install-ci
+      - run: make test-e2e E2E_ARGS="--artifacts test-results/e2e"
+      - name: E2e diagnostics on failure
+        if: failure()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: e2e-failure
+          path: test-results/e2e/
+          if-no-files-found: warn
+          retention-days: 14
@@ -118,7 +118,8 @@ making a single model call.
 
 ### Reproducing CI and coverage
 
-Run `make validate-ci` for the native checks and clean distribution validation.
+Run `make validate-ci` for native checks, clean distribution validation, and
+installed CLI e2e tests against that same validated wheel.
 `make validate-native` runs static checks, dependency license policy, coverage,
 the offline self-check, and installed extension checks without building the
 distribution archives. CI runs the same test suite across supported Python
@@ -130,6 +131,11 @@ example `make coverage PYTEST_ARGS="--junitxml=junit.xml"`. See
 [Local and CI validation](docs/engineering/ci.md) for bundle membership, report
 artifacts, and failure diagnosis.
 
+Run `make test-e2e` for the isolated CLI workflow alone, or
+`make test-e2e WHEEL_DIR=dist` to use the single existing wheel in `dist/`
+without rebuilding. See [Installed CLI e2e tests](docs/engineering/e2e.md) for
+setup, offline execution, scenarios, and diagnostics.
+
 ### Choosing models and providers
 
 Three providers are built in, each needing only a credential. Pick the shipped

@@ -1,3 +1,4 @@
 # The test suite runs from a checkout. setuptools would otherwise ship part of
 # it in the sdist, without the conftest.py files it needs to run.
 prune tests
+prune tests_e2e
@@ -26,7 +26,8 @@ UV_VERSION := $(shell sed -n 's/^required-version = "==\([^"]*\)"/\1/p' pyprojec
 # change would trigger a second repo-wide reformat.
 PY_SRC   ?= src scripts templates
 PY_TESTS ?= tests conftest.py
-PY_ALL   := $(PY_SRC) $(PY_TESTS)
+PY_E2E   ?= tests_e2e
+PY_ALL   := $(PY_SRC) $(PY_TESTS) $(PY_E2E)
 
 # Worker count for the parallel suite. `auto` follows the local core count,
 # which suits a CPU-bound suite with every LLM call mocked (~2x on a laptop).
@@ -60,7 +61,7 @@ COV_FAIL_UNDER ?= 84
 .DEFAULT_GOAL := help
 
 .PHONY: help bootstrap check-tools check-lock install-dev install-ci install-kernel health-check check-notebooks-clean lint lint-fix format format-check check-all \
-        check-hooks check-all-fix validate-native validate-ci test test-fast coverage smoke dist check-dist check-release-tag check-extensions check-doc-links clean-notebooks check-license-headers update-license-headers check-dependency-licenses clean
+        check-hooks check-all-fix validate-native validate-ci test test-fast test-e2e coverage smoke dist check-dist check-release-tag check-extensions check-doc-links clean-notebooks check-license-headers update-license-headers check-dependency-licenses clean
 
 
 help:  ## Show the available targets
@@ -128,6 +129,9 @@ check-all-fix: lint-fix format  ## Fix everything fixable, then you re-review
 
 # PYTEST_ARGS is a passthrough, so a one-off run does not need a new target:
 #   make test PYTEST_ARGS="--durations=25 -k financial"
+E2E_ARGS ?=
+WHEEL_DIR ?=
+
 PYTEST_ARGS ?=
 
 test:  ## Run the full test suite (parallel)
@@ -142,9 +146,15 @@ coverage:  ## Run tests with coverage and enforce the floor
 		--cov-report="xml:$(COV_XML)" \
 		--cov-fail-under=$(COV_FAIL_UNDER) $(PYTEST_ARGS)
 
+# Keep the installed suite separate: ordinary pytest's root conftest blocks
+# loopback, and the e2e runner must never import the editable source tree.
+test-e2e: check-tools  ## Test an installed wheel through CLI subprocesses and a local HTTP provider
+	$(UV) run --frozen python scripts/run_e2e.py --uv "$(UV)" $(if $(WHEEL_DIR),--wheel-dir "$(WHEEL_DIR)") $(E2E_ARGS)
+
 validate-native: check-all check-dependency-licenses coverage smoke check-extensions  ## Check source and installed extension behavior
 
-validate-ci: validate-native check-dist  ## Run the native CI checks, including distribution validation
+validate-ci: validate-native check-dist  ## Validate source, then test the same clean distribution artifacts
+	$(MAKE) test-e2e WHEEL_DIR=dist
 
 smoke:  ## Offline self-check of config, registries, and report builders
 	$(UV) run usersim smoke

@@ -2,7 +2,7 @@
 
 CI calls repository Make targets so failures can be reproduced locally. After
 `make install-dev`, run `make validate-ci` to execute the native validation
-bundle, including distribution checks.
+bundle, including distribution checks and installed CLI tests.
 
 | Target | Checks |
 | --- | --- |
@@ -13,13 +13,20 @@ bundle, including distribution checks.
 | `make check-extensions` | Out-of-tree installed extension behavior. |
 | `make validate-native` | The five targets above. |
 | `make check-dist` | Build and validate archives, then check a clean wheel installation. |
-| `make validate-ci` | `validate-native` plus `check-dist`. |
+| `make test-e2e` | Installed CLI workflow against a loopback provider in isolated environments. |
+| `make validate-ci` | `validate-native` and `check-dist`, followed by e2e against the resulting wheel. |
 
 The distribution archives are produced by `check-dist`; extension checks build
 a separate temporary wheel. CI also runs the test suite across Python 3.11
 through 3.14; a local bundle checks the active
 Python environment. Provider health checks run separately and need credentials.
 
+The separate `Installed CLI e2e` job runs the [installed workflow](e2e.md) on
+Python 3.12 through `make test-e2e`. It retains failure diagnostics under
+`test-results/e2e/` as `e2e-failure` for 14 days. The local bundle passes
+`WHEEL_DIR=dist` after archive validation so the e2e test installs that same
+wheel without rebuilding.
+
 ## Coverage
 
 The Python 3.12 CI matrix job runs `make coverage`; the other Python versions

@@ -0,0 +1,67 @@
+# Installed CLI end-to-end tests
+
+Run `make test-e2e` to test a built UserSim wheel through its CLI and real HTTP client. The harness uses synthetic stored inputs and a deterministic OpenAI-compatible provider on loopback; it requires no provider credentials or downloaded persona datasets.
+
+Setup needs uv, a supported Python, and access to dependency indexes and the tokenizer data URL. The repository's frozen lock supplies runtime dependencies. The test harness installs its own pinned pytest and PyArrow into a second environment. No development dependencies or editable UserSim install enter the runtime environment.
+
+```bash
+make test-e2e
+```
+
+The equivalent direct command is:
+
+```bash
+uv run --frozen python scripts/run_e2e.py
+```
+
+Pass an existing wheel to test the original release artifact without rebuilding:
+
+```bash
+uv run --frozen python scripts/run_e2e.py --wheel dist/usersim-VERSION-py3-none-any.whl
+# Or require exactly one wheel in the release directory:
+uv run --frozen python scripts/run_e2e.py --wheel-dir dist
+```
+
+`--wheel` and `--wheel-dir` are mutually exclusive and cannot be combined with `--run-only`. An empty or ambiguous wheel directory fails setup. `--uv /path/to/uv` selects the setup executable when it is not the one on `PATH`.
+
+## Isolation and offline execution
+
+The runner creates a directory outside the checkout containing `runtime/`, `harness/`, a copied test suite, a fresh home directory, and a tokenizer cache. It verifies that UserSim imports from `runtime/`. After admission, the runner writes empty `.env` and `.env.local` files in its owned work directory on every invocation, including `--run-only`. Admission checks these files and the direct preparation, cache, environment, and readiness entries before any output changes: symlinks, wrong file/directory kinds, and multiply linked writable files are rejected. Normal interpreter symlinks inside a prepared virtualenv remain supported. These stop the CLI's upward dotenv search before it reaches files outside the harness. CLI subprocesses run in fresh case directories beneath that boundary outside the source tree; the tests never import UserSim from the checkout or patch its probes, scorers, storage, or reporting.
+
+The runtime receives a small environment allowlist and a synthetic local API key. External sockets are rejected by a test-only `sitecustomize` guard; loopback remains available to the fake provider. Hugging Face offline settings, extension-discovery suppression, and `NEMO_TELEMETRY_ENABLED=false` and other telemetry opt-outs are explicit. A subprocess verifies that installed DataDesigner telemetry is disabled. The ordinary suite's network guard and root conftest are not loaded.
+
+Prepare environments while network access is available, then execute them without installation or downloads:
+
+```bash
+uv run --frozen python scripts/run_e2e.py --work-dir /tmp/usersim-e2e --prepare-only
+uv run --frozen python scripts/run_e2e.py --work-dir /tmp/usersim-e2e --run-only
+```
+
+`--work-dir` must resolve outside the checkout and cannot be an ancestor of it. Paths containing `..` use their canonical locations for this check; symlinked work and artifact paths are rejected. Reuse the same path for both phases; the prepared environment and copied suite are a snapshot of setup. Changes to tests or wheels require another preparation. A new preparation removes the previous `installed.json` receipt before changing the environment, so an interrupted setup cannot advertise the old wheel as a valid prepared runtime.
+
+## Scenarios and assertions
+
+| Scenario | Contract checked |
+| --- | --- |
+| Complete workflow | `simulate → eval → report → select` preserves identities across Parquet reload, obtains real judge scores, writes report HTML, and selects the scored rows. |
+| Stored-input resume | A second simulation reuses its run and trajectory IDs, preserves `input_id`, writes no duplicate rows, and makes no provider requests. |
+| Tool calling | A weather function call travels through the real HTTP client, tool execution, and persisted transcript. A transient server error is retried. |
+| Non-Latin locale | A Japanese input keeps its locale and native-script user text through persistence and evaluation. |
+| Guarded tool request | A three-turn health-disclosure scenario forwards explicit `tool_choice`, preserves configured `extra_body`, and merges per-call nested reasoning settings. Its shipped synthetic clinical profile emits the expected placeholder warning. |
+| Malformed response | Invalid evaluator JSON produces a nonzero CLI exit and empty judge results rather than successful scores. |
+| Failed request | A provider HTTP 400 causes a nonzero CLI exit without published evaluation rows. |
+| Network boundary | An attempted external connection fails with the harness's explicit network error. |
+
+These cases verify client integration and artifact contracts. They do not measure model quality or establish acceptance by real providers. The health-disclosure case uses the shipped placeholder bank to exercise request plumbing and does not establish clinical validity.
+
+## Results and diagnosis
+
+Each execution exclusively creates `output/e2e/run-*/` and prints its exact path. Use `--artifacts ROOT` to select another output root; existing attempts and unrelated root files are left alone. A root on a bind mount works because only a new child is created, rather than renaming the mount. The runner prints its external work directory and copies scenario logs, synthetic inputs, recorded provider request bodies, trajectory/evaluation/selection Parquet, and report files into the artifact directory. `junit.xml` records the test result. The exported `installed.json` receipt retains the runtime import and asset locations and adds the selected wheel path and `wheel_sha256` captured during preparation. The digest binds prepared/run-only evidence to the wheel bytes, even if the original path later names a different file. Existing prepared environments without the digest need preparation again to provide this evidence. `installed.json` is the sole installation receipt. Captured requests contain only synthetic data and the local test model names.
+
+Rejected admission leaves the work and output roots unchanged. Admission checks work/output paths, owned work entries, run-only readiness, and selected wheel inputs that could be removed by work-directory preparation. Safe paths containing `..` are canonicalized after lexical symlink checks. Once admitted, a new attempt directory is created before setup or execution, so even an early failure cannot mix previous scores, JUnit, or receipts into the attempt. The output root can contain earlier results or other caller-owned files; no marker, rotation, or automatic cleanup is required. Remove result directories when you no longer need them.
+
+`--prepare-only` changes preparation state in the external work directory and creates no result directory. Successful preparation writes `installed.json`; an interrupted preparation removes readiness, so `--run-only` refuses it. Execution copies that receipt into its fresh result directory alongside JUnit and scenario files. CI uploads the output root recursively, including each attempt's diagnostics.
+
+Read a failed command's `<scenario>/<command>.log`, then compare `<scenario>/requests.json` with the expected payload. The work directory retains both environments and uncopied DataDesigner diagnostics for inspection. Setup commands, CLI subprocesses, provider readiness, and the complete pytest process have timeouts. An empty test collection fails.
+
+The runner does not remove the external work directory. Remove it after diagnosis to reclaim the installed environments and tokenizer cache.
@@ -9,11 +9,11 @@ tracked file or prevent setuptools from packaging it.
 | --- | --- | --- |
 | Git | `.gitignore` | Python bytecode, environments, editor state, caches, coverage/JUnit reports, build output, notebook checkpoints, debug transcripts, credentials, private model/spec layers, local checkout/worktree directories, and private agent overrides stay local. Root `data/`, `artifacts/`, and `output/` hold generated run data. Financial-services embedding vectors are model-specific local outputs. |
 | Hooks | `.pre-commit-config.yaml` | Top-level underscore directories hold private material outside the distribution; generated `uv.lock` is formatted by uv. Notebook JSON skips text rewriting hooks but remains subject to output stripping and file-size checks. The size gate checks all files, with exact exceptions for the two existing safety_chat_pressure and safety_agentic sample YAML corpora above 600 KB. They remain subject to syntax and conflict checks. Ruff uses the locked development version. |
-| Lint | `Makefile`, `pyproject.toml` | Source, gate scripts, templates, tests, and root conftest are checked. Notebooks and run-output directories are outside Ruff scope; notebook outputs have a separate gate. |
+| Lint | `Makefile`, `pyproject.toml` | Source, gate scripts, templates, ordinary and installed e2e tests, and root conftest are checked. Notebooks and run-output directories are outside Ruff scope; notebook outputs have a separate gate. |
 | SPDX | `scripts/update_license_headers.py` | Python, shell, and template files under the declared source roots, plus Makefile and conftest, require headers. Cache/environment directories and private top-level underscore directories are skipped. Markdown, YAML, TOML, and binary assets are outside this header policy. |
 | Documentation | `scripts/check_doc_links.py` | Tracked Markdown is checked, except private top-level underscore paths. Local links and repository main-branch links resolve against the checkout; external URLs and anchors are not fetched or validated. |
 | Coverage | `pyproject.toml`, `Makefile` | Runtime package coverage excludes explicit `pragma: no cover` lines, type-only branches, and abstract `NotImplementedError` branches. These are reporting exclusions, not Git rules. No type-check exclusions are configured yet. |
-| Packaging | `MANIFEST.in`, setuptools configuration in `pyproject.toml` | Tests are omitted from the sdist. Only `usersim*` packages are discovered; typed markers, shipped model TOML, and required asset banks are included. Generated embedding vectors are explicitly excluded. `make check-dist` inspects both archives and runs an installed wheel. |
+| Packaging | `MANIFEST.in`, setuptools configuration in `pyproject.toml` | Both `tests/` and `tests_e2e/` are omitted from the sdist. Only `usersim*` packages are discovered; typed markers, shipped model TOML, and required asset banks are included. Generated embedding vectors are explicitly excluded. `make check-dist` inspects both archives and runs an installed wheel. |
 | Container context | Not yet configured | No test container or `.dockerignore` exists at this stage. Container work must explicitly exclude credentials, host environments, caches, private files, and Git metadata while including required fixtures and assets, then verify the actual build context. |
 
 `.gitattributes` is not an ignore file. Its uv.lock attributes collapse generated