From 7e78334a0956c5abf71205119ad08daf7e2e5bcf Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Fri, 16 Oct 2026 14:41:50 +0000 Subject: [PATCH] build: prepare PR06 with reviewed workflow simplifications Signed-off-by: Aaron Gonzales --- .dockerignore | 51 ++++ .github/CODEOWNERS | 4 + .github/workflows/ci.yml | 19 ++ CONTRIBUTING.md | 7 + Makefile | 12 + containers/Dockerfile.test | 22 ++ docs/engineering/container-tests.md | 98 ++++++++ docs/engineering/e2e.md | 2 +- scripts/run_container_tests.py | 293 +++++++++++++++++++++++ tests/fixtures/exclusion-policy.json | 40 +++- tests/test_container_runner.py | 341 +++++++++++++++++++++++++++ 11 files changed, 878 insertions(+), 11 deletions(-) create mode 100644 .dockerignore create mode 100644 containers/Dockerfile.test create mode 100644 docs/engineering/container-tests.md create mode 100644 scripts/run_container_tests.py create mode 100644 tests/test_container_runner.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..a2dfc6b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,51 @@ +# Preserve source, tests, templates, .github and docs; exclude host state. +.git +**/.git +.venv +**/.venv +venv +env +.env +.env.* +**/.env +**/.env.* +**/*.local.toml +**/*.local.yaml +**/__pycache__ +**/*.py[cod] +**/*.egg-info +**/.pytest_cache +**/.ruff_cache +**/.cache +**/.coverage* +**/coverage.xml +**/htmlcov +**/.ipynb_checkpoints +dist +build +artifacts +test-results +output +data +.dd-branch +.cursor +.claude +.codex +.vscode +.idea +.agent-work/ +**/debug_llm_responses.jsonl +**/debug_litellm_requests.jsonl +**/*.pem +**/*.key +.DS_Store + +# Private top-level work areas and agent overrides. +_*/ +.worktrees/ +**/AGENTS.local.md +**/CLAUDE.local.md + +# Generated model-specific vectors; retain the source asset banks. +src/usersim/engine/assets/financial_services/**/embeddings.parquet +**/junit.xml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 0cf184d..b8c7c82 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -27,3 +27,7 @@ /tests/test_update_license_headers.py @NVIDIA-NeMo/user_sim_maintainers /docs/engineering/ @NVIDIA-NeMo/user_sim_maintainers /tests/fixtures/exclusion-policy.json @NVIDIA-NeMo/user_sim_maintainers +/containers/ @NVIDIA-NeMo/user_sim_maintainers +/.dockerignore @NVIDIA-NeMo/user_sim_maintainers +/tests/test_container_runner.py @NVIDIA-NeMo/user_sim_maintainers +/tests/test_coverage_gate.py @NVIDIA-NeMo/user_sim_maintainers diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a1ce9ca..0e4823e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -133,3 +133,22 @@ jobs: path: test-results/e2e/ if-no-files-found: warn retention-days: 14 + + container: + name: CPU container tests + runs-on: ubuntu-latest + timeout-minutes: 65 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Build and run isolated CPU validation + run: make test-container + + - name: Container test diagnostics + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: container-test-results + path: test-results/container/ + include-hidden-files: true + if-no-files-found: warn diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 24975f8..2ed9802 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -390,3 +390,10 @@ Participation is governed by the [Contributor Covenant](CODE_OF_CONDUCT.md). See [Live provider health checks](docs/engineering/health-checks.md) for credential precedence, local commands, and the pass, skip, and failure outcomes. These checks make real model requests and are separate from offline contributor validation. + +## Container validation + +Run `make test-container` to build a fresh CPU test image and run coverage plus +installed-wheel e2e without external networking. See +[CPU container tests](docs/engineering/container-tests.md) for Docker prerequisites, +resource limits, context policy, and exported diagnostics. diff --git a/Makefile b/Makefile index 34ef38a..5eda56e 100644 --- a/Makefile +++ b/Makefile @@ -296,3 +296,15 @@ clean: ## Remove caches, coverage output, compiled artifacts, build output and find . -type f -name '*.py[co]' -not -path './.venv*' -delete 2>/dev/null || true rm -rf .pytest_cache .ruff_cache .coverage coverage.xml htmlcov build dist src/*.egg-info @echo "Cleaned." + +# Container validation uses only host Python and Docker; dependencies live in the image. +CONTAINER_ARGS ?= +.PHONY: container-audit container-build-test test-container +container-build-test: ## Build the CPU test image with locked dependencies + python3 scripts/run_container_tests.py build $(CONTAINER_ARGS) + +test-container: container-build-test ## Build fresh and run offline CPU coverage and e2e + python3 scripts/run_container_tests.py run $(CONTAINER_ARGS) + +container-audit: ## Audit the local Docker context before image build + python3 scripts/run_container_tests.py audit $(CONTAINER_ARGS) diff --git a/containers/Dockerfile.test b/containers/Dockerfile.test new file mode 100644 index 0000000..44cf06d --- /dev/null +++ b/containers/Dockerfile.test @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Multi-platform manifest verified with docker buildx imagetools inspect. +FROM python:3.12-slim-bookworm@sha256:34386ef0cb081344d7ec1c103ba398e6e9f64e9ab3a1509accc92a4e24a07258 AS tests +RUN apt-get update && apt-get install -y --no-install-recommends git make \ + && python -c 'import shutil; shutil.rmtree("/var/lib/apt/lists")' +WORKDIR /workspace +COPY . . +# pyproject.toml owns the uv version; no second pin belongs here. +RUN python -c 'import tomllib, subprocess; p=tomllib.load(open("pyproject.toml", "rb")); v=p["tool"]["uv"]["required-version"]; assert v.startswith("=="), "container requires an exact uv declaration"; subprocess.run(["python", "-m", "pip", "install", "--no-cache-dir", "uv"+v], check=True)' +ENV UV_PROJECT_ENVIRONMENT=/opt/unit UV_PYTHON_DOWNLOADS=never \ + DO_NOT_TRACK=1 HF_HUB_DISABLE_TELEMETRY=1 LITELLM_TELEMETRY=False +RUN uv sync --locked && uv build --wheel --out-dir /opt/wheels \ + && python scripts/run_e2e.py --work-dir /opt/e2e --wheel /opt/wheels/*.whl --prepare-only \ + && chmod -R a+rwX /workspace /opt/e2e +ENV UV_OFFLINE=1 UV_NO_SYNC=1 UV_CACHE_DIR=/tmp/uv-cache HOME=/tmp/home +CMD ["python", "scripts/run_container_tests.py", "inside"] + +# Export this stage to inspect exactly the context used by COPY above. +FROM scratch AS context-audit +COPY . / diff --git a/docs/engineering/container-tests.md b/docs/engineering/container-tests.md new file mode 100644 index 0000000..040cc44 --- /dev/null +++ b/docs/engineering/container-tests.md @@ -0,0 +1,98 @@ +# CPU container tests + +The test image prepares a source unit-test environment and a separate installed-wheel +e2e environment during build. Execution uses the same `make coverage` and +`make test-e2e` commands as native validation, with external networking disabled. + +Install Docker with a running daemon. Build dependencies require internet access; +test execution needs no provider credentials. The image uses Python 3.12 pinned by +manifest digest and reads the exact uv version from `pyproject.toml`. + +```bash +make container-build-test +make test-container +``` + +Artifacts appear in fresh `test-results/container/run-*/` directories: build and container logs, +`unit.log`, `unit.xml`, `coverage.xml`, `.coverage`, `cleanup.log`, and the e2e +artifact directory. +A failing suite or timeout returns nonzero. Both suites run so one failure does +not hide the other suite's diagnostics. The host runner allows one hour for image build and another hour for execution; +each suite allows 25 minutes. Runtime limits are two CPUs, two pytest workers, +8 GiB memory with no additional swap, and 512 processes. Temporary storage is +limited to 1 GiB and permits executable test fixtures. Tests run with the caller's UID/GID so exported files remain +owned by the caller. + +Only the artifact directory is mounted from the host. The image includes source, +test fixtures, documentation, templates, and workflow files that unit tests read. +`.dockerignore` excludes Git metadata, host environments, private endpoint +configurations, credentials, editor/agent settings, caches and generated output. +Git is installed for tests that create their own temporary repositories. +The e2e runner operates outside `/workspace` against its installed wheel and +prepares its tokenizer cache during build. Docker's disabled external network +still permits the fake provider to use container loopback. + +Every image build first runs a required context audit. It exports a synthetic +scratch context with Docker's native ignore semantics and checks every shared +example in `tests/fixtures/exclusion-policy.json`. Generated financial-services +`embeddings.parquet` sidecars stay outside the context; source asset banks remain. +Run this bounded check separately with: + +```bash +make container-audit +``` + +The audit, image build, execution, and cleanup require `/var/run/docker.sock` +to be a Unix socket. +They reject nonempty inherited `DOCKER_*`, `BUILDX_*`, and `BUILDKIT_*` selectors +before invoking Docker, including host, context, client config, TLS, and builder +settings. Unset these variables explicitly when using this local validation +workflow. Each operation uses a temporary empty client config, explicit +`--host unix:///var/run/docker.sock`; image builds use the `default` builder +with the `docker` driver. Execution and cleanup share the same admitted client +configuration and environment, keeping the temporary config alive until cleanup +finishes. Stored default or remote contexts cannot select a different daemon +for running the image. A failed builder check, missing daemon, failed export, or +incorrect canary result returns nonzero. No missing prerequisite becomes a successful skip. +The scratch audit pulls no image and supplies `--network=none`. + +Artifacts include `audit-settings.json`, `audit-builder.log`, `audit-export.log`, +and `audit-result.json`; the receipt records every example and its result. +Temporary client config and synthetic context files are removed afterward. +Ordinary pytest mocks Docker orchestration and never invokes a Docker executable. +Actual daemon access belongs to this explicit audit and container validation. + +Each `audit`, `build`, or `run` invocation exclusively creates a `run-*` directory +beneath `--artifacts ROOT` and prints its exact path before starting. A `build` +attempt contains its audit and build output; an execution attempt contains the +container, cleanup, unit, and e2e logs. Earlier attempt directories and unrelated +root files are untouched. There are no ownership markers or history migrations; +remove result directories when you no longer need them. + +The host mounts only the new execution directory at `/artifacts`. The `inside` +action writes directly into that already-owned directory, without another +container-level directory or cleanup step. Installed e2e creates its own fresh +attempt below `/artifacts/e2e`. Preparation remains in `/opt/e2e`, separate from +results. Failed attempts retain their partial diagnostics without exposing a +previous attempt's passing audit, coverage, JUnit, or scores as current. CI can +continue uploading the selected output root recursively. + +`cleanup.log` records the container name, cleanup status, exit code, and captured +output, including partial timeout output. Successful removal and the exact +already-removed response for that container preserve the primary result. +Other removal errors require recovery and return nonzero if the primary run +succeeded. A primary test failure or timeout remains the returned status even +when cleanup fails. The diagnostic identifies the container and local endpoint +for inspection and manual recovery; cleanup never falls back to another daemon. + +For direct runner overrides: + +```bash +python scripts/run_container_tests.py build --image usersim-tests:local +python scripts/run_container_tests.py run --image usersim-tests:local --artifacts /tmp/usersim-test-artifacts --timeout 3600 +``` + +After changing the base image, resolve and review its public manifest digest, +update the Dockerfile, and rebuild the image before running validation. Changes +to source or fixtures also require rebuilding; execution never mounts the host +checkout over the image. diff --git a/docs/engineering/e2e.md b/docs/engineering/e2e.md index 0511512..bc66820 100644 --- a/docs/engineering/e2e.md +++ b/docs/engineering/e2e.md @@ -37,7 +37,7 @@ uv run --frozen python scripts/run_e2e.py --work-dir /tmp/usersim-e2e --prepare- uv run --frozen python scripts/run_e2e.py --work-dir /tmp/usersim-e2e --run-only ``` -`--work-dir` must resolve outside the checkout and cannot be an ancestor of it. Paths containing `..` use their canonical locations for this check; symlinked work and artifact paths are rejected. Reuse the same path for both phases; the prepared environment and copied suite are a snapshot of setup. Changes to tests or wheels require another preparation. A new preparation removes the previous `installed.json` receipt before changing the environment, so an interrupted setup cannot advertise the old wheel as a valid prepared runtime. +`--work-dir` must resolve outside the checkout and cannot be an ancestor of it. Paths containing `..` use their canonical locations for this check; symlinked work and artifact paths are rejected. Reuse the same path for both phases; the prepared environment and copied suite are a snapshot of setup. Changes to tests or wheels require another preparation. A new preparation removes the previous `installed.json` receipt before changing the environment, so an interrupted setup cannot advertise the old wheel as a valid prepared runtime. The container prepares these environments at image build time and runs the second command with external networking disabled. ## Scenarios and assertions diff --git a/scripts/run_container_tests.py b/scripts/run_container_tests.py new file mode 100644 index 0000000..649fa1c --- /dev/null +++ b/scripts/run_container_tests.py @@ -0,0 +1,293 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Build and run CPU validation without exposing the host checkout or credentials.""" + +from __future__ import annotations + +import argparse +import json +import os +import shlex +import shutil +import signal +import subprocess +import sys +import tempfile +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def run_logged( + command: list[str], log: Path, *, timeout: int, cwd: Path = ROOT, env: dict[str, str] | None = None +) -> int: + """Preserve diagnostics even when a command fails or times out.""" + print("Running:", " ".join(command), flush=True) + with log.open("w") as stream: + with subprocess.Popen( + command, cwd=cwd, env=env, stdout=stream, stderr=subprocess.STDOUT, start_new_session=True + ) as process: + try: + code = process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + # Stop descendants as well as Make; leave no timed-out suite running. + os.killpg(process.pid, signal.SIGKILL) + process.wait() + print(f"Timed out after {timeout}s; see {log}", file=sys.stderr) + return 124 + print(f"Exit {code}; log: {log}", flush=True) + return code + + +def attempt_directory(root: Path) -> Path: + if any(part.is_symlink() for part in (root, *root.parents)): + raise ValueError("container artifact root must not use symlinks") + root = root.resolve() + if root == ROOT or root in ROOT.parents: + raise ValueError("container artifact root must not overlap the checkout root or its ancestors") + root.mkdir(parents=True, exist_ok=True) + artifacts = Path(tempfile.mkdtemp(prefix="run-", dir=root)) + print(f"Container attempt artifacts: {artifacts}", flush=True) + return artifacts + + +def cleanup_container(client: list[str], environment: dict[str, str], name: str, log: Path) -> int: + """Record bounded cleanup diagnostics without replacing the primary exit status.""" + command = client + ["rm", "-f", name] + code, status, stdout, stderr = 1, "failed", "", "" + try: + result = subprocess.run(command, env=environment, capture_output=True, text=True, timeout=30, check=False) + code, stdout, stderr = result.returncode, result.stdout, result.stderr + if ( + code in {0, 1} + and not stdout.strip() + and stderr.strip() == f"Error response from daemon: No such container: {name}" + ): + status = "already-absent" + elif code == 0 and not stderr.strip(): + status = "removed" + except subprocess.TimeoutExpired as error: + code, status = 124, "timed-out" + stdout, stderr = error.stdout or "", error.stderr or "" + if isinstance(stdout, bytes): + stdout = stdout.decode(errors="replace") + if isinstance(stderr, bytes): + stderr = stderr.decode(errors="replace") + stderr += "\nCleanup timed out after 30s" + except OSError as error: + stderr = str(error) + try: + log.write_text( + f"Container: {name}\nStatus: {status}\nExit code: {code}\nstdout:\n{stdout}\nstderr:\n{stderr}\n" + ) + except OSError as error: + print(f"Could not retain cleanup diagnostics in {log}: {error}", file=sys.stderr) + status = "failed" + if status in {"removed", "already-absent"}: + return 0 + print( + f"Cleanup failed for {name} on unix://{LOCAL_SOCKET}; it may still be running. " + f"Manual recovery required; see {log}", + file=sys.stderr, + ) + return 1 + + +def inside(artifacts: Path = Path("/artifacts")) -> int: + """Use the same Make contracts as native validation.""" + artifacts.mkdir(exist_ok=True) + common: dict[str, str] = {**os.environ, "COVERAGE_FILE": str(artifacts / ".coverage")} + unit: dict[str, str] = { + **common, + "PYTEST_WORKERS": "2", + "PYTEST_ARGS": f"--junitxml={artifacts / 'unit.xml'}", + "COV_XML": str(artifacts / "coverage.xml"), + } + unit.pop("E2E_ARGS", None) + unit.pop("WHEEL_DIR", None) + e2e: dict[str, str] = { + **common, + "E2E_ARGS": shlex.join(["--work-dir", "/opt/e2e", "--run-only", "--artifacts", str(artifacts / "e2e")]), + } + codes = [ + run_logged(["make", task], artifacts / f"{name}.log", timeout=1500, env=environment) + for name, task, environment in (("unit", "coverage", unit), ("e2e", "test-e2e", e2e)) + ] + return next((code for code in codes if code), 0) + + +LOCAL_SOCKET = Path("/var/run/docker.sock") + + +def local_client(config: Path) -> tuple[list[str], dict[str, str]]: + """Admit only the local socket before running even a Docker probe.""" + selectors = sorted( + name for name, value in os.environ.items() if value and name.startswith(("DOCKER_", "BUILDX_", "BUILDKIT_")) + ) + if selectors: + raise ValueError("Unset inherited Docker/BuildKit selectors: " + ", ".join(selectors)) + if not LOCAL_SOCKET.is_socket(): + raise ValueError(f"Required local Docker Unix socket unavailable: {LOCAL_SOCKET}") + config.mkdir(exist_ok=True) + environment = { + name: value for name, value in os.environ.items() if not name.startswith(("DOCKER_", "BUILDX_", "BUILDKIT_")) + } + environment["DOCKER_CONFIG"] = str(config) + endpoint = f"unix://{LOCAL_SOCKET}" + return ["docker", "--config", str(config), "--host", endpoint], environment + + +def audit_context(artifacts: Path, *, timeout: int) -> int: + """Export synthetic canaries using Docker's actual exclusion semantics.""" + with tempfile.TemporaryDirectory(prefix="usersim-context-audit-") as temporary: + work = Path(temporary) + client, environment = local_client(work / "docker-client") + settings = { + "endpoint": f"unix://{LOCAL_SOCKET}", + "builder": "default", + "driver": "docker", + "config": str(work / "docker-client"), + "inherited_selectors": "rejected", + } + (artifacts / "audit-settings.json").write_text(json.dumps(settings, indent=2) + "\n") + # With an empty client config and explicit --host, the default docker + # driver has no stored remote context or builder to select. + code = run_logged( + client + ["buildx", "inspect", "default"], + artifacts / "audit-builder.log", + timeout=min(timeout, 30), + env=environment, + ) + if code: + return code + inspection = (artifacts / "audit-builder.log").read_text() + if not any(line.split() == ["Driver:", "docker"] for line in inspection.splitlines()): + raise ValueError("Context audit requires the default docker driver") + context, exported = work / "context", work / "exported" + context.mkdir() + shutil.copyfile(ROOT / ".dockerignore", context / ".dockerignore") + # A scratch-only recipe avoids resolving or pulling the normal base image. + (context / "Dockerfile").write_text("FROM scratch AS context-audit\nCOPY . /\n") + cases = json.loads((ROOT / "tests/fixtures/exclusion-policy.json").read_text()) + if ( + not cases + or not any(case["docker_included"] for case in cases) + or not any(not case["docker_included"] for case in cases) + ): + raise ValueError("Context audit requires positive and negative examples") + for case in cases: + target = context / case["path"] + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("synthetic context canary\n") + code = run_logged( + client + + [ + "build", + "--builder", + "default", + "--network=none", + "--target", + "context-audit", + "--output", + f"type=local,dest={exported}", + str(context), + ], + artifacts / "audit-export.log", + timeout=min(timeout, 90), + env=environment, + ) + if code: + return code + failures = [case["path"] for case in cases if (exported / case["path"]).is_file() != case["docker_included"]] + receipt = {"settings": settings, "checked": cases, "failures": failures} + (artifacts / "audit-result.json").write_text(json.dumps(receipt, indent=2) + "\n") + if failures: + raise ValueError("Docker context policy mismatch: " + ", ".join(failures)) + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("audit", "build", "run", "inside")) + parser.add_argument("--image", default="usersim-tests:local") + parser.add_argument( + "--artifacts", type=Path, default=ROOT / "test-results/container", help="Root for fresh per-attempt output." + ) + parser.add_argument("--timeout", type=int, default=3600) + arguments = list(argv) if argv is not None else sys.argv[1:] + args = parser.parse_args([*arguments, *shlex.split(os.environ.get("CONTAINER_ARGS", ""))]) + if args.action == "inside": + return inside() + if args.timeout <= 0: + parser.error("--timeout must be positive") + artifacts = attempt_directory(args.artifacts.absolute()) + if args.action in ("audit", "build"): + try: + code = audit_context(artifacts, timeout=args.timeout) + if code or args.action == "audit": + return code + with tempfile.TemporaryDirectory(prefix="usersim-docker-client-") as temporary: + client, environment = local_client(Path(temporary) / "config") + return run_logged( + client + + [ + "build", + "--builder", + "default", + "--target", + "tests", + "-f", + "containers/Dockerfile.test", + "-t", + args.image, + ".", + ], + artifacts / "build.log", + timeout=args.timeout, + env=environment, + ) + except (OSError, ValueError) as error: + print(f"Container audit/build failed: {error}", file=sys.stderr) + return 1 + try: + with tempfile.TemporaryDirectory(prefix="usersim-docker-client-") as temporary: + client, environment = local_client(Path(temporary) / "config") + name = f"usersim-tests-{uuid.uuid4().hex[:12]}" + command = client + [ + "run", + "--rm", + "--name", + name, + "--network=none", + "--cpus=2", + "--memory=8g", + "--memory-swap=8g", + "--pids-limit=512", + "--cap-drop=ALL", + "--security-opt=no-new-privileges", + "--user", + f"{os.getuid()}:{os.getgid()}", + "--tmpfs", + "/tmp:rw,exec,nosuid,size=1g", + "--mount", + f"type=bind,source={artifacts},target=/artifacts", + args.image, + ] + primary = 1 + try: + primary = run_logged(command, artifacts / "container.log", timeout=args.timeout, env=environment) + except (OSError, ValueError, subprocess.TimeoutExpired) as error: + print(f"Container run failed for {name}: {error}", file=sys.stderr) + finally: + # A killed docker client can leave its container running after timeout. + cleanup = cleanup_container(client, environment, name, artifacts / "cleanup.log") + return primary or cleanup + except (OSError, ValueError, subprocess.TimeoutExpired) as error: + print(f"Container run/cleanup failed: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/fixtures/exclusion-policy.json b/tests/fixtures/exclusion-policy.json index 97047a8..d646030 100644 --- a/tests/fixtures/exclusion-policy.json +++ b/tests/fixtures/exclusion-policy.json @@ -1,42 +1,62 @@ [ { "path": ".env.local", - "git_ignored": true + "git_ignored": true, + "docker_included": false }, { "path": "models.local.toml", - "git_ignored": true + "git_ignored": true, + "docker_included": false }, { "path": "spec.local.yaml", - "git_ignored": true + "git_ignored": true, + "docker_included": false }, { "path": "output/run=abc/data.parquet", - "git_ignored": true + "git_ignored": true, + "docker_included": false }, { "path": "debug_llm_responses.jsonl", - "git_ignored": true + "git_ignored": true, + "docker_included": false }, { "path": ".agent-work/plans/engineering-workflows.md", - "git_ignored": true + "git_ignored": true, + "docker_included": false }, { "path": "src/usersim/cli/models_default.toml", - "git_ignored": false + "git_ignored": false, + "docker_included": true }, { "path": "src/usersim/engine/assets/example/data/required.json", - "git_ignored": false + "git_ignored": false, + "docker_included": true }, { "path": "tests/fixtures/output/expected.json", - "git_ignored": false + "git_ignored": false, + "docker_included": true + }, + { + "path": "src/usersim/engine/assets/financial_services/example/embeddings.parquet", + "git_ignored": true, + "docker_included": false + }, + { + "path": "src/usersim/engine/assets/financial_services/example/bank.json", + "git_ignored": false, + "docker_included": true }, { "path": "data/generated.json", - "git_ignored": true + "git_ignored": true, + "docker_included": false } ] diff --git a/tests/test_container_runner.py b/tests/test_container_runner.py new file mode 100644 index 0000000..dcd0166 --- /dev/null +++ b/tests/test_container_runner.py @@ -0,0 +1,341 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Container orchestration preserves failures, limits and host isolation.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import pytest + +SPEC = importlib.util.spec_from_file_location( + "container_runner", Path(__file__).parents[1] / "scripts/run_container_tests.py" +) +runner = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(runner) + + +def test_failure_keeps_diagnostics(tmp_path): + log = tmp_path / "failure.log" + assert runner.run_logged([sys.executable, "-c", "print('diagnostic'); raise SystemExit(7)"], log, timeout=5) == 7 + assert "diagnostic" in log.read_text() + + +@pytest.mark.parametrize("status", [0, 9, 124]) +def test_run_limits_mounts_cleanup_and_exit_status(monkeypatch, tmp_path, status): + commands = [] + configurations = [] + + def verify_client(command, environment): + assert command[:2] == ["docker", "--config"] + config = Path(command[2]) + assert config.is_dir(), "client configuration expired before Docker operation" + assert list(config.iterdir()) == [], "stored Docker settings entered isolated config" + assert command[3:5] == ["--host", "unix:///var/run/docker.sock"] + assert environment["DOCKER_CONFIG"] == str(config) + configurations.append(config) + + def fake_logged(command, log, **kwargs): + verify_client(command, kwargs.get("env", {})) + commands.append((command, kwargs["env"])) + return status + + cleanup = [] + + def fake_cleanup(command, **kwargs): + verify_client(command, kwargs.get("env", {})) + cleanup.append((command, kwargs["env"])) + return runner.subprocess.CompletedProcess(command, 0, "removed", "") + + for selector in list(runner.os.environ): + if selector.startswith(("DOCKER_", "BUILDX_", "BUILDKIT_")): + monkeypatch.delenv(selector) + # Stored default settings must not select a different daemon than the build. + home = tmp_path / "home" + (home / ".docker").mkdir(parents=True) + (home / ".docker/config.json").write_text('{"currentContext":"remote-canary"}') + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setattr(runner.Path, "is_socket", lambda self: True) + monkeypatch.setattr(runner, "run_logged", fake_logged) + monkeypatch.setattr(runner.subprocess, "run", fake_cleanup) + assert runner.main(["run", "--artifacts", str(tmp_path)]) == status + command, environment = commands[0] + assert "--network=none" in command + assert "--cpus=2" in command + assert "--memory=8g" in command + assert "--memory-swap=8g" in command + assert "/tmp:rw,exec,nosuid,size=1g" in command + assert command.count("--mount") == 1 + attempt = next(tmp_path.glob("run-*")) + assert f"type=bind,source={attempt},target=/artifacts" in command + assert "--env-file" not in command + assert cleanup == [(command[:5] + ["rm", "-f", command[command.index("--name") + 1]], environment)] + assert cleanup[0][1] is environment + assert configurations[0] == configurations[1] + assert all(not config.exists() for config in configurations) + + +def test_logged_timeout_preserves_partial_output(tmp_path): + log = tmp_path / "partial.log" + command = [sys.executable, "-c", "import time; print('before timeout', flush=True); time.sleep(10)"] + assert runner.run_logged(command, log, timeout=1) == 124 + assert "before timeout" in log.read_text() + + +@pytest.mark.parametrize("statuses, expected", [([0, 0], 0), ([5, 0], 5), ([0, 6], 6), ([124, 0], 124)]) +def test_inside_runs_both_shared_targets_and_propagates_failure(monkeypatch, tmp_path, statuses, expected): + calls = [] + environments = [] + results = iter(statuses) + + def fake_logged(command, log, **kwargs): + calls.append((command, log)) + environments.append(kwargs["env"]) + return next(results) + + monkeypatch.setenv("COVERAGE_FILE", "original") + monkeypatch.setattr(runner, "run_logged", fake_logged) + assert runner.inside(tmp_path) == expected + assert [command for command, _ in calls] == [["make", "coverage"], ["make", "test-e2e"]] + assert [log.name for _, log in calls] == ["unit.log", "e2e.log"] + + assert "E2E_ARGS" not in environments[0] + assert environments[0]["PYTEST_WORKERS"] == "2" + assert environments[0]["COV_XML"] == str(tmp_path / "coverage.xml") + assert all(log.parent == tmp_path for _, log in calls) + assert not list(tmp_path.glob("run-*")) + assert "--run-only" in environments[1]["E2E_ARGS"] + + +def test_build_requires_successful_audit(monkeypatch, tmp_path): + calls = [] + monkeypatch.setattr(runner, "audit_context", lambda *a, **kw: 7) + monkeypatch.setattr(runner, "run_logged", lambda *a, **kw: calls.append(a)) + assert runner.main(["build", "--artifacts", str(tmp_path)]) == 7 + assert calls == [] + + +@pytest.mark.parametrize("value", ["tcp://example.invalid:2375", "unix:///tmp/remote.sock"]) +def test_audit_rejects_endpoint_before_socket_probe(monkeypatch, tmp_path, value): + monkeypatch.setenv("DOCKER_HOST", value) + monkeypatch.setattr(runner.Path, "is_socket", lambda self: pytest.fail("socket probe preceded admission")) + monkeypatch.setattr(runner.subprocess, "Popen", lambda *a, **kw: pytest.fail("Docker invoked")) + assert runner.main(["audit", "--artifacts", str(tmp_path)]) == 1 + + +def test_audit_requires_local_socket(monkeypatch, tmp_path): + monkeypatch.setattr(runner, "LOCAL_SOCKET", tmp_path / "missing.sock") + monkeypatch.setattr(runner.subprocess, "Popen", lambda *a, **kw: pytest.fail("Docker invoked")) + assert runner.main(["audit", "--artifacts", str(tmp_path)]) == 1 + + +@pytest.mark.parametrize( + "builder_status, export_status, leak, missing, expected", + [ + (7, 0, False, False, 7), + (0, 9, False, False, 9), + (0, 0, True, False, 1), + (0, 0, False, True, 1), + (0, 0, False, False, 0), + ], +) +def test_audit_export_is_required_and_checked( + monkeypatch, tmp_path, builder_status, export_status, leak, missing, expected +): + monkeypatch.setattr(runner, "local_client", lambda config: (["docker"], {})) + + def fake_logged(command, log, **kwargs): + if "inspect" in command: + log.write_text("Driver: docker\n") + return builder_status + exported = Path(command[command.index("--output") + 1].split("dest=", 1)[1]) + context = Path(command[-1]) + for case in runner.json.loads((runner.ROOT / "tests/fixtures/exclusion-policy.json").read_text()): + if case["docker_included"] and not missing or not case["docker_included"] and leak: + target = exported / case["path"] + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text((context / case["path"]).read_text()) + return export_status + + monkeypatch.setattr(runner, "run_logged", fake_logged) + assert runner.main(["audit", "--artifacts", str(tmp_path)]) == expected + + +def test_audit_rejects_nonlocal_builder_driver(monkeypatch, tmp_path): + monkeypatch.setattr(runner, "local_client", lambda config: (["docker"], {})) + + def fake_logged(command, log, **kwargs): + assert "inspect" in command, "Context transfer preceded builder admission" + log.write_text("Driver: remote\n") + return 0 + + monkeypatch.setattr(runner, "run_logged", fake_logged) + assert runner.main(["audit", "--artifacts", str(tmp_path)]) == 1 + + +def test_client_pins_transport_and_isolates_config(monkeypatch, tmp_path): + for name in runner.os.environ: + if name.startswith(("DOCKER_", "BUILDX_", "BUILDKIT_")): + monkeypatch.delenv(name) + monkeypatch.setattr(runner.Path, "is_socket", lambda self: True) + monkeypatch.setenv("DOCKER_HOST", "") + client, environment = runner.local_client(tmp_path / "client") + assert client == ["docker", "--config", str(tmp_path / "client"), "--host", "unix:///var/run/docker.sock"] + assert environment["DOCKER_CONFIG"] == str(tmp_path / "client") + assert "DOCKER_HOST" not in environment + + +def test_run_missing_local_socket_invokes_no_docker(monkeypatch, tmp_path): + calls = [] + monkeypatch.setattr(runner, "LOCAL_SOCKET", tmp_path / "missing.sock") + monkeypatch.setattr(runner, "run_logged", lambda *a, **kw: calls.append(a) or 0) + monkeypatch.setattr(runner.subprocess, "run", lambda *a, **kw: calls.append(a)) + assert runner.main(["run", "--artifacts", str(tmp_path)]) == 1 + assert calls == [] + + +@pytest.mark.parametrize( + "selector", + [ + "DOCKER_HOST", + "DOCKER_CONTEXT", + "DOCKER_CONFIG", + "DOCKER_TLS_VERIFY", + "DOCKER_CERT_PATH", + "BUILDX_BUILDER", + "BUILDX_CONFIG", + "BUILDKIT_HOST", + "DOCKER_BUILDKIT", + "BUILDX_BAKE_FILE", + "BUILDKIT_PROGRESS", + ], +) +def test_each_inherited_selector_is_rejected_before_socket_probe(monkeypatch, tmp_path, selector): + monkeypatch.setenv(selector, "hostile-selector") + monkeypatch.setattr(runner.Path, "is_socket", lambda self: pytest.fail("socket probe preceded admission")) + with pytest.raises(ValueError, match=selector): + runner.local_client(tmp_path / "client") + assert not (tmp_path / "client").exists() + + +@pytest.mark.parametrize("action", ["audit", "build", "run"]) +def test_action_wiring_rejects_remote_transport_without_any_command(monkeypatch, tmp_path, action): + monkeypatch.setenv("DOCKER_HOST", "tcp://example.invalid:2375") + monkeypatch.setattr(runner.subprocess, "run", lambda *args, **kwargs: pytest.fail("Docker invoked")) + monkeypatch.setattr(runner, "run_logged", lambda *args, **kwargs: pytest.fail("Docker invoked")) + assert runner.main([action, "--artifacts", str(tmp_path)]) == 1 + + +@pytest.mark.parametrize( + "outcome,expected,message", + [ + ("removed", 0, "container removed"), + ("already-absent", 0, "Status: already-absent"), + ("already-absent-zero", 0, "Status: already-absent"), + ("denied-zero", 1, "removal permission denied"), + ("wrong-absent", 1, "different-name"), + ("mixed-absent", 1, "removal permission denied"), + ("denied", 1, "partial cleanup output"), + ("timeout", 1, "partial output"), + ("socket-error", 1, "cleanup socket unavailable"), + ], +) +def test_each_cleanup_outcome_retains_diagnostics(monkeypatch, tmp_path, capsys, outcome, expected, message): + client, environment, name = ( + ["docker", "--host", "unix:///var/run/docker.sock"], + {"DOCKER_CONFIG": "isolated"}, + "synthetic-container", + ) + + def execute(command, **kwargs): + assert command == client + ["rm", "-f", name] + assert kwargs["env"] is environment + if outcome == "timeout": + raise runner.subprocess.TimeoutExpired(command, 30, output=b"partial output", stderr=b"cleanup timeout") + if outcome == "socket-error": + raise OSError("cleanup socket unavailable") + absent = f"Error response from daemon: No such container: {name}\n" + responses = { + "removed": (0, "container removed\n", ""), + "already-absent": (1, "", absent), + "already-absent-zero": (0, "", absent), + "denied-zero": (0, "partial cleanup output", "removal permission denied"), + "wrong-absent": (1, "", "Error response from daemon: No such container: different-name\n"), + "mixed-absent": (1, "", absent + "removal permission denied"), + "denied": (1, "partial cleanup output", "removal permission denied"), + } + return runner.subprocess.CompletedProcess(command, *responses[outcome]) + + monkeypatch.setattr(runner.subprocess, "run", execute) + log = tmp_path / "cleanup.log" + assert runner.cleanup_container(client, environment, name, log) == expected + assert message in log.read_text() + if expected: + assert name in capsys.readouterr().err + + +@pytest.mark.parametrize("primary,expected", [(0, 1), (9, 9), (124, 124)]) +def test_cleanup_failure_precedence_is_explicit(monkeypatch, tmp_path, primary, expected): + monkeypatch.setattr(runner, "local_client", lambda config: (["docker"], {})) + monkeypatch.setattr(runner, "run_logged", lambda *args, **kwargs: primary) + monkeypatch.setattr(runner, "cleanup_container", lambda *args: 1) + assert runner.main(["run", "--artifacts", str(tmp_path)]) == expected + + +@pytest.mark.parametrize("action", ["audit", "build", "run"]) +def test_fresh_attempts_keep_later_failures_separate(monkeypatch, tmp_path, action): + root = tmp_path / "results" + root.mkdir() + (root / "operator-note.txt").write_text("unrelated") + attempts = [] + codes = iter([0, 7]) + + def audit(destination, **kwargs): + attempts.append(destination) + code = next(codes) + (destination / ("failure.log" if code else "audit-result.json")).write_text("partial" if code else "passed") + return code + + def logged(command, log, **kwargs): + if action == "run": + attempts.append(log.parent) + code = next(codes) + if not code: + (log.parent / "unit.xml").write_text("passed") + else: + code = 0 + log.write_text("partial" if code else "passed") + return code + + monkeypatch.setattr(runner, "audit_context", audit) + monkeypatch.setattr(runner, "local_client", lambda config: (["docker"], {})) + monkeypatch.setattr(runner, "run_logged", logged) + monkeypatch.setattr(runner, "cleanup_container", lambda *args: 0) + assert runner.main([action, "--artifacts", str(root)]) == 0 + assert runner.main([action, "--artifacts", str(root)]) == 7 + assert len(attempts) == 2 and attempts[0] != attempts[1] + success = {"audit": "audit-result.json", "build": "build.log", "run": "unit.xml"}[action] + assert (attempts[0] / success).read_text() == "passed" + assert not (attempts[1] / success).exists() + assert (root / "operator-note.txt").read_text() == "unrelated" + + +@pytest.mark.parametrize("kind", ["symlink", "source", "source-ancestor"]) +def test_output_root_admission_preserves_rejected_paths(monkeypatch, tmp_path, kind): + source = tmp_path / "checkout" + source.mkdir() + (source / "tracked.txt").write_text("unchanged") + monkeypatch.setattr(runner, "ROOT", source) + root = source + if kind == "symlink": + root = tmp_path / "link" + root.symlink_to(source, target_is_directory=True) + elif kind == "source-ancestor": + root = tmp_path + with pytest.raises(ValueError): + runner.attempt_directory(root) + assert (source / "tracked.txt").read_text() == "unchanged" + assert not list(tmp_path.rglob("run-*"))