diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b302013a..6609286d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,11 @@ on: pull_request: branches: [main] workflow_dispatch: + inputs: + v3_release_candidate_sha: + description: Full candidate SHA for the six-platform V3 release gate + required: false + type: string schedule: # Monday 04:17 Asia/Shanghai. - cron: "17 20 * * 0" @@ -30,6 +35,7 @@ jobs: all: ${{ steps.classify.outputs.all }} core: ${{ steps.classify.outputs.core }} full: ${{ steps.classify.outputs.full }} + release: ${{ steps.classify.outputs.release }} steps: - uses: actions/checkout@v4 with: @@ -41,9 +47,11 @@ jobs: BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} HEAD_SHA: ${{ github.sha }} FULL_RUN: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} + RELEASE_CANDIDATE_SHA: ${{ inputs.v3_release_candidate_sha }} run: | args=(--event "$EVENT_NAME" --base "$BASE_SHA" --head "$HEAD_SHA") if [[ "$FULL_RUN" == true ]]; then args+=(--full); fi + if [[ -n "$RELEASE_CANDIDATE_SHA" ]]; then args+=(--release-candidate-sha "$RELEASE_CANDIDATE_SHA"); fi python scripts/ci_changes.py "${args[@]}" core-test: @@ -257,6 +265,194 @@ jobs: name: update-acceptance-${{ matrix.os }} path: update-acceptance.json + v3-platform-validation: + needs: changes + if: needs.changes.outputs.release == 'true' + name: V3 production matrix (${{ matrix.platform }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + include: + - { platform: linux-py312, os: ubuntu-latest, python: '3.12' } + - { platform: linux-py313, os: ubuntu-latest, python: '3.13' } + - { platform: macos-py312, os: macos-latest, python: '3.12' } + - { platform: macos-py313, os: macos-latest, python: '3.13' } + - { platform: windows-py312, os: windows-latest, python: '3.12' } + - { platform: windows-py313, os: windows-latest, python: '3.13' } + env: + CANDIDATE_SHA: ${{ github.event.pull_request.head.sha || inputs.v3_release_candidate_sha || github.sha }} + PLATFORM_ID: ${{ matrix.platform }} + KYROZEN_BROWSER_TESTS: '1' + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || inputs.v3_release_candidate_sha || github.sha }} + - name: Verify exact candidate + shell: bash + run: | + [[ "$CANDIDATE_SHA" =~ ^([0-9a-f]{40}|[0-9a-f]{64})$ ]] + test "$(git rev-parse HEAD)" = "$CANDIDATE_SHA" + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + cache: pip + cache-dependency-path: | + pyproject.toml + requirements.txt + - uses: actions/setup-go@v5 + with: + go-version: '1.27.1' + cache-dependency-path: tui/go.sum + - name: Install Linux sandbox and browser system dependencies + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install --no-install-recommends -y bubblewrap + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + python -m pip install playwright + python -m playwright install-deps chromium + - name: Install project and browser dependencies + shell: bash + run: | + python -m pip install --upgrade pip + python -m pip install -e '.[all]' + python -m playwright install chromium + - name: Require the Linux filesystem sandbox + if: runner.os == 'Linux' + shell: bash + run: echo 'KYROZEN_SANDBOX_REQUIRED=1' >> "$GITHUB_ENV" + - name: Run complete unit suite + if: runner.os != 'Windows' + shell: bash + run: | + set -euo pipefail + python -m unittest discover -s tests -p 'test_*.py' -v 2>&1 | tee unit.log + - name: Run integration suite + if: runner.os != 'Windows' + shell: bash + run: | + set -euo pipefail + PYTHONPATH=tests python -m unittest tests.test_durable_tasks_gateway tests.test_gateway_startup_logging -v 2>&1 | tee integration.log + - name: Run supported-feature workflow and TUI acceptance + env: + PYTHONIOENCODING: utf-8 + shell: bash + run: | + set -euo pipefail + python scripts/agent_workflow_acceptance.py 2>&1 | tee agent-acceptance.log + python scripts/subagent_workflow_acceptance.py 2>&1 | tee subagent-acceptance.log + binary="../kyrozen-tui" + if [[ "$RUNNER_OS" == Windows ]]; then binary="../kyrozen-tui.exe"; fi + (cd tui && go test ./... && go build -o "$binary" .) 2>&1 | tee tui-tests.log + if [[ "$RUNNER_OS" == Windows ]]; then + echo 'not_run: interactive PTY acceptance requires POSIX fcntl/pty; Windows Go TUI tests/build and agent workflow acceptance are required.' | tee tui-acceptance.log + else + python scripts/tui_workflow_acceptance.py "${binary#../}" 2>&1 | tee tui-acceptance.log + fi + - name: Run 30-minute production soak + shell: bash + run: | + set -euo pipefail + python scripts/production_soak.py --seconds 1800 \ + --output soak.json --log-dir soak-logs 2>&1 | tee soak.log + - name: Record platform result + if: success() + shell: bash + run: | + python - <<'PY' + import importlib.metadata + import json + import os + import platform + from pathlib import Path + soak = json.loads(Path("soak.json").read_text(encoding="utf-8")) + dependencies = { + distribution.metadata["Name"]: distribution.version + for distribution in importlib.metadata.distributions() + if distribution.metadata.get("Name") + } + result = { + "schema_version": 1, + "candidate_sha": os.environ["CANDIDATE_SHA"], + "platform": os.environ["PLATFORM_ID"], + "unit": "not_run" if os.environ["RUNNER_OS"] == "Windows" else "passed", + "integration": "not_run" if os.environ["RUNNER_OS"] == "Windows" else "passed", + "acceptance": "passed", + "tui_interactive_acceptance": "not_run_posix_pty_unavailable" if os.environ["RUNNER_OS"] == "Windows" else "passed", + "soak_seconds": int(soak["elapsed_seconds"]), + "environment": { + "runner": os.environ["ImageOS"] if "ImageOS" in os.environ else os.environ["RUNNER_OS"], + "os": platform.platform(), + "python": platform.python_version(), + }, + "dependencies": dependencies, + "evidence_paths": ([ + "agent-acceptance.log", "subagent-acceptance.log", "tui-tests.log", + "tui-acceptance.log", "soak.log", "soak.json", + ] if os.environ["RUNNER_OS"] == "Windows" else [ + "unit.log", "integration.log", "agent-acceptance.log", + "subagent-acceptance.log", "tui-tests.log", "tui-acceptance.log", + "soak.log", "soak.json", + ]), + } + if os.environ["RUNNER_OS"] == "Windows": + result["supported_features"] = "passed" + Path("platform.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + PY + - name: Upload platform logs and result + if: always() + uses: actions/upload-artifact@v4 + with: + name: v3-platform-${{ matrix.platform }} + path: | + platform.json + unit.log + integration.log + agent-acceptance.log + subagent-acceptance.log + tui-acceptance.log + tui-tests.log + soak.log + soak.json + soak-logs/ + if-no-files-found: warn + retention-days: 30 + + collect-r266: + needs: [changes, v3-platform-validation] + if: needs.changes.outputs.release == 'true' && needs.v3-platform-validation.result == 'success' + runs-on: ubuntu-latest + env: + CANDIDATE_SHA: ${{ github.event.pull_request.head.sha || inputs.v3_release_candidate_sha || github.sha }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || inputs.v3_release_candidate_sha || github.sha }} + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - uses: actions/download-artifact@v4 + with: + pattern: v3-platform-* + path: platform-results + merge-multiple: false + - name: Collect unsigned exact-candidate R266 record + run: >- + python -m benchmarks.v3_release_platforms + --results-dir platform-results + --output-dir v3-r266-evidence + --candidate-sha "$CANDIDATE_SHA" + --command-set .github/workflows/ci.yml + - name: Upload unsigned R266 evidence bundle + uses: actions/upload-artifact@v4 + with: + name: v3-r266-${{ github.event.pull_request.head.sha || inputs.v3_release_candidate_sha || github.sha }} + path: v3-r266-evidence/ + if-no-files-found: error + retention-days: 30 + ci-result: name: CI result if: always() @@ -270,6 +466,8 @@ jobs: - installer-syntax - docker-persistence - update-install + - v3-platform-validation + - collect-r266 runs-on: ubuntu-latest steps: - name: Require every selected check to pass @@ -290,6 +488,9 @@ jobs: UPDATE_RESULT: ${{ needs.update-install.result }} DOCKER_EXPECTED: ${{ needs.changes.outputs.docker }} DOCKER_RESULT: ${{ needs.docker-persistence.result }} + RELEASE_EXPECTED: ${{ needs.changes.outputs.release }} + RELEASE_MATRIX_RESULT: ${{ needs.v3-platform-validation.result }} + RELEASE_COLLECT_RESULT: ${{ needs.collect-r266.result }} run: | set -euo pipefail test "$CHANGES_RESULT" = success @@ -315,6 +516,8 @@ jobs: require_selected "Installer syntax" "$UPDATE_EXPECTED" "$INSTALLER_RESULT" require_selected "Installed update" "$UPDATE_EXPECTED" "$UPDATE_RESULT" require_selected "Docker persistence" "$DOCKER_EXPECTED" "$DOCKER_RESULT" + require_selected "V3 release matrix" "$RELEASE_EXPECTED" "$RELEASE_MATRIX_RESULT" + require_selected "V3 R266 evidence collection" "$RELEASE_EXPECTED" "$RELEASE_COLLECT_RESULT" if [ "$CORE_EXPECTED" != true ]; then require_unselected "Unselected core job" "$CORE_RESULT"; fi if [ "$PYTHON_EXPECTED" != true ]; then require_unselected "Unselected Python tests" "$PYTHON_RESULT"; fi if [ "$ALL_EXPECTED" != true ]; then require_unselected "Unselected full Python 3.12 tests" "$PYTHON_312_RESULT"; fi @@ -323,3 +526,5 @@ jobs: require_unselected "Unselected installer syntax" "$INSTALLER_RESULT" require_unselected "Unselected installed update" "$UPDATE_RESULT" require_unselected "Unselected Docker persistence" "$DOCKER_RESULT" + if [ "$RELEASE_EXPECTED" != true ]; then require_unselected "Unselected V3 release matrix" "$RELEASE_MATRIX_RESULT"; fi + if [ "$RELEASE_EXPECTED" != true ]; then require_unselected "Unselected V3 R266 collection" "$RELEASE_COLLECT_RESULT"; fi diff --git a/benchmarks/v3_release_gates.py b/benchmarks/v3_release_gates.py new file mode 100644 index 00000000..8ed993d0 --- /dev/null +++ b/benchmarks/v3_release_gates.py @@ -0,0 +1,262 @@ +"""Validate and sign exact-candidate V3 release-gate evidence bundles.""" +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import tempfile + +GATE_IDS = tuple(f"R{number}" for number in range(250, 267)) +REQUIRED_PLATFORMS = ( + "linux-py312", "linux-py313", "macos-py312", "macos-py313", "windows-py312", "windows-py313", +) +SIGNER_FINGERPRINT = "D3897E8FF704149D609ACF878C118C04263D2A4C" +_SHA256 = re.compile(r"^[0-9a-f]{64}$") +_GIT_SHA = re.compile(r"^(?:[0-9a-f]{40}|[0-9a-f]{64})$") + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _contained_file(root: Path, relative: str) -> Path: + if not isinstance(relative, str) or not relative or Path(relative).is_absolute(): + raise ValueError("evidence artifact path must be a nonempty relative path") + path = (root / relative).resolve() + try: + path.relative_to(root) + except ValueError: + raise ValueError("evidence artifact escapes evidence directory") from None + if not path.is_file(): + raise ValueError("evidence artifact is not a regular file") + return path + + +def _check_record(record: object, gate_id: str, candidate_sha: str, evidence_dir: Path) -> list[dict]: + if not isinstance(record, dict) or record.get("schema_version") != 1: + raise ValueError(f"{gate_id}: unsupported evidence record schema") + if record.get("gate_id") != gate_id: + raise ValueError(f"{gate_id}: record gate ID does not match its filename") + if record.get("candidate_sha") != candidate_sha: + raise ValueError(f"{gate_id}: candidate SHA does not match the current candidate") + for name in ("command_set_sha256", "fixture_set_sha256"): + if not isinstance(record.get(name), str) or not _SHA256.fullmatch(record[name]): + raise ValueError(f"{gate_id}: {name} must be a lowercase SHA-256 digest") + environment = record.get("environment") + if not isinstance(environment, dict) or any(not isinstance(environment.get(key), str) or not environment[key] + for key in ("runner", "os", "python")): + raise ValueError(f"{gate_id}: runner, OS, and Python environment details are required") + if not isinstance(record.get("dependencies"), dict): + raise ValueError(f"{gate_id}: dependency inventory must be an object") + for path_key, hash_key, description in ( + ("command_set_path", "command_set_sha256", "command-set"), + ("fixture_set_path", "fixture_set_sha256", "fixture-set"), + ): + path = _contained_file(evidence_dir, record.get(path_key)) + if _sha256(path) != record[hash_key]: + raise ValueError(f"{gate_id}: {description} hash mismatch") + result = record.get("result") + if not isinstance(result, str) or result not in {"passed", "failed", "inconclusive"}: + raise ValueError(f"{gate_id}: invalid result") + if result != "passed": + if not isinstance(record.get("failure_class"), str) or not record["failure_class"].strip(): + raise ValueError(f"{gate_id}: nonpassing evidence requires a failure class") + raise ValueError(f"{gate_id}: gate did not pass ({record['failure_class']})") + if record.get("failure_class") is not None: + raise ValueError(f"{gate_id}: passing evidence cannot have a failure class") + refs = record.get("evidence") + if not isinstance(refs, list) or not refs: + raise ValueError(f"{gate_id}: at least one evidence artifact is required") + artifacts = [] + for ref in refs: + if not isinstance(ref, dict) or not isinstance(ref.get("sha256"), str) or not _SHA256.fullmatch(ref["sha256"]): + raise ValueError(f"{gate_id}: invalid evidence artifact reference") + path = _contained_file(evidence_dir, ref.get("path")) + actual = _sha256(path) + if actual != ref["sha256"]: + raise ValueError(f"{gate_id}: evidence hash mismatch for {ref['path']}") + artifacts.append({"path": path.relative_to(evidence_dir).as_posix(), "sha256": actual}) + if gate_id == "R266": + platforms = record.get("platform_results") + if not isinstance(platforms, list) or len(platforms) != len(REQUIRED_PLATFORMS): + raise ValueError("R266: platform matrix must contain exactly six required platforms") + if any(not isinstance(item, dict) or not isinstance(item.get("platform"), str) + for item in platforms): + raise ValueError("R266: platform matrix must contain exactly six required platforms") + indexed = {item["platform"]: item for item in platforms} + if set(indexed) != set(REQUIRED_PLATFORMS): + raise ValueError("R266: platform matrix must contain exactly six required platforms") + refs = record.get("evidence") + artifact_paths = {ref["path"] for ref in refs + if isinstance(ref, dict) and isinstance(ref.get("path"), str)} if isinstance(refs, list) else set() + for platform in REQUIRED_PLATFORMS: + item = indexed[platform] + if platform.startswith("windows-"): + checks_failed = ( + item.get("unit") != "not_run" + or item.get("integration") != "not_run" + or item.get("supported_features") != "passed" + or item.get("acceptance") != "passed" + or item.get("tui_interactive_acceptance") != "not_run_posix_pty_unavailable" + ) + else: + checks_failed = any(item.get(check) != "passed" + for check in ("unit", "integration", "acceptance")) + if checks_failed: + raise ValueError(f"R266: platform validation failed for {platform}") + if not isinstance(item.get("soak_seconds"), int) or item["soak_seconds"] < 1800: + raise ValueError(f"R266: platform {platform} has not passed the 30-minute soak") + evidence_paths = item.get("evidence_paths") + if not isinstance(evidence_paths, list) or not evidence_paths or any( + not isinstance(path, str) or path not in artifact_paths for path in evidence_paths): + raise ValueError(f"R266: platform evidence is missing or unlinked for {platform}") + return artifacts + + +def validate_evidence_bundle(evidence_dir: Path, candidate_sha: str, *, verify_signature) -> dict: + """Validate every signed gate record and return a manifest bound to candidate_sha.""" + evidence_dir = Path(evidence_dir).resolve(strict=True) + if not _GIT_SHA.fullmatch(candidate_sha): + raise ValueError("candidate SHA must be a full Git object ID") + gates = {} + for gate_id in GATE_IDS: + try: + record_path = _contained_file(evidence_dir, f"{gate_id}.json") + signature_path = _contained_file(evidence_dir, f"{gate_id}.json.asc") + except ValueError: + raise ValueError(f"missing gate record or signature for {gate_id}") + if not verify_signature(record_path, signature_path): + raise ValueError(f"{gate_id}: signature verification failed") + try: + record = json.loads(record_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"{gate_id}: invalid JSON record: {exc}") from None + artifacts = _check_record(record, gate_id, candidate_sha, evidence_dir) + gates[gate_id] = { + "record_path": record_path.name, + "record_sha256": _sha256(record_path), + "signature_path": signature_path.name, + "signature_sha256": _sha256(signature_path), + "command_set_sha256": record["command_set_sha256"], + "fixture_set_sha256": record["fixture_set_sha256"], + "command_set_path": record["command_set_path"], + "fixture_set_path": record["fixture_set_path"], + "environment": record["environment"], + "dependencies": record["dependencies"], + "evidence": artifacts, + **({"platform_results": record["platform_results"]} if gate_id == "R266" else {}), + } + return { + "schema_version": 1, + "candidate_sha": candidate_sha, + "signer_fingerprint": SIGNER_FINGERPRINT, + "created_at": datetime.now(timezone.utc).isoformat(), + "result": "passed", + "gates": gates, + } + + +def _gpg_verify(record_path: Path, signature_path: Path) -> bool: + try: + result = subprocess.run( + ["gpg", "--batch", "--status-fd", "1", "--verify", str(signature_path), str(record_path)], + check=False, capture_output=True, text=True, timeout=30, + ) + except (OSError, subprocess.TimeoutExpired): + return False + if result.returncode: + return False + for line in result.stdout.splitlines(): + if line.startswith("[GNUPG:] VALIDSIG "): + fields = line.split() + signer = fields[2] if len(fields) > 2 else "" + primary = fields[11] if len(fields) > 11 else signer + return SIGNER_FINGERPRINT in {signer, primary} + return False + + +def _current_candidate(repo: Path) -> str: + head = subprocess.run(["git", "-C", str(repo), "rev-parse", "--verify", "HEAD"], + check=True, capture_output=True, text=True, timeout=15).stdout.strip() + dirty = subprocess.run(["git", "-C", str(repo), "status", "--porcelain"], + check=True, capture_output=True, text=True, timeout=15).stdout + if dirty: + raise ValueError("candidate worktree must be clean before release evidence is validated") + return head + + +def _require_outside_checkout(path: Path, repo: Path, label: str) -> None: + try: + path.relative_to(repo) + except ValueError: + return + raise ValueError(f"{label} must be outside the candidate checkout") + + +def _write_signed_manifest(manifest: dict, destination: Path) -> None: + destination = destination.resolve() + signature_destination = destination.with_name(destination.name + ".asc") + if destination.exists() or signature_destination.exists(): + raise ValueError("manifest output already exists; choose a new path") + destination.parent.mkdir(parents=True, exist_ok=True) + payload = (json.dumps(manifest, indent=2, sort_keys=True) + "\n").encode() + with tempfile.TemporaryDirectory(prefix="v3-release-evidence-", dir=destination.parent) as temporary: + temporary = Path(temporary) + record = temporary / destination.name + signature = temporary / f"{destination.name}.asc" + record.write_bytes(payload) + try: + result = subprocess.run( + ["gpg", "--batch", "--yes", "--local-user", SIGNER_FINGERPRINT, + "--armor", "--detach-sign", "--output", str(signature), str(record)], + check=False, capture_output=True, text=True, timeout=30, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise ValueError(f"could not sign release manifest: {exc}") from None + if result.returncode or not signature.is_file() or not _gpg_verify(record, signature): + raise ValueError("release manifest GPG signing or fingerprint verification failed") + os.link(signature, signature_destination) + try: + os.link(record, destination) + except OSError: + try: + signature_destination.unlink() + except OSError: + pass + raise + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", type=Path, default=Path.cwd()) + parser.add_argument("--evidence-dir", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + args = parser.parse_args(argv) + try: + repo = args.repo.resolve(strict=True) + evidence_dir = args.evidence_dir.resolve(strict=True) + manifest_path = args.manifest.resolve() + _require_outside_checkout(evidence_dir, repo, "evidence directory") + _require_outside_checkout(manifest_path, repo, "manifest") + candidate = _current_candidate(repo) + manifest = validate_evidence_bundle(evidence_dir, candidate, verify_signature=_gpg_verify) + _write_signed_manifest(manifest, manifest_path) + except (OSError, subprocess.SubprocessError, ValueError) as exc: + print(f"release blocked: {exc}", file=sys.stderr) + return 1 + print(f"release evidence passed for {candidate}; signed manifest: {args.manifest}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/v3_release_platforms.py b/benchmarks/v3_release_platforms.py new file mode 100644 index 00000000..2e2e9d5b --- /dev/null +++ b/benchmarks/v3_release_platforms.py @@ -0,0 +1,199 @@ +"""Collect six-platform CI artifacts into an unsigned R266 evidence record.""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import sys +import tempfile + +from benchmarks.v3_fixtures import FIXTURES +from benchmarks.v3_release_gates import REQUIRED_PLATFORMS + +_GIT_SHA = re.compile(r"^(?:[0-9a-f]{40}|[0-9a-f]{64})$") + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _copy_artifact(source_root: Path, staging: Path, platform: str, relative: object) -> str: + if not isinstance(relative, str) or not relative or Path(relative).is_absolute(): + raise ValueError(f"{platform}: evidence artifact path must be relative") + source = (source_root / relative).resolve() + try: + source.relative_to(source_root) + except ValueError: + raise ValueError(f"{platform}: evidence artifact escapes its result directory") from None + if not source.is_file(): + raise ValueError(f"{platform}: evidence artifact is missing or not a file: {relative}") + destination_relative = (Path("platforms") / platform / relative).as_posix() + destination = staging / destination_relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + return destination_relative + + +def _matches_platform(platform: str, environment: dict) -> bool: + os_name, python_id = platform.rsplit("-", 1) + python_version = f"3.{python_id[-2:]}" + actual_os = environment["os"].lower() + os_matches = { + "linux": "linux" in actual_os, + "macos": "mac" in actual_os or "darwin" in actual_os, + "windows": "windows" in actual_os, + }[os_name] + return os_matches and environment["python"].startswith(python_version) + + +def collect_platform_evidence(results_dir: Path, output_dir: Path, candidate_sha: str, + command_set: Path) -> dict: + """Build an unsigned, candidate-bound R266.json bundle from passing matrix artifacts.""" + results_dir = Path(results_dir).resolve(strict=True) + command_set = Path(command_set).resolve(strict=True) + output_dir = Path(output_dir).resolve() + if not _GIT_SHA.fullmatch(candidate_sha): + raise ValueError("candidate SHA must be a full Git object ID") + if output_dir.exists(): + raise ValueError("evidence output directory already exists") + if not command_set.is_file(): + raise ValueError("command set must be a regular file") + output_dir.parent.mkdir(parents=True, exist_ok=True) + found_platforms = {path.name.removeprefix("v3-platform-") for path in results_dir.iterdir() + if path.name.startswith("v3-platform-")} + if found_platforms != set(REQUIRED_PLATFORMS): + raise ValueError("results directory must contain exactly the six required platform artifacts") + + with tempfile.TemporaryDirectory(prefix="v3-r266-", dir=output_dir.parent) as temporary: + staging = Path(temporary) / "bundle" + staging.mkdir() + command_path = staging / "commands" / command_set.name + command_path.parent.mkdir(parents=True) + shutil.copyfile(command_set, command_path) + fixture_path = staging / "fixtures" / "v3-fixtures.json" + fixture_path.parent.mkdir(parents=True) + fixture_path.write_text(json.dumps(FIXTURES, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + evidence = [] + platform_results = [] + dependency_sets = {} + for platform in REQUIRED_PLATFORMS: + result_dir = results_dir / f"v3-platform-{platform}" + result_path = result_dir / "platform.json" + if result_dir.is_symlink() or not result_dir.is_dir() or result_path.is_symlink(): + raise ValueError(f"{platform}: platform result directory or metadata cannot be a symlink") + try: + result = json.loads(result_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + raise ValueError(f"{platform}: missing or invalid platform result") from None + if not isinstance(result, dict) or result.get("schema_version") != 1: + raise ValueError(f"{platform}: unsupported platform result schema") + if result.get("candidate_sha") != candidate_sha or result.get("platform") != platform: + raise ValueError(f"{platform}: candidate SHA or platform does not match") + if platform.startswith("windows-"): + if (result.get("unit") != "not_run" or result.get("integration") != "not_run" + or result.get("supported_features") != "passed" + or result.get("tui_interactive_acceptance") != "not_run_posix_pty_unavailable"): + raise ValueError(f"{platform}: supported-feature acceptance did not pass") + elif any(result.get(check) != "passed" for check in ("unit", "integration")): + raise ValueError(f"{platform}: platform validation did not pass (unit/integration)") + elif result.get("tui_interactive_acceptance") != "passed": + raise ValueError(f"{platform}: interactive TUI acceptance did not pass") + if result.get("acceptance") != "passed": + raise ValueError(f"{platform}: platform validation did not pass (acceptance)") + soak_seconds = result.get("soak_seconds") + if isinstance(soak_seconds, bool) or not isinstance(soak_seconds, int) or soak_seconds < 1800: + raise ValueError(f"{platform}: 30-minute soak has not passed") + environment = result.get("environment") + dependencies = result.get("dependencies") + if not isinstance(environment, dict) or any( + not isinstance(environment.get(key), str) or not environment[key] + for key in ("runner", "os", "python")): + raise ValueError(f"{platform}: runner, OS, and Python details are required") + if not isinstance(dependencies, dict) or not dependencies: + raise ValueError(f"{platform}: dependency inventory must be an object") + paths = result.get("evidence_paths") + if (not isinstance(paths, list) or not paths + or any(not isinstance(path, str) for path in paths) + or len(paths) != len(set(paths)) or "platform.json" in paths): + raise ValueError(f"{platform}: evidence artifact list is required") + if not _matches_platform(platform, environment): + raise ValueError(f"{platform}: runner OS or Python version does not match") + copied_paths = [_copy_artifact(result_dir, staging, platform, "platform.json")] + copied_paths.extend(_copy_artifact(result_dir, staging, platform, path) for path in paths) + for relative in copied_paths: + evidence.append({"path": relative, "sha256": _sha256(staging / relative)}) + dependency_sets[platform] = dependencies + platform_results.append({ + "platform": platform, + "unit": result["unit"], + "integration": result["integration"], + "acceptance": "passed", + "soak_seconds": soak_seconds, + "evidence_paths": copied_paths, + "environment": environment, + "dependencies": dependencies, + **({"supported_features": "passed"} if platform.startswith("windows-") else {}), + "tui_interactive_acceptance": result["tui_interactive_acceptance"], + }) + + command_relative = (Path("commands") / command_set.name).as_posix() + fixture_relative = "fixtures/v3-fixtures.json" + record = { + "schema_version": 1, + "gate_id": "R266", + "candidate_sha": candidate_sha, + "command_set_path": command_relative, + "command_set_sha256": _sha256(staging / command_relative), + "fixture_set_path": fixture_relative, + "fixture_set_sha256": _sha256(staging / fixture_relative), + "environment": {"runner": "GitHub Actions matrix", "os": "Linux, macOS, Windows", + "python": "3.12 and 3.13"}, + "dependencies": {"platforms": dependency_sets}, + "result": "passed", + "failure_class": None, + "evidence": evidence, + "platform_results": platform_results, + } + record_path = staging / "R266.json" + record_path.write_text(json.dumps(record, indent=2, sort_keys=True) + "\n", encoding="utf-8") + try: + os.mkdir(output_dir) + except FileExistsError: + raise ValueError("evidence output directory already exists") from None + try: + for item in staging.iterdir(): + os.rename(item, output_dir / item.name) + except OSError: + shutil.rmtree(output_dir) + raise + return record + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--results-dir", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--candidate-sha", required=True) + parser.add_argument("--command-set", type=Path, required=True) + args = parser.parse_args(argv) + try: + record = collect_platform_evidence(args.results_dir, args.output_dir, + args.candidate_sha, args.command_set) + except (OSError, ValueError) as exc: + print(f"R266 evidence collection blocked: {exc}", file=sys.stderr) + return 1 + print(f"collected unsigned R266 evidence for {record['candidate_sha']} at {args.output_dir}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/index.md b/docs/index.md index 065d8275..86a9f76f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -27,6 +27,7 @@ This index covers the current product guides, developer references, and dated en - [Development guide](development.md) and [contributor guide](../CONTRIBUTING.md) cover setup, layout, checks, and pull requests. - [V3 matched evaluation](v3-evaluation.md) documents the frozen cohort, independent checks, and spend controls. +- [V3 release evidence](v3-release-evidence.md) documents signed, exact-candidate gates and the six-platform test and soak evidence workflow. - [Reports index](reports.md) explains the dates, scope, and limits of recorded audits, validation runs, and benchmarks. - [Modular refactor validation](modular-refactor-validation.md), [production audit](production-audit-2026-10-04.md), [shipping audit](production-shipping-audit-2026-10-04.md), and [self-learning audit](self-learning-audit-2026-10-03.md) preserve historical evidence. - [V3 architecture ownership plan](superpowers/plans/2026-10-07-v3-issue-227.md) records the scoped implementation and verification for issue #227. diff --git a/docs/v3-release-evidence.md b/docs/v3-release-evidence.md new file mode 100644 index 00000000..3341ebf9 --- /dev/null +++ b/docs/v3-release-evidence.md @@ -0,0 +1,44 @@ +# V3 release evidence + +Release publication is blocked until signed records for R250–R266 all pass against the exact clean candidate commit. Each record is detached-GPG-signed by the pinned release-owner key, includes command and fixture files with verified SHA-256 digests, runner/OS/Python and dependency details, and references hashed result artifacts. Failed or inconclusive records must identify a failure class and can never produce a passing manifest. + +R266 carries the supported-environment matrix: Linux, macOS, and Windows on Python 3.12 and 3.13. Linux and macOS records must show passing full unit and gateway integration suites. Windows records mark those POSIX-dependent suites `not_run` and must show passing supported-feature workflow acceptance, including Go TUI tests and build. The interactive TUI PTY scenario is explicitly recorded as `not_run_posix_pty_unavailable` on Windows because its harness requires POSIX `fcntl`/`pty`; it must pass on Linux and macOS. All six records require passing supported-feature acceptance and at least 1,800 seconds of soak evidence. Each platform entry names nonempty `evidence_paths`, and every path must reference a hashed artifact in the enclosing record. + +The CI workflow runs the R266 matrix for pull requests or pushes that change its command set, fixture set, collector, or soak script. For an explicit run, dispatch `.github/workflows/ci.yml` with `v3_release_candidate_sha` set to the full candidate commit SHA. It checks out that exact SHA, runs full Python and gateway integration suites on Linux and macOS, and runs agent/sub-agent acceptance plus Go TUI tests and builds on all six platform/Python combinations. The POSIX interactive TUI PTY scenario runs on Linux and macOS only; Windows records its unavailability. Each platform also runs a 30-minute production soak. Windows records its Python unit and gateway integration checks as `not_run`; its supported-feature acceptance is required to pass. The workflow uploads per-platform logs even on failure for diagnosis and emits an R266 bundle only after every matrix job passes. The collector copies and hashes the command workflow, frozen fixture set, platform metadata, test logs, and soak results. The bundle is intentionally unsigned because the release-owner private key must stay off CI. Download it, place it in the evidence directory outside the candidate checkout, inspect the evidence, and sign `R266.json` with the pinned release-owner key before validating the complete release bundle: + +The soak runner gathers memory, thread, handle, and descendant-process evidence through Linux `/proc`, macOS `ps`/`lsof`, and Windows PowerShell. It uses only the Python standard library and platform tools; it does not add a Python dependency. + +```sh +gpg --batch --local-user D3897E8FF704149D609ACF878C118C04263D2A4C \ + --armor --detach-sign --output R266.json.asc R266.json +``` + +The collector rejects stale candidate SHAs, missing artifacts, failed checks, short soaks, unexpected platform artifacts, and existing output directories. + +The dispatch workflow is evidence collection, not a substitute for inspecting its result. A failed matrix run produces diagnostic artifacts but no passing R266 record; rerun only after classifying the failure as product or runner/infrastructure. + +Store each record and its artifacts under one evidence directory, using these names: + +```text +R250.json +R250.json.asc +... +R266.json +R266.json.asc +commands/R250.txt +fixtures/v3-fixtures.json +results/... +``` + +Records use schema version 1. Their required fields are `gate_id`, full `candidate_sha`, `command_set_path` and `command_set_sha256`, `fixture_set_path` and `fixture_set_sha256`, `environment` (`runner`, `os`, and `python`), `dependencies`, `result`, `failure_class`, and a nonempty `evidence` array of relative artifact paths and SHA-256 hashes. R266 also requires the six `platform_results` described above, each linked to its own hashed logs. Artifacts and records may not resolve outside the evidence directory. + +After all records have been collected, validate and sign the aggregate manifest from a clean checkout. Keep both output files outside the checkout so writing the manifest cannot change the candidate SHA: + +```sh +python -m benchmarks.v3_release_gates \ + --repo . \ + --evidence-dir /outside/checkout/v3-evidence \ + --manifest /outside/checkout/v3-release-manifest.json +``` + +The command verifies every record signature against the pinned GPG fingerprint, checks the exact checkout SHA, validates all artifacts and platform gates, then emits a signed manifest and detached signature. Any missing, stale, malformed, failed, unsigned, or inconclusive evidence returns failure and emits no manifest. It publishes the signature first and the manifest last, and refuses to overwrite either existing output; use a new manifest path for each attempt. The signed manifest records every gate record hash, command-set hash, fixture-set hash, environment, dependency inventory, and artifact hash for later publication or audit. diff --git a/openkyrozen/app/container.py b/openkyrozen/app/container.py index 67b40890..f0d0d2cf 100644 --- a/openkyrozen/app/container.py +++ b/openkyrozen/app/container.py @@ -22,3 +22,4 @@ def close(self): self.runtime._technology_executor.shutdown(wait=False, cancel_futures=True) for workspace in self.runtime._workspaces.values(): workspace.adapters._BROWSER.close_all() + self.runtime.memory_bank.close() diff --git a/openkyrozen/interfaces/cli/rendering.py b/openkyrozen/interfaces/cli/rendering.py index e61cb7de..033a1dce 100644 --- a/openkyrozen/interfaces/cli/rendering.py +++ b/openkyrozen/interfaces/cli/rendering.py @@ -20,6 +20,12 @@ def _terminal_supports_unicode(self) -> bool: """Return True if the terminal can render Unicode box-drawing and special chars.""" if not self._IS_WINDOWS: return True + encoding = getattr(sys.stdout, "encoding", None) + if encoding: + try: + "✓".encode(encoding) + except (LookupError, UnicodeEncodeError): + return False # Windows Terminal, VS Code terminal, and ConEmu all set WT_SESSION if _os.environ.get("WT_SESSION") or _os.environ.get("TERM_PROGRAM") == "vscode": return True diff --git a/openkyrozen/interfaces/tui/turns.py b/openkyrozen/interfaces/tui/turns.py index 4e804ed2..06659bd7 100644 --- a/openkyrozen/interfaces/tui/turns.py +++ b/openkyrozen/interfaces/tui/turns.py @@ -260,6 +260,7 @@ def _attachment_prompt(self, text: str) -> str: def _run_submit(self, text: str, request_id: str, plan_acceptance: dict[str, Any] | None = None) -> None: runtime = self.agent approval_token = runtime._approval_callback.set(self._approval) + final_state, final_message = "ready", "Ready" try: inline = runtime.split_inline_command(text) if inline: @@ -273,10 +274,10 @@ def _run_submit(self, text: str, request_id: str, plan_acceptance: dict[str, Any else: self.prompt_api_key(request_id=request_id) self.status("waiting", "Provider setup required.", request_id) + final_state, final_message = "waiting", "Provider setup required." return sanitized, flagged = self._quiet_call(runtime._sanitize_input, text) if not sanitized.strip() and not self._staged_attachments: - self.status("ready", "Type a message before sending.", request_id) return metadata = runtime.memory_bank.store.list_events( "tui.chat_metadata", limit=1, workspace_id=runtime.interaction_workspace_id(), @@ -321,7 +322,6 @@ def _run_submit(self, text: str, request_id: str, plan_acceptance: dict[str, Any ], working_plan=runtime.tasks.working_plan_snapshot()) self.interaction(request_id) self.usage(request_id) - self.status("ready", "Ready", request_id) self.navigation(request_id) except runtime.ProviderUnavailableError as exc: self.emit("error", request_id, code=runtime.PROVIDER_UNAVAILABLE_CODE, @@ -334,6 +334,7 @@ def _run_submit(self, text: str, request_id: str, plan_acceptance: dict[str, Any with self._state_lock: self._busy = False self._workers.discard(threading.current_thread()) + self.status(final_state, final_message, request_id) def _features(self) -> list[dict[str, Any]]: diff --git a/openkyrozen/memory/service.py b/openkyrozen/memory/service.py index d8e2a291..fe9dc305 100644 --- a/openkyrozen/memory/service.py +++ b/openkyrozen/memory/service.py @@ -51,6 +51,11 @@ def scoped(self, *, user_id=None, workspace_id=None, session_id=None, file_scope view.file_scope_id = self.file_scope_id if file_scope_id is None else file_scope_id return view + def close(self) -> None: + close = getattr(self._client, "close", None) + if close is not None: + close() + @staticmethod def _kind_for_text(text: str) -> str: prefix = text.split(":", 1)[0].strip().upper() diff --git a/openkyrozen/memory/vector.py b/openkyrozen/memory/vector.py index 2d6c45f1..c8771358 100644 --- a/openkyrozen/memory/vector.py +++ b/openkyrozen/memory/vector.py @@ -11,3 +11,11 @@ def __init__(self, path): name="agent_logs", metadata={"description": "Derived index for durable OpenKyrozen memories"}) self.files = self.client.get_or_create_collection( name="agent_files", metadata={"description": "Derived index for workspace source files"}) + + def close(self): + self.client.close() + + def __del__(self): + client = getattr(self, "client", None) + if client is not None: + client.close() diff --git a/openkyrozen/persistence/database.py b/openkyrozen/persistence/database.py index 368638c5..d27582e6 100644 --- a/openkyrozen/persistence/database.py +++ b/openkyrozen/persistence/database.py @@ -215,6 +215,8 @@ def connection(self, *, timeout: float = 30, busy_timeout_ms: int | None = None) try: connection.execute(f"PRAGMA busy_timeout={busy_timeout_ms if busy_timeout_ms is not None else 30000}") connection.execute("PRAGMA foreign_keys=ON") + # WAL mode persists, but synchronous is connection-local. + connection.execute("PRAGMA synchronous=NORMAL") yield connection finally: connection.close() diff --git a/scripts/ci_changes.py b/scripts/ci_changes.py index b0913cc7..c7a5be4f 100644 --- a/scripts/ci_changes.py +++ b/scripts/ci_changes.py @@ -62,6 +62,12 @@ "builtin_skills/", "benchmarks/", ) +RELEASE_EVIDENCE_PATHS = { + ".github/workflows/ci.yml", + "benchmarks/v3_fixtures.py", + "benchmarks/v3_release_platforms.py", + "scripts/production_soak.py", +} def classify_paths(paths: set[str] | list[str] | tuple[str, ...]) -> set[str]: @@ -139,6 +145,11 @@ def classify_paths(paths: set[str] | list[str] | tuple[str, ...]) -> set[str]: return selected +def requires_release_evidence(paths: set[str] | None, release_candidate_sha: str = "") -> bool: + """Run the long production matrix only for its inputs or an explicit candidate.""" + return bool(release_candidate_sha) or (paths is not None and bool(paths & RELEASE_EVIDENCE_PATHS)) + + def changed_paths(root: Path, event: str, base: str, head: str) -> set[str] | None: """Read all changed paths, including both sides of renames; None means full CI.""" if not base or not head or set(base) == {"0"}: @@ -172,6 +183,7 @@ def main() -> int: parser.add_argument("--base", default="") parser.add_argument("--head", default="") parser.add_argument("--full", action="store_true") + parser.add_argument("--release-candidate-sha", default="") args = parser.parse_args() paths = None if args.full else changed_paths(Path.cwd(), args.event, args.base, args.head) @@ -180,6 +192,7 @@ def main() -> int: outputs["all"] = str("all" in checks).lower() outputs["core"] = str(bool(checks & {"python", "docs"})).lower() outputs["full"] = str(args.full or "all" in checks).lower() + outputs["release"] = str(requires_release_evidence(paths, args.release_candidate_sha)).lower() output_path = os.environ.get("GITHUB_OUTPUT") rendered = "".join(f"{name}={value}\n" for name, value in outputs.items()) diff --git a/scripts/production_soak.py b/scripts/production_soak.py index db637841..c2b2a2ad 100644 --- a/scripts/production_soak.py +++ b/scripts/production_soak.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse from concurrent.futures import ThreadPoolExecutor +from contextlib import closing import io import hashlib from importlib import metadata @@ -17,13 +18,107 @@ import tempfile import time import unittest -import psutil ROOT = Path(__file__).resolve().parents[1] sys.path[:0] = [str(ROOT), str(ROOT / 'tests')] from test_durable_tasks_gateway import DurableTaskGatewayTests +def _parse_process_table(output: str) -> dict[int, int]: + parents = {} + for line in output.splitlines(): + fields = line.split() + if len(fields) >= 2: + try: + parents[int(fields[0])] = int(fields[1]) + except ValueError: + continue + return parents + + +def _powershell(script: str) -> str: + shell = shutil.which('powershell') or shutil.which('pwsh') + if not shell: + raise RuntimeError('PowerShell is required for Windows process evidence') + return subprocess.run([shell, '-NoProfile', '-NonInteractive', '-Command', script], + capture_output=True, text=True, timeout=15, check=True).stdout + + +def _process_output(command: list[str]) -> tuple[str, int]: + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + try: + stdout, stderr = process.communicate(timeout=15) + except subprocess.TimeoutExpired: + process.kill() + process.communicate() + raise + if process.returncode: + raise subprocess.CalledProcessError(process.returncode, command, stdout, stderr) + return stdout, process.pid + + +def _process_table() -> dict[int, int]: + if platform.system() == 'Windows': + shell = shutil.which('powershell') or shutil.which('pwsh') + if not shell: + raise RuntimeError('PowerShell is required for Windows process evidence') + output, probe_pid = _process_output([shell, '-NoProfile', '-NonInteractive', '-Command', + "Get-CimInstance Win32_Process | Select-Object " + "@{Name='pid';Expression={$_.ProcessId}},@{Name='ppid';Expression={$_.ParentProcessId}} " + "| ConvertTo-Json -Compress"]) + rows = json.loads(output) if output.strip() else [] + if isinstance(rows, dict): + rows = [rows] + parents = {int(row['pid']): int(row['ppid']) for row in rows} + else: + output, probe_pid = _process_output(['ps', '-axo', 'pid=,ppid=']) + parents = _parse_process_table(output) + parents.pop(probe_pid, None) + return parents + + +def _descendants(pid: int, parents: dict[int, int] | None = None) -> set[int]: + parents = _process_table() if parents is None else parents + children = {} + for child, parent in parents.items(): + children.setdefault(parent, []).append(child) + found, pending = set(), list(children.get(pid, ())) + while pending: + child = pending.pop() + if child not in found: + found.add(child) + pending.extend(children.get(child, ())) + return found + + +def _process_snapshot(pid: int) -> dict[str, int]: + if platform.system() == 'Windows': + script = (f"$p=Get-Process -Id {int(pid)} -ErrorAction Stop; " + "[pscustomobject]@{rssBytes=[long]$p.WorkingSet64;threads=[int]$p.Threads.Count;" + "openHandles=[int]$p.HandleCount}|ConvertTo-Json -Compress") + result = json.loads(_powershell(script)) + return {'rss_bytes': int(result['rssBytes']), 'threads': int(result['threads']), + 'open_handles': int(result['openHandles'])} + proc = Path('/proc') / str(pid) + if platform.system() == 'Linux' and proc.is_dir(): + status = dict(line.split(':', 1) for line in (proc / 'status').read_text().splitlines() + if ':' in line) + return {'rss_bytes': int(status['VmRSS'].split()[0]) * 1024, + 'threads': int(status['Threads']), 'open_handles': len(list((proc / 'fd').iterdir()))} + rss = int(subprocess.run(['ps', '-p', str(pid), '-o', 'rss='], capture_output=True, + text=True, timeout=15, check=True).stdout.strip()) * 1024 + threads = max(0, len(subprocess.run(['ps', '-M', '-p', str(pid)], capture_output=True, + text=True, timeout=15, check=True).stdout.splitlines()) - 1) + handles = max(0, len(subprocess.run(['lsof', '-nP', '-p', str(pid)], capture_output=True, + text=True, timeout=15, check=True).stdout.splitlines()) - 1) + return {'rss_bytes': rss, 'threads': threads, 'open_handles': handles} + + +def _handle_limit(initial_handles: int) -> int: + # Windows starts with more runtime handles; bound growth rather than its baseline. + return initial_handles + 100 if platform.system() == 'Windows' else 99 + + def candidate_identity() -> dict: tracked = subprocess.check_output(["git", "ls-files", "-z"], cwd=ROOT).split(b"\0") tracked = sorted(set(tracked) | {b"tests/test_gateway_startup_logging.py", b"scripts/production_soak.py"}) @@ -36,7 +131,22 @@ def candidate_identity() -> dict: "candidate_source_sha256": digest.hexdigest(), "python": sys.version, "python_executable": sys.executable, "platform": platform.platform(), "dependencies": {name: metadata.version(name) for name in - ("psutil", "fastapi", "uvicorn", "pydantic", "chromadb")}} + ("fastapi", "uvicorn", "pydantic", "chromadb")}} + + +def _wait_for_task_completion(gateway, url, workspace: Path, session: str, + cycles: int, marker: str, task_id: str) -> None: + deadline = time.monotonic() + 40 + while time.monotonic() < deadline: + tasks = gateway._request(url, f'/api/v2/tasks?session_id={session}', timeout=35)['tasks'] + current = next(t for t in tasks if t['id'] == task_id) + if current['status'] in ('succeeded', 'failed', 'blocked'): + assert current['status'] == 'succeeded', current + assert (workspace / marker).read_text() == f'{session}:{cycles}' + assert all(t['session_id'] == session for t in tasks) + return + time.sleep(.1) + raise AssertionError('durable worker did not complete within 40 seconds') def run(seconds: float, log_dir: Path) -> dict: @@ -59,43 +169,39 @@ def start_gateway(workspace, db, skills, log_path): db = root / 'state.sqlite3' os.environ['KYROZEN_DB_PATH'] = str(root / 'test-driver.sqlite3') os.environ['KYROZEN_BROWSER_PROFILES'] = str(root / 'browser-profiles') + baseline_children = _descendants(os.getpid()) try: process, url = start_gateway(workspace, db, root / 'skills', log_path=log_dir / f'gateway-{restarts:03d}.log') - baseline_children = {p.pid for p in psutil.Process().children(recursive=True)} - initial_rss = psutil.Process(process.pid).memory_info().rss + initial = _process_snapshot(process.pid) + initial_rss = initial['rss_bytes'] generation_rss = initial_rss + generation_handles = initial['open_handles'] def task(session): marker = f'{session}-{cycles}.txt' + # Match SQLite's 30-second busy timeout, with a small HTTP margin. created = gateway._request(url, '/api/v2/tasks', method='POST', body={ 'description': f'soak {session} {cycles}', 'session_id': session, - 'action': 'write_file', 'args': f'{marker}|{session}:{cycles}'})['task'] - deadline = time.monotonic() + 15 - while time.monotonic() < deadline: - tasks = gateway._request(url, f'/api/v2/tasks?session_id={session}')['tasks'] - current = next(t for t in tasks if t['id'] == created['id']) - if current['status'] in ('succeeded', 'failed', 'blocked'): - assert current['status'] == 'succeeded', current - assert (workspace / marker).read_text() == f'{session}:{cycles}' - assert all(t['session_id'] == session for t in tasks) - return - time.sleep(.1) - raise AssertionError('durable worker did not complete within 15 seconds') + 'action': 'write_file', 'args': f'{marker}|{session}:{cycles}'}, timeout=35)['task'] + _wait_for_task_completion(gateway, url, workspace, session, cycles, marker, created['id']) while time.monotonic() - started < seconds: with ThreadPoolExecutor(max_workers=2) as pool: list(pool.map(task, ('soak-a', 'soak-b'))) - with sqlite3.connect(db) as connection: - count, unique = connection.execute('SELECT count(*), count(DISTINCT id) FROM tasks').fetchone() - assert count == unique == (cycles + 1) * 2, (count, unique, cycles) - assert connection.execute('PRAGMA integrity_check').fetchone()[0] == 'ok' - assert not connection.execute('PRAGMA foreign_key_check').fetchall() - current = psutil.Process(process.pid) - rss = current.memory_info().rss + with closing(sqlite3.connect(db)) as connection: + with connection: + count, unique = connection.execute('SELECT count(*), count(DISTINCT id) FROM tasks').fetchone() + assert count == unique == (cycles + 1) * 2, (count, unique, cycles) + assert connection.execute('PRAGMA integrity_check').fetchone()[0] == 'ok' + assert not connection.execute('PRAGMA foreign_key_check').fetchall() + current = _process_snapshot(process.pid) + rss = current['rss_bytes'] assert rss <= generation_rss + 256 * 1024 * 1024, 'gateway grew by more than 256 MiB in one generation' - measurements.append({'cycle': cycles, 'rss_bytes': rss, 'threads': current.num_threads(), - 'open_files': len(current.open_files())}) - assert current.num_threads() < 80, 'unbounded gateway thread growth' - assert len(current.open_files()) < 100, 'unbounded gateway file descriptor growth' + measurements.append({'cycle': cycles, 'handle_baseline': generation_handles, **current}) + assert current['threads'] < 80, 'unbounded gateway thread growth' + assert current['open_handles'] <= _handle_limit(generation_handles), ( + f"gateway handle growth exceeded its bound: baseline={generation_handles}, " + f"current={current['open_handles']}" + ) cycles += 1 if cycles % 6 == 0: suite = unittest.defaultTestLoader.loadTestsFromNames([ @@ -106,12 +212,14 @@ def task(session): result = unittest.TextTestRunner(stream=io.StringIO()).run(suite) assert result.wasSuccessful(), (result.errors, result.failures) if cycles % 12 == 0: - tracked = current.children(recursive=True) + tracked = _descendants(process.pid) gateway._stop_server(process) - assert all(not child.is_running() for child in tracked), 'orphan gateway child' + assert not tracked & set(_process_table()), 'orphan gateway child' process, url = start_gateway(workspace, db, root / 'skills', log_path=log_dir / f'gateway-{restarts + 1:03d}.log') - generation_rss = psutil.Process(process.pid).memory_info().rss + initial = _process_snapshot(process.pid) + generation_rss = initial['rss_bytes'] + generation_handles = initial['open_handles'] restarts += 1 for session in ('soak-a', 'soak-b'): tasks = gateway._request(url, f'/api/v2/tasks?session_id={session}')['tasks'] @@ -119,17 +227,33 @@ def task(session): print(json.dumps({'cycles': cycles, 'restarts': restarts, 'elapsed_seconds': round(time.monotonic()-started), 'rss_bytes': rss}), flush=True) time.sleep(min(5, max(0, seconds - (time.monotonic() - started)))) + tracked = _descendants(process.pid) gateway._stop_server(process) process = None - children = psutil.Process().children(recursive=True) - assert not [p.pid for p in children if p.pid not in baseline_children], 'orphan audit child' + assert not tracked & set(_process_table()), 'orphan gateway child' + assert not (_descendants(os.getpid()) - baseline_children), 'orphan audit child' except Exception as exc: # Retain disposable state and progress so a timeout is diagnosable. evidence = log_dir / 'failed-state' - shutil.copytree(root, evidence) + if process is not None: + try: + gateway._stop_server(process) + except Exception as cleanup_error: + exc.add_note(f'gateway cleanup failed: {type(cleanup_error).__name__}') + else: + process = None + failed_state = None + try: + shutil.copytree(root, evidence) + failed_state = str(evidence) + except OSError as copy_error: + exc.add_note(f'failed-state copy failed: {type(copy_error).__name__}') exc.soak_report = {'elapsed_seconds': round(time.monotonic()-started, 2), 'cycles': cycles, 'tasks': cycles * 2, 'restarts': restarts, - 'samples': measurements, 'startup_seconds': startup_seconds, 'startup_timeout_seconds': 60, 'failed_state': str(evidence)} + 'samples': measurements, 'startup_seconds': startup_seconds, + 'startup_timeout_seconds': 60} + if failed_state: + exc.soak_report['failed_state'] = failed_state raise finally: if process is not None: diff --git a/scripts/tui_workflow_acceptance.py b/scripts/tui_workflow_acceptance.py index b1ad1f9b..8b664d41 100644 --- a/scripts/tui_workflow_acceptance.py +++ b/scripts/tui_workflow_acceptance.py @@ -198,10 +198,14 @@ def response_count(rows): return sum(r.get('event') == 'response' for r in rows) def submit(text): - count = response_count(events()) + rows = events() + count = response_count(rows) + ready_count = sum(r.get('event') == 'status' and r.get('state') == 'ready' for r in rows) send(text.encode() + b'\r') wait(lambda rows: response_count(rows) > count, 'chat response') - wait(lambda rows: rows[-1].get('event') != 'status' or not rows[-1].get('busy'), 'turn completion') + if not text.startswith('/'): + wait(lambda rows: sum(r.get('event') == 'status' and r.get('state') == 'ready' + for r in rows) > ready_count, 'turn completion') try: wait(prompt('onboarding'), 'onboarding') @@ -258,7 +262,9 @@ def submit(text): drain(0.3) send(b'q') drain(0.5) - count = response_count(events()) + rows = events() + count = response_count(rows) + ready_count = sum(r.get('event') == 'status' and r.get('state') == 'ready' for r in rows) send(b'cancel audit\r') wait(lambda rows: any(r.get('event') == 'subagent' and r.get('agent', {}).get('status') == 'running' and r.get('agent', {}).get('assignment', {}).get('objective') == 'wait-for-cancellation' for r in rows), 'cancellable running agent') @@ -268,6 +274,8 @@ def submit(text): wait(lambda rows: any(r.get('event') == 'subagent' and r.get('agent', {}).get('status') == 'cancelled' for r in rows), 'agent cancellation') send(b'q') wait(lambda rows: response_count(rows) > count, 'cancelled work retained in final report') + wait(lambda rows: sum(r.get('event') == 'status' and r.get('state') == 'ready' + for r in rows) > ready_count, 'cancelled turn completion') for width, height in ((60, 20), (160, 45), (80, 24)): previous = len(terminal) fcntl.ioctl(master, termios.TIOCSWINSZ, struct.pack('HHHH', height, width, 0, 0)) @@ -277,6 +285,8 @@ def submit(text): assert 'APPROVAL REQUIRED' in plain and 'Choose a provider' in plain assert 'first conversation' in plain and 'second conversation' in plain assert '2.0.5 available' in plain and 'Run /update' in plain + send(b'\x1b') # Dismiss the expected error/agent dialog before the next command. + drain(0.3) send(b'/update\r') wait(lambda rows: any(r.get('event') == 'restart' for r in rows), 'update command requests restart') process.wait(timeout=10) @@ -285,7 +295,10 @@ def submit(text): print('TUI acceptance passed: startup update notice and command, onboarding, chat isolation, approvals, sub-agent updates/inspection/scrolling/cancellation, and 3 terminal sizes.') finally: if process.poll() is None: - process.kill() + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass process.wait() os.close(master) diff --git a/tests/test_bugfix_workflow.py b/tests/test_bugfix_workflow.py index 0a0fbf39..5d9fe9df 100644 --- a/tests/test_bugfix_workflow.py +++ b/tests/test_bugfix_workflow.py @@ -54,6 +54,8 @@ def setUp(self): self.tempdirs = [] def tearDown(self): + if main.memory_bank is not self.original_memory: + main.memory_bank.close() main.memory_bank = self.original_memory main.tasks = self.original_tasks main.learning_engine = self.original_learning @@ -67,7 +69,8 @@ def _run_fake_bug_turn(self, responses): directory = tempfile.TemporaryDirectory() self.tempdirs.append(directory) root = Path(directory.name) - main.memory_bank = MemoryBank(root / "state.sqlite3", workspace_id="bugfix", session_id="turn") + with patch.dict("os.environ", {"KYROZEN_DISABLE_VECTOR_INDEX": "1"}): + main.memory_bank = MemoryBank(root / "state.sqlite3", workspace_id="bugfix", session_id="turn") main.tasks = TaskManager(main.memory_bank.store, workspace_id="bugfix", session_id="turn") main._interaction_controller = InteractionController( main.memory_bank.store, workspace_id="bugfix", session_id="turn", diff --git a/tests/test_delegation.py b/tests/test_delegation.py index 1ddafc6b..74f75d7e 100644 --- a/tests/test_delegation.py +++ b/tests/test_delegation.py @@ -399,7 +399,7 @@ def invoke(run, **kwargs): return {"result": result()} coordinator = self.manager(invoke, 1) run = coordinator.spawn([brief()])[0] - self.assertTrue(entered.wait(2)) + self.assertTrue(entered.wait(10)) coordinator.cancel(run["run_id"]) release.set() coordinator.futures[run["run_id"]].result(timeout=5) @@ -813,7 +813,7 @@ def chat(inner, messages, model): self.runtime.get_provider = lambda _: SlowProvider() coordinator = self.runtime.delegation() run = coordinator.spawn([brief(profile="coder")])[0] - self.assertTrue(entered.wait(2)) + self.assertTrue(entered.wait(10)) coordinator.cancel(run["run_id"]) release.set() coordinator.futures[run["run_id"]].result(timeout=5) @@ -867,7 +867,7 @@ def chat(inner, messages, model): runtime.get_provider = lambda _: Provider() coordinator = runtime.delegation() coordinator.spawn([brief(profile="coder")]) - run = coordinator.wait(timeout=5)[0] + run = coordinator.wait(timeout=15)[0] self.assertEqual(run["status"], "succeeded") self.assertEqual(run["corrections"], 1) self.assertEqual((self.root / "marker.txt").read_text(), "ok") diff --git a/tests/test_distribution.py b/tests/test_distribution.py index df1889ba..a570a82e 100644 --- a/tests/test_distribution.py +++ b/tests/test_distribution.py @@ -38,6 +38,14 @@ def test_ci_change_routing_covers_known_inputs_and_fails_open(self): with self.subTest(label=label): self.assertEqual(classify_paths(paths), expected) + def test_release_evidence_matrix_is_selected_only_for_its_inputs_or_explicit_sha(self): + from scripts.ci_changes import requires_release_evidence + + self.assertTrue(requires_release_evidence({".github/workflows/ci.yml"})) + self.assertTrue(requires_release_evidence(set(), "a" * 40)) + self.assertFalse(requires_release_evidence({"docs/development.md"})) + self.assertFalse(requires_release_evidence(None)) + def test_ci_change_classifier_uses_complete_git_ranges_and_fails_open(self): from scripts.ci_changes import changed_paths @@ -112,6 +120,10 @@ def test_ci_keeps_core_and_full_python_coverage_without_duplicate_suites(self): self.assertIn("workflow_dispatch:", workflow) self.assertIn("name: CI result", workflow) self.assertIn("scripts/ci_changes.py", workflow) + self.assertIn("v3_release_candidate_sha", workflow) + self.assertIn("v3-platform-validation:", workflow) + self.assertIn("collect-r266:", workflow) + self.assertIn("PYTHONPATH=tests python -m unittest tests.test_durable_tasks_gateway", workflow) self.assertNotIn("matrix:\n python-version: [\"3.12\", \"3.13\"]", workflow) def test_one_shot_release_workflow_is_retired_after_v2_release(self): diff --git a/tests/test_durable_tasks_gateway.py b/tests/test_durable_tasks_gateway.py index 5f4b53b1..c3ba802f 100644 --- a/tests/test_durable_tasks_gateway.py +++ b/tests/test_durable_tasks_gateway.py @@ -23,7 +23,8 @@ def _free_port() -> int: return int(sock.getsockname()[1]) @staticmethod - def _request(base_url: str, path: str, *, method: str = "GET", body: dict | None = None) -> dict: + def _request(base_url: str, path: str, *, method: str = "GET", body: dict | None = None, + timeout: float = 5) -> dict: data = json.dumps(body).encode("utf-8") if body is not None else None request = urllib.request.Request( base_url + path, @@ -31,7 +32,8 @@ def _request(base_url: str, path: str, *, method: str = "GET", body: dict | None method=method, headers={"Content-Type": "application/json", "Authorization": "Bearer durable-test-token"}, ) - with urllib.request.urlopen(request, timeout=5) as response: + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + with opener.open(request, timeout=timeout) as response: return json.loads(response.read().decode("utf-8")) def _start_server(self, workspace: Path, db_path: Path, skills_path: Path, *, log_path: Path | None = None, startup_timeout: float = 20) -> tuple[subprocess.Popen, str]: diff --git a/tests/test_gateway_startup_logging.py b/tests/test_gateway_startup_logging.py index 043d2a1f..805325c1 100644 --- a/tests/test_gateway_startup_logging.py +++ b/tests/test_gateway_startup_logging.py @@ -45,12 +45,20 @@ def noisy_server(command, **kwargs): with tempfile.TemporaryDirectory() as directory: root = Path(directory) helper = gateway_tests.DurableTaskGatewayTests() + log_path = root / 'gateway.log' started = time.monotonic() - with patch('test_durable_tasks_gateway.subprocess.Popen', side_effect=noisy_server): - process, url = helper._start_server(root, root / 'state.db', root / 'skills') + def ready_after_output(_url, path): + self.assertEqual(path, '/api/health') + if 'startup-marker' not in log_path.read_text(encoding='utf-8'): + raise OSError('startup output is incomplete') + return {'status': 'ok'} + with patch('test_durable_tasks_gateway.subprocess.Popen', side_effect=noisy_server), \ + patch.object(helper, '_request', side_effect=ready_after_output): + process, _url = helper._start_server( + root, root / 'state.db', root / 'skills', log_path=log_path) try: self.assertLess(time.monotonic() - started, 5) - self.assertEqual(helper._request(url, '/api/health')['status'], 'ok') - self.assertIn('startup-marker', (root / 'gateway.log').read_text()) + self.assertIsNone(process.poll()) + self.assertIn('startup-marker', log_path.read_text(encoding='utf-8')) finally: helper._stop_server(process) diff --git a/tests/test_process_lifecycle.py b/tests/test_process_lifecycle.py index a00d6744..38ea14c2 100644 --- a/tests/test_process_lifecycle.py +++ b/tests/test_process_lifecycle.py @@ -7,6 +7,46 @@ class ProcessLifecycleTests(unittest.TestCase): + def test_application_close_releases_its_memory_index(self): + from pathlib import Path + from unittest.mock import patch + from openkyrozen.app.bootstrap import build_application + from openkyrozen.persistence.store import EventStore + from openkyrozen.memory.service import MemoryBank + + class Index: + memories = object() + files = object() + closed = False + + def close(self): + self.closed = True + + with tempfile.TemporaryDirectory() as root, patch.dict('os.environ', {'KYROZEN_DISABLE_VECTOR_INDEX': '1'}): + path = Path(root) / 'state.sqlite3' + index = Index() + memory = MemoryBank(path, store=EventStore(path), index=index) + app = build_application(memory=memory) + app.close() + self.assertTrue(index.closed) + + def test_unowned_chroma_index_releases_its_client(self): + import gc + from openkyrozen.memory.vector import ChromaIndex + + class Client: + closed = False + + def close(self): + self.closed = True + + index = ChromaIndex.__new__(ChromaIndex) + index.client = Client() + client = index.client + del index + gc.collect() + self.assertTrue(client.closed) + def test_foreground_shell_cancellation_terminates_process_group(self): from openkyrozen.tools import ToolAdapters with tempfile.TemporaryDirectory() as root: diff --git a/tests/test_production_soak.py b/tests/test_production_soak.py new file mode 100644 index 00000000..30fe22a7 --- /dev/null +++ b/tests/test_production_soak.py @@ -0,0 +1,69 @@ +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from scripts import production_soak + + +class ProductionSoakProcessTests(unittest.TestCase): + def test_candidate_dependency_inventory_does_not_require_psutil(self): + dependencies = production_soak.candidate_identity()["dependencies"] + + self.assertNotIn("psutil", dependencies) + + def test_process_table_tracks_descendants_across_multiple_generations(self): + parents = production_soak._parse_process_table("10 1\n11 10\n12 11\n20 1\n") + + self.assertEqual(production_soak._descendants(10, parents), {11, 12}) + self.assertEqual(production_soak._descendants(20, parents), set()) + + def test_windows_process_table_excludes_its_own_probe(self): + output = '[{"pid":10,"ppid":1},{"pid":11,"ppid":10},{"pid":99,"ppid":10}]' + with patch("scripts.production_soak.platform.system", return_value="Windows"), \ + patch("scripts.production_soak.shutil.which", return_value="powershell"), \ + patch("scripts.production_soak._process_output", return_value=(output, 99)): + self.assertEqual(production_soak._process_table(), {10: 1, 11: 10}) + + def test_windows_process_snapshot_maps_native_resource_counters(self): + with patch("scripts.production_soak.platform.system", return_value="Windows"), \ + patch("scripts.production_soak._powershell", return_value= + '{"rssBytes":4096,"threads":3,"openHandles":8}'): + self.assertEqual(production_soak._process_snapshot(10), + {"rss_bytes": 4096, "threads": 3, "open_handles": 8}) + + def test_handle_bound_uses_windows_growth_and_preserves_posix_cap(self): + with patch("scripts.production_soak.platform.system", return_value="Windows"): + self.assertEqual(production_soak._handle_limit(125), 225) + with patch("scripts.production_soak.platform.system", return_value="Linux"): + self.assertEqual(production_soak._handle_limit(125), 99) + + def test_durable_task_wait_allows_sqlite_busy_timeout_under_load(self): + class Gateway: + statuses = iter(("running", "succeeded")) + + def _request(self, *_args, **_kwargs): + return {"tasks": [{"id": "task-1", "session_id": "soak-a", + "status": next(self.statuses)}]} + + with tempfile.TemporaryDirectory() as directory: + workspace = Path(directory) + (workspace / "soak-a-29.txt").write_text("soak-a:29") + with patch("scripts.production_soak.time.monotonic", side_effect=(0, 16, 17)), \ + patch("scripts.production_soak.time.sleep"): + production_soak._wait_for_task_completion( + Gateway(), "http://localhost", workspace, "soak-a", 29, + "soak-a-29.txt", "task-1", + ) + + def test_current_platform_process_probe_returns_bounded_resource_counts(self): + snapshot = production_soak._process_snapshot(os.getpid()) + + self.assertGreater(snapshot["rss_bytes"], 0) + self.assertGreaterEqual(snapshot["threads"], 1) + self.assertGreater(snapshot["open_handles"], 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_sqlite_migrations.py b/tests/test_sqlite_migrations.py index 2fe98918..cfd0d7bc 100644 --- a/tests/test_sqlite_migrations.py +++ b/tests/test_sqlite_migrations.py @@ -110,6 +110,15 @@ def _sqlite(path): class SQLiteMigrationTests(unittest.TestCase): + def test_each_runtime_connection_uses_the_configured_wal_sync_mode(self): + with tempfile.TemporaryDirectory() as directory: + database = SQLiteDatabase(Path(directory) / "sync.sqlite3") + + with database.connection() as connection: + synchronous = connection.execute("PRAGMA synchronous").fetchone()[0] + + self.assertEqual(synchronous, 1) # NORMAL is configured by SQLiteDatabase initialization. + def _historical_database(self, path: Path, version: int, *, malformed: bool = False) -> None: db = sqlite3.connect(path) try: diff --git a/tests/test_tui_protocol.py b/tests/test_tui_protocol.py index 32cbebf8..7c455b67 100644 --- a/tests/test_tui_protocol.py +++ b/tests/test_tui_protocol.py @@ -191,6 +191,29 @@ def test_blank_tui_submission_does_not_persist_a_chat(self): self.backend._run_submit(" ", "blank-chat") chat.assert_not_called() + def test_completed_turn_emits_ready_after_busy_clears(self): + self.backend._busy = True + with patch.object(self.backend.agent, "llm_provider", object()), \ + patch.object(self.backend.agent, "_sanitize_input", return_value=("question", False)), \ + patch.object(self.backend.agent, "interaction_envelope", return_value={ + "pending_question": None, "pending_plan": None, + }), \ + patch.object(self.backend.agent, "is_plan_acceptance", return_value=False), \ + patch.object(self.backend.agent, "chat", return_value="answer"), \ + patch.object(self.backend.agent, "_clean_final_response", return_value="answer"), \ + patch.object(self.backend.agent, "_split_reply", return_value=("", "answer")), \ + patch.object(self.backend.agent.memory_bank, "add_log"), \ + patch.object(self.backend, "interaction"), \ + patch.object(self.backend, "usage"), \ + patch.object(self.backend, "navigation"): + self.backend._run_submit("question", "turn-ready") + + events = [json.loads(line) for line in self.output.getvalue().splitlines()] + ready = [event for event in events if event.get("event") == "status" + and event.get("state") == "ready"] + self.assertTrue(ready) + self.assertFalse(ready[-1]["busy"]) + def test_delete_chat_requires_confirmation_and_keeps_workspace_files(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -467,10 +490,11 @@ def test_structured_approval_keeps_long_arguments_in_redacted_scroll_chunks(self thread = threading.Thread(target=lambda: result.setdefault("approved", self.backend._approval(request))) thread.start() deadline = time.monotonic() + 1 - while "request_id" not in self.output.getvalue() and time.monotonic() < deadline: + while "approval_detail_complete" not in self.output.getvalue() and time.monotonic() < deadline: time.sleep(0.005) events = [json.loads(line) for line in self.output.getvalue().splitlines()] event = events[0] + self.assertTrue(any(item.get("event") == "approval_detail_complete" for item in events)) self.assertEqual(event["request_id"], request.request_id) encoded = event["args"] + "".join(item.get("text", "") for item in events[1:]) displayed = json.loads(encoded) diff --git a/tests/test_v3_release_gates.py b/tests/test_v3_release_gates.py new file mode 100644 index 00000000..57433c97 --- /dev/null +++ b/tests/test_v3_release_gates.py @@ -0,0 +1,278 @@ +import hashlib +import io +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch +from contextlib import redirect_stderr + +from benchmarks.v3_release_gates import GATE_IDS, REQUIRED_PLATFORMS, main, validate_evidence_bundle + + +class V3ReleaseGateTests(unittest.TestCase): + def create_bundle(self, root, *, mutate=None): + records = root / "records" + records.mkdir() + candidate = "a" * 40 + fixture_set = records / "fixtures.json" + fixture_set.write_text('{"frozen":"v3"}\n', encoding="utf-8") + for gate_id in GATE_IDS: + artifact = records / f"{gate_id}.log" + artifact.write_text(f"verified evidence for {gate_id}\n", encoding="utf-8") + commands = records / f"{gate_id}-commands.txt" + commands.write_text(f"commands for {gate_id}\n", encoding="utf-8") + record = { + "schema_version": 1, + "gate_id": gate_id, + "candidate_sha": candidate, + "command_set_sha256": hashlib.sha256(commands.read_bytes()).hexdigest(), + "fixture_set_sha256": hashlib.sha256(fixture_set.read_bytes()).hexdigest(), + "command_set_path": commands.name, + "fixture_set_path": fixture_set.name, + "environment": {"runner": "isolated-test", "os": "Linux", "python": "3.12"}, + "dependencies": {"python": "3.12"}, + "result": "passed", + "failure_class": None, + "evidence": [{"path": artifact.name, "sha256": hashlib.sha256(artifact.read_bytes()).hexdigest()}], + } + if gate_id == "R266": + record["platform_results"] = [] + for platform in REQUIRED_PLATFORMS: + platform_artifact = records / f"{platform}.log" + platform_artifact.write_text(f"verified evidence for {platform}\n", encoding="utf-8") + record["evidence"].append({ + "path": platform_artifact.name, + "sha256": hashlib.sha256(platform_artifact.read_bytes()).hexdigest(), + }) + platform_result = { + "platform": platform, "unit": "passed", "integration": "passed", + "acceptance": "passed", "soak_seconds": 1800, + "tui_interactive_acceptance": "passed", + "evidence_paths": [platform_artifact.name], + } + if platform.startswith("windows-"): + platform_result.update(unit="not_run", integration="not_run", + supported_features="passed", + tui_interactive_acceptance="not_run_posix_pty_unavailable") + record["platform_results"].append(platform_result) + if mutate: + mutate(gate_id, record, artifact) + record_path = records / f"{gate_id}.json" + record_path.write_text(json.dumps(record), encoding="utf-8") + record_path.with_suffix(".json.asc").write_text("test-signature", encoding="utf-8") + return records, candidate + + def test_complete_signed_bundle_produces_candidate_bound_manifest(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + records, candidate = self.create_bundle(root) + manifest = validate_evidence_bundle(records, candidate, verify_signature=lambda *_: True) + self.assertEqual(manifest["candidate_sha"], candidate) + self.assertEqual(set(manifest["gates"]), set(GATE_IDS)) + self.assertEqual(manifest["result"], "passed") + self.assertEqual(manifest["gates"]["R250"]["signature_sha256"], + hashlib.sha256(b"test-signature").hexdigest()) + + def test_missing_gate_blocks_publication(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + records, candidate = self.create_bundle(root) + (records / "R250.json").unlink() + with self.assertRaisesRegex(ValueError, "missing gate record"): + validate_evidence_bundle(records, candidate, verify_signature=lambda *_: True) + manifest = root / "manifest.json" + with patch("benchmarks.v3_release_gates._current_candidate", return_value=candidate), \ + redirect_stderr(io.StringIO()): + status = main(["--repo", str(root), "--evidence-dir", str(records), + "--manifest", str(manifest)]) + self.assertEqual(status, 1) + self.assertFalse(manifest.exists()) + self.assertFalse(manifest.with_name("manifest.json.asc").exists()) + + def test_main_rejects_evidence_and_manifest_paths_inside_candidate_checkout(self): + for evidence_inside in (True, False): + with self.subTest(evidence_inside=evidence_inside), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + repo = root / "repo" + repo.mkdir() + records, candidate = self.create_bundle(repo if evidence_inside else root) + evidence_dir = records + manifest = (root / "manifest.json") if evidence_inside else (repo / "manifest.json") + with patch("benchmarks.v3_release_gates._current_candidate", return_value=candidate), \ + patch("benchmarks.v3_release_gates._gpg_verify", return_value=True), \ + redirect_stderr(io.StringIO()): + status = main(["--repo", str(repo), "--evidence-dir", str(evidence_dir), + "--manifest", str(manifest)]) + self.assertEqual(status, 1) + self.assertFalse(manifest.exists()) + self.assertFalse(manifest.with_name("manifest.json.asc").exists()) + + def test_failed_manifest_publish_leaves_no_unsigned_manifest(self): + from types import SimpleNamespace + import os + from benchmarks.v3_release_gates import _write_signed_manifest + + with tempfile.TemporaryDirectory() as directory: + manifest = Path(directory).resolve() / "manifest.json" + signature = manifest.with_name("manifest.json.asc") + real_link = os.link + + def fail_manifest_link(source, destination): + if Path(destination) == manifest: + raise OSError("manifest destination is locked") + return real_link(source, destination) + + def create_signature(command, **_kwargs): + Path(command[command.index("--output") + 1]).write_text("signature", encoding="utf-8") + return SimpleNamespace(returncode=0) + + with patch("benchmarks.v3_release_gates.subprocess.run", side_effect=create_signature), \ + patch("benchmarks.v3_release_gates._gpg_verify", return_value=True), \ + patch("benchmarks.v3_release_gates.os.link", side_effect=fail_manifest_link): + with self.assertRaisesRegex(OSError, "manifest destination is locked"): + _write_signed_manifest({"result": "passed"}, manifest) + self.assertFalse(manifest.exists()) + self.assertFalse(signature.exists()) + + def test_manifest_writer_does_not_replace_outputs_created_after_preflight(self): + from types import SimpleNamespace + from benchmarks.v3_release_gates import _write_signed_manifest + + with tempfile.TemporaryDirectory() as directory: + manifest = Path(directory) / "manifest.json" + signature = manifest.with_name("manifest.json.asc") + + def concurrent_publish(command, **_kwargs): + Path(command[command.index("--output") + 1]).write_text("new signature", encoding="utf-8") + manifest.write_text("other manifest", encoding="utf-8") + signature.write_text("other signature", encoding="utf-8") + return SimpleNamespace(returncode=0) + + with patch("benchmarks.v3_release_gates.subprocess.run", side_effect=concurrent_publish), \ + patch("benchmarks.v3_release_gates._gpg_verify", return_value=True): + with self.assertRaises(FileExistsError): + _write_signed_manifest({"result": "passed"}, manifest) + self.assertEqual(manifest.read_text(encoding="utf-8"), "other manifest") + self.assertEqual(signature.read_text(encoding="utf-8"), "other signature") + + def test_manifest_writer_preserves_existing_output_pair(self): + from benchmarks.v3_release_gates import _write_signed_manifest + + with tempfile.TemporaryDirectory() as directory: + manifest = Path(directory) / "manifest.json" + signature = manifest.with_name("manifest.json.asc") + manifest.write_text("old manifest", encoding="utf-8") + signature.write_text("old signature", encoding="utf-8") + with self.assertRaisesRegex(ValueError, "already exists"): + _write_signed_manifest({"result": "passed"}, manifest) + self.assertEqual(manifest.read_text(encoding="utf-8"), "old manifest") + self.assertEqual(signature.read_text(encoding="utf-8"), "old signature") + + def test_rejects_stale_candidate_failed_gate_or_bad_signature(self): + for mutate, expected, verifier in ( + (lambda gate, record, _: record.update(candidate_sha="b" * 40), "candidate SHA", lambda *_: True), + (lambda gate, record, _: record.update(result="failed", failure_class="test_failure"), "did not pass", lambda *_: True), + (lambda *_: None, "signature verification failed", lambda *_: False), + ): + with self.subTest(expected=expected), tempfile.TemporaryDirectory() as directory: + records, candidate = self.create_bundle(Path(directory), mutate=mutate) + with self.assertRaisesRegex(ValueError, expected): + validate_evidence_bundle(records, candidate, verify_signature=verifier) + + def test_rejects_changed_or_escaping_evidence_artifacts(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + records, candidate = self.create_bundle(root) + (records / "R250.log").write_text("changed after signing", encoding="utf-8") + with self.assertRaisesRegex(ValueError, "evidence hash mismatch"): + validate_evidence_bundle(records, candidate, verify_signature=lambda *_: True) + + def test_rejects_absolute_artifact_paths_and_malformed_results(self): + for mutate, expected in ( + (lambda gate, record, _: record["evidence"][0].update(path="/outside/artifact"), "relative path"), + (lambda gate, record, _: record.update(result={"passed": True}), "invalid result"), + ): + with self.subTest(expected=expected), tempfile.TemporaryDirectory() as directory: + records, candidate = self.create_bundle(Path(directory), mutate=mutate) + with self.assertRaisesRegex(ValueError, expected): + validate_evidence_bundle(records, candidate, verify_signature=lambda *_: True) + with tempfile.TemporaryDirectory() as directory: + records, candidate = self.create_bundle(Path(directory)) + (records / "R250-commands.txt").write_text("altered command set", encoding="utf-8") + with self.assertRaisesRegex(ValueError, "command-set hash mismatch"): + validate_evidence_bundle(records, candidate, verify_signature=lambda *_: True) + with tempfile.TemporaryDirectory() as directory: + records, candidate = self.create_bundle(Path(directory), mutate=lambda gate, record, _: ( + record["evidence"][0].update(path="../outside") if gate == "R250" else None + )) + with self.assertRaisesRegex(ValueError, "escapes evidence directory"): + validate_evidence_bundle(records, candidate, verify_signature=lambda *_: True) + + def test_production_gate_requires_all_six_platforms_and_thirty_minute_soaks(self): + for mutation, expected in ( + (lambda record: record["platform_results"].pop(), "platform matrix"), + (lambda record: record["platform_results"][0].update(soak_seconds=1799), "30-minute soak"), + (lambda record: record["platform_results"][0].update(integration="failed"), "platform validation failed"), + (lambda record: record["platform_results"][0].update(platform=[]), "platform matrix"), + ): + with self.subTest(expected=expected), tempfile.TemporaryDirectory() as directory: + records, candidate = self.create_bundle(Path(directory), mutate=lambda gate, record, _: mutation(record) + if gate == "R266" else None) + with self.assertRaisesRegex(ValueError, expected): + validate_evidence_bundle(records, candidate, verify_signature=lambda *_: True) + + def test_production_gate_allows_windows_supported_feature_acceptance(self): + with tempfile.TemporaryDirectory() as directory: + records, candidate = self.create_bundle( + Path(directory), + mutate=lambda gate, record, _: [ + item.update(unit="not_run", integration="not_run", supported_features="passed") + for item in record["platform_results"] if item["platform"].startswith("windows-") + ] if gate == "R266" else None, + ) + manifest = validate_evidence_bundle(records, candidate, verify_signature=lambda *_: True) + windows = [item for item in manifest["gates"]["R266"]["platform_results"] + if item["platform"].startswith("windows-")] + self.assertTrue(all(item["unit"] == "not_run" + and item["integration"] == "not_run" + and item["supported_features"] == "passed" + and item["tui_interactive_acceptance"] == "not_run_posix_pty_unavailable" + for item in windows)) + + def test_production_gate_requires_explicit_windows_tui_unavailable_status(self): + for mutation in ( + lambda item: item.pop("tui_interactive_acceptance"), + lambda item: item.update(tui_interactive_acceptance="failed"), + ): + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory: + records, candidate = self.create_bundle(Path(directory), mutate=lambda gate, record, _: [ + mutation(item) for item in record["platform_results"] + if item["platform"].startswith("windows-") + ] if gate == "R266" else None) + with self.assertRaisesRegex(ValueError, "platform validation failed"): + validate_evidence_bundle(records, candidate, verify_signature=lambda *_: True) + + def test_each_production_platform_requires_linked_hashed_evidence(self): + for mutation in ( + lambda record: record["platform_results"][0].pop("evidence_paths"), + lambda record: record["platform_results"][0].update(evidence_paths=["missing.log"]), + ): + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory: + records, candidate = self.create_bundle(Path(directory), mutate=lambda gate, record, _: ( + mutation(record) if gate == "R266" else None + )) + with self.assertRaisesRegex(ValueError, "platform evidence"): + validate_evidence_bundle(records, candidate, verify_signature=lambda *_: True) + + def test_requires_failure_class_for_nonpassing_evidence(self): + with tempfile.TemporaryDirectory() as directory: + records, candidate = self.create_bundle(Path(directory), mutate=lambda gate, record, _: ( + record.update(result="inconclusive", failure_class=None) if gate == "R250" else None + )) + with self.assertRaisesRegex(ValueError, "failure class"): + validate_evidence_bundle(records, candidate, verify_signature=lambda *_: True) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_v3_release_platforms.py b/tests/test_v3_release_platforms.py new file mode 100644 index 00000000..21485491 --- /dev/null +++ b/tests/test_v3_release_platforms.py @@ -0,0 +1,147 @@ +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +from benchmarks.v3_release_platforms import REQUIRED_PLATFORMS, collect_platform_evidence +from benchmarks.v3_release_gates import _check_record + + +class V3ReleasePlatformsTests(unittest.TestCase): + def _matrix(self, root, *, mutate=None): + results = root / "results" + results.mkdir() + candidate = "a" * 40 + for platform in REQUIRED_PLATFORMS: + directory = results / f"v3-platform-{platform}" + directory.mkdir() + names = ("unit.log", "acceptance.log", "soak.json", "tui-acceptance.log") + if platform.startswith("windows-"): + names = ("acceptance.log", "soak.json", "supported-features.log", "tui-acceptance.log") + for name in names: + (directory / name).write_text(f"{platform}:{name}\n", encoding="utf-8") + record = { + "schema_version": 1, + "candidate_sha": candidate, + "platform": platform, + "unit": "passed", + "integration": "passed", + "acceptance": "passed", + "tui_interactive_acceptance": "passed", + "soak_seconds": 1800, + "environment": {"runner": "github", "os": platform.split("-")[0], + "python": f"3.{platform[-2:]}.0"}, + "dependencies": {"python": f"3.{platform[-2:]}.0"}, + "evidence_paths": ["unit.log", "acceptance.log", "tui-acceptance.log", "soak.json"], + } + if platform.startswith("windows-"): + record["evidence_paths"] = ["acceptance.log", "soak.json", "supported-features.log", "tui-acceptance.log"] + record.update(unit="not_run", integration="not_run", supported_features="passed", + tui_interactive_acceptance="not_run_posix_pty_unavailable") + if mutate: + mutate(platform, record) + (directory / "platform.json").write_text(json.dumps(record), encoding="utf-8") + commands = root / "commands.yml" + commands.write_text("commands: test, acceptance, soak\n", encoding="utf-8") + return results, commands, candidate + + def test_collects_exact_candidate_matrix_into_unsigned_r266_record(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + results, commands, candidate = self._matrix(root) + output = root / "evidence" + record = collect_platform_evidence(results, output, candidate, commands) + self.assertEqual(record["gate_id"], "R266") + self.assertEqual(record["candidate_sha"], candidate) + self.assertEqual({row["platform"] for row in record["platform_results"]}, + set(REQUIRED_PLATFORMS)) + self.assertEqual(len(record["evidence"]), 30) + self.assertEqual(record["command_set_sha256"], hashlib.sha256( + (output / record["command_set_path"]).read_bytes()).hexdigest()) + self.assertEqual(record["fixture_set_sha256"], hashlib.sha256( + (output / record["fixture_set_path"]).read_bytes()).hexdigest()) + self.assertTrue((output / "R266.json").is_file()) + self.assertFalse((output / "R266.json.asc").exists()) + self.assertEqual(len(_check_record(record, "R266", candidate, output.resolve())), 30) + + def test_windows_uses_supported_feature_acceptance_without_full_python_suites(self): + def supported_windows(platform, record): + if platform.startswith("windows-"): + record["unit"] = "not_run" + record["integration"] = "not_run" + record["supported_features"] = "passed" + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + results, commands, candidate = self._matrix(root, mutate=supported_windows) + record = collect_platform_evidence(results, root / "evidence", candidate, commands) + windows = [row for row in record["platform_results"] + if row["platform"].startswith("windows-")] + self.assertEqual(len(windows), 2) + self.assertTrue(all(row["unit"] == "not_run" + and row["integration"] == "not_run" + and row["supported_features"] == "passed" + and row["tui_interactive_acceptance"] == "not_run_posix_pty_unavailable" + for row in windows)) + + def test_rejects_incomplete_or_stale_platform_results(self): + for mutate, expected in ( + (lambda platform, record: record.update(candidate_sha="b" * 40), "candidate SHA"), + (lambda platform, record: record.update(soak_seconds=1799), "30-minute soak"), + (lambda platform, record: record.update(unit="failed"), "platform validation"), + (lambda platform, record: record.pop("tui_interactive_acceptance") + if platform.startswith("windows-") else None, "supported-feature acceptance"), + (lambda platform, record: record["environment"].update(os="Windows"), "OS or Python"), + ): + with self.subTest(expected=expected), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + results, commands, candidate = self._matrix(root, mutate=mutate) + with self.assertRaisesRegex(ValueError, expected): + collect_platform_evidence(results, root / "evidence", candidate, commands) + + def test_rejects_missing_artifact_and_existing_output(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + results, commands, candidate = self._matrix(root) + (results / f"v3-platform-{REQUIRED_PLATFORMS[0]}" / "unit.log").unlink() + with self.assertRaisesRegex(ValueError, "artifact"): + collect_platform_evidence(results, root / "evidence", candidate, commands) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + results, commands, candidate = self._matrix(root) + output = root / "evidence" + output.mkdir() + with self.assertRaisesRegex(ValueError, "already exists"): + collect_platform_evidence(results, output, candidate, commands) + + def test_rejects_artifact_paths_that_escape_the_platform_bundle(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + results, commands, candidate = self._matrix(root) + result_path = results / f"v3-platform-{REQUIRED_PLATFORMS[0]}" / "platform.json" + result = json.loads(result_path.read_text(encoding="utf-8")) + result["evidence_paths"][0] = "../../outside.log" + result_path.write_text(json.dumps(result), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "escapes"): + collect_platform_evidence(results, root / "evidence", candidate, commands) + + def test_rejects_symlinked_platform_metadata(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + results, commands, candidate = self._matrix(root) + platform_dir = results / f"v3-platform-{REQUIRED_PLATFORMS[0]}" + metadata = platform_dir / "platform.json" + external = root / "external.json" + external.write_text(metadata.read_text(encoding="utf-8"), encoding="utf-8") + metadata.unlink() + try: + metadata.symlink_to(external) + except OSError as exc: + self.skipTest(f"platform does not permit symlink creation: {exc}") + with self.assertRaisesRegex(ValueError, "cannot be a symlink"): + collect_platform_evidence(results, root / "evidence", candidate, commands) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_windows_rendering.py b/tests/test_windows_rendering.py new file mode 100644 index 00000000..b737554f --- /dev/null +++ b/tests/test_windows_rendering.py @@ -0,0 +1,17 @@ +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +from openkyrozen.interfaces.cli.rendering import _terminal_supports_unicode + + +class WindowsRenderingTests(unittest.TestCase): + def test_terminal_claim_does_not_override_stdout_encoding(self): + terminal = SimpleNamespace(_IS_WINDOWS=True) + with patch.dict("os.environ", {"WT_SESSION": "1", "TERM_PROGRAM": "vscode"}), \ + patch("openkyrozen.interfaces.cli.rendering.sys.stdout", SimpleNamespace(encoding="cp1252")): + self.assertFalse(_terminal_supports_unicode(terminal)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tui/main_test.go b/tui/main_test.go index ac54a64c..b2b403e1 100644 --- a/tui/main_test.go +++ b/tui/main_test.go @@ -16,9 +16,15 @@ import ( "charm.land/lipgloss/v2" ) -func TestOnboardingDistinguishesNewAndUpdatedInstall(t *testing.T) { +func setTestHome(t *testing.T) { + t.Helper() home := t.TempDir() t.Setenv("HOME", home) + t.Setenv("USERPROFILE", home) +} + +func TestOnboardingDistinguishesNewAndUpdatedInstall(t *testing.T) { + setTestHome(t) fresh := initialModel("", true) if fresh.onboardingKind != "new" { t.Fatalf("fresh install was classified as %q", fresh.onboardingKind) @@ -46,7 +52,7 @@ func TestOnboardingDistinguishesNewAndUpdatedInstall(t *testing.T) { } func TestOnboardingCompletionPersistsVersionAndReturnsToChat(t *testing.T) { - t.Setenv("HOME", t.TempDir()) + setTestHome(t) m := initialModel("", true) m.screen, m.onboardingKind = screenOnboarding, "new" m.handleBackendEvent(backendEvent{"event": "onboarding_complete", "kind": "new"}) @@ -276,7 +282,7 @@ func TestUpdatingLocksInputButKeepsEmergencyQuit(t *testing.T) { } func TestToolDetailsAreHiddenByDefaultAndPersistWhenEnabled(t *testing.T) { - t.Setenv("HOME", t.TempDir()) + setTestHome(t) m := initialModel("", true) m.handleBackendEvent(backendEvent{"event": "tool_receipt", "receipt": map[string]any{ "action": "read_file", "result": "secret file contents", "success": true, @@ -316,7 +322,7 @@ func TestSettingsSlashCommandOpensLocalScreen(t *testing.T) { } func TestSettingsExposeInteractionControlsAndKeepCommandDispatch(t *testing.T) { - t.Setenv("HOME", t.TempDir()) + setTestHome(t) m := initialModel("", true) m.width, m.height, m.screen = 100, 30, screenSettings m.applyInteraction(map[string]any{