feat(evidence): retain opt-in test execution receipts and dependency observations

.github/workflows/trace-adapters-tests.yml

@@ -11,6 +11,7 @@ on:
       - "integrations/google-adk/**"
       - "integrations/openshell/**"
       - "integrations/openai-agents/**"
+      - "scripts/test_evidence.py"
       - "requirements/adapters*.in"
       - "requirements/adapters*.txt"
       - "requirements/sdk-*.in"
@@ -27,6 +28,7 @@ on:
       - "integrations/google-adk/**"
       - "integrations/openshell/**"
       - "integrations/openai-agents/**"
+      - "scripts/test_evidence.py"
       - "requirements/adapters*.in"
       - "requirements/adapters*.txt"
       - "requirements/sdk-*.in"
@@ -169,7 +171,22 @@ jobs:
       - name: Test released LangChain and LangGraph interoperability
         run: |
           pip install --require-hashes -r requirements/sdk-langchain.txt
-          python -m pytest integrations/langchain/test_langgraph_interop.py -q
+          python scripts/test_evidence.py capture \
+            --integration integrations/langchain \
+            --output "$RUNNER_TEMP/langchain-evidence" \
+            --evidence-class synthetic \
+            -- python -m pytest integrations/langchain/test_langgraph_interop.py -q
+      - name: Report declaration and execution consistency
+        if: always()
+        run: python scripts/test_evidence.py check "$RUNNER_TEMP/langchain-evidence"
+      - name: Retain LangChain execution receipt and log
+        if: always()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: langchain-execution-evidence
+          path: ${{ runner.temp }}/langchain-evidence/
+          retention-days: 30
+          if-no-files-found: warn
 
   llamaindex-workflow-adapter:
     runs-on: ubuntu-latest

.github/workflows/validate.yml

@@ -36,6 +36,8 @@ jobs:
         run: |
           python scripts/validate_compatibility.py
           python -m pytest tests/test_validate_compatibility.py -q
+      - name: Test execution evidence receipts
+        run: python -m pytest tests/test_test_evidence.py -q
       - name: Check generated integration index
         run: python scripts/generate_integration_index.py --check
       - name: Test and check generated Marketplace catalog

docs/test-evidence.md

@@ -0,0 +1,51 @@
+# Retained execution evidence
+
+`tested_against` is a declaration. A version floor check alone does not show
+that a command ran against those versions. The opt-in recorder links an
+explicit command to a source revision, installed distribution versions,
+manifest digest, timestamps, exit status and the digest of its retained log.
+
+Run from a clean checkout in the same environment that runs the test:
+
+```sh
+python scripts/test_evidence.py capture \
+  --integration integrations/<name> \
+  --output /tmp/integration-evidence \
+  --evidence-class synthetic \
+  -- python -m pytest integrations/<name>/tests -q
+python scripts/test_evidence.py check /tmp/integration-evidence
+```
+
+The output directory must be new and outside the checkout (or ignored by Git).
+Use a unique directory per run. Only explicitly requested Python commands run;
+the tool does not discover or execute vendor code. Dependencies are installed
+by the caller from existing released-package locks. It records installed
+metadata, so tests that switch environments or install dependencies internally
+are outside its supported model. Use the same interpreter throughout.
+
+In an integration's existing GitHub workflow, wrap its current Python test
+command, then upload the entire directory with `if: always()` and an explicit
+retention period. Both `receipt.json` and `output.log` are needed. The recorder
+captures the Actions run URL when running in Actions. A timeout or failed test
+still produces a receipt and a failing command status. Do not upload logs that
+contain credentials or sensitive workload data.
+
+The check returns `consistent` only for a clean matching revision, unchanged
+manifest, exact declared/installed versions, successful command and matching
+log digest. Missing evidence, undeclared dependencies, failed runs and stale
+or mismatched content fail the check. A declaration without a receipt remains
+declaration-only; absence does not prove the integration was never tested.
+Evidence expiry after artifact retention likewise requires a new run.
+
+Evidence class is explicitly chosen: synthetic fixtures, captured evidence,
+or live operation. The tool cannot independently verify that classification.
+Receipts are unsigned local observations, not attestations: a digest catches
+accidental substitution but not someone rewriting both receipt and log.
+Review command coverage and trusted CI provenance separately. A successful
+command can run zero tests; the tool does not infer conformance or completeness.
+Neither a consistent receipt nor a passing test changes community/verified
+tiers, compatibility floors, or maintainer approval requirements.
+
+The recorder tests exercise real subprocess success, failure and timeout plus
+missing evidence, changed manifests, mismatched versions/revisions and log
+substitution. Run `python -m pytest tests/test_test_evidence.py -q`.

integrations/langchain/integration.yaml

@@ -19,4 +19,4 @@ trace_roles:
   - record-producer
 trace_conformance_level: 0
 tested_against:
-  agentrust-trace: "0.9.0"
+  agentrust-trace: "0.11.0"

scripts/test_evidence.py

@@ -0,0 +1,133 @@
+"""Opt-in execution receipts. Local evidence is not tier approval or attestation."""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import importlib.metadata
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+from datetime import datetime, timezone
+
+import yaml
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def digest(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def capture(root: Path, integration: str, output: Path, command: list[str],
+            evidence_class: str, timeout: int = 600) -> dict:
+    """Run an explicitly selected command in this interpreter's environment."""
+    if timeout <= 0:
+        raise ValueError("Timeout must be positive")
+    manifest = (root / integration / "integration.yaml").resolve()
+    manifest.relative_to(root.resolve())
+    declared = yaml.safe_load(manifest.read_text()).get("tested_against") or {}
+    observed = {}
+    for package in declared:
+        try:
+            observed[package] = importlib.metadata.version(package)
+        except importlib.metadata.PackageNotFoundError:
+            observed[package] = None
+    revision = subprocess.check_output(
+        ["git", "rev-parse", "HEAD"], cwd=root, text=True,
+    ).strip()
+    dirty = bool(subprocess.check_output(
+        ["git", "status", "--porcelain", "--untracked-files=all"], cwd=root, text=True,
+    ).strip())
+    if not command or command[0] != sys.executable:
+        raise ValueError("Command must start with this Python interpreter (use -- python ...)")
+    output.mkdir(parents=True, exist_ok=False)
+    started = datetime.now(timezone.utc).isoformat()
+    log = output / "output.log"
+    timed_out = False
+    with log.open("wb") as stream:
+        try:
+            result = subprocess.run(command, cwd=root, stdout=stream,
+                                    stderr=subprocess.STDOUT, timeout=timeout, check=False)
+            returncode = result.returncode
+        except subprocess.TimeoutExpired:
+            returncode = None
+            timed_out = True
+    receipt = {
+        "version": 1, "integration": integration, "revision": revision,
+        "dirty": dirty, "manifest_sha256": digest(manifest),
+        "declared_packages": declared, "observed_packages": observed,
+        "command": command, "evidence_class": evidence_class,
+        "started_at": started, "finished_at": datetime.now(timezone.utc).isoformat(),
+        "returncode": returncode, "timed_out": timed_out,
+        "log_sha256": digest(log),
+        "run_url": (f"https://github.com/{os.environ['GITHUB_REPOSITORY']}/actions/runs/"
+                    f"{os.environ['GITHUB_RUN_ID']}" if os.environ.get("GITHUB_ACTIONS") == "true"
+                    else None),
+    }
+    (output / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n")
+    return receipt
+
+
+def validate(root: Path, directory: Path) -> list[str]:
+    """Check a retained receipt against the current checkout and log bytes."""
+    try:
+        receipt = json.loads((directory / "receipt.json").read_text())
+        manifest = (root / receipt["integration"] / "integration.yaml").resolve()
+        manifest.relative_to(root.resolve())
+        declared = yaml.safe_load(manifest.read_text()).get("tested_against") or {}
+        revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip()
+        checks = {
+            "unsupported receipt version": receipt["version"] == 1,
+            "source revision mismatch": receipt["revision"] == revision,
+            "dirty source snapshot": receipt["dirty"] is False,
+            "current checkout is dirty": not subprocess.check_output(
+                ["git", "status", "--porcelain", "--untracked-files=all"], cwd=root, text=True,
+            ).strip(),
+            "manifest changed": receipt["manifest_sha256"] == digest(manifest),
+            "declared versions changed": receipt["declared_packages"] == declared,
+            "missing tested_against declaration": bool(declared),
+            "installed versions differ": receipt["observed_packages"] == declared,
+            "command did not pass": receipt["returncode"] == 0 and receipt["timed_out"] is False,
+            "log digest mismatch": receipt["log_sha256"] == digest(directory / "output.log"),
+            "invalid evidence class": receipt["evidence_class"] in {"synthetic", "captured", "live"},
+            "missing command": isinstance(receipt["command"], list) and bool(receipt["command"]),
+        }
+        for field in ("started_at", "finished_at"):
+            if datetime.fromisoformat(receipt[field]).tzinfo is None:
+                raise ValueError("Timestamp requires timezone")
+        if datetime.fromisoformat(receipt["finished_at"]) < datetime.fromisoformat(receipt["started_at"]):
+            raise ValueError("Finish precedes start")
+        return [message for message, passed in checks.items() if not passed]
+    except (KeyError, ValueError, TypeError, OSError) as exc:
+        return [f"invalid or missing evidence: {exc}"]
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    sub = parser.add_subparsers(dest="mode", required=True)
+    record = sub.add_parser("capture")
+    record.add_argument("--integration", required=True)
+    record.add_argument("--output", type=Path, required=True)
+    record.add_argument("--evidence-class", choices=["synthetic", "captured", "live"], required=True)
+    record.add_argument("--timeout", type=int, default=600)
+    record.add_argument("command", nargs=argparse.REMAINDER)
+    check = sub.add_parser("check")
+    check.add_argument("directory", type=Path)
+    args = parser.parse_args()
+    if args.mode == "check":
+        errors = validate(ROOT, args.directory)
+        print(json.dumps({"status": "inconsistent" if errors else "consistent", "errors": errors}))
+        return int(bool(errors))
+    command = args.command
+    if command[:1] == ["--"]:
+        command = command[1:]
+    if command[:1] == ["python"]:
+        command[0] = sys.executable
+    receipt = capture(ROOT, args.integration, args.output, command, args.evidence_class, args.timeout)
+    return int(receipt["returncode"] != 0 or receipt["timed_out"])
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tests/test_test_evidence.py

@@ -0,0 +1,76 @@
+"""Receipts distinguish execution, consistency and missing evidence."""
+import importlib.util
+import json
+from pathlib import Path
+import subprocess
+import sys
+
+import pytest
+
+spec = importlib.util.spec_from_file_location("test_evidence", Path(__file__).parents[1] / "scripts/test_evidence.py")
+evidence = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(evidence)
+
+
+@pytest.fixture
+def checkout(tmp_path):
+    root = tmp_path / "repo"
+    root.mkdir()
+    subprocess.run(["git", "init", "-q", str(root)], check=True)
+    directory = root / "integrations/demo"
+    directory.mkdir(parents=True)
+    (directory / "integration.yaml").write_text(
+        f"tested_against:\n  pytest: '{evidence.importlib.metadata.version('pytest')}'\n")
+    subprocess.run(["git", "add", "."], cwd=root, check=True)
+    subprocess.run(["git", "-c", "user.name=Test", "-c", "user.email=test@example.invalid",
+                    "commit", "-qm", "fixture"], cwd=root, check=True)
+    return root, tmp_path / "receipt"
+
+
+def run(checkout, code="print('executed')", timeout=600):
+    root, output = checkout
+    return evidence.capture(root, "integrations/demo", output,
+                            [sys.executable, "-c", code], "synthetic", timeout)
+
+
+def test_success_and_log_binding(checkout):
+    run(checkout)
+    root, output = checkout
+    assert evidence.validate(root, output) == []
+    (output / "output.log").write_text("substituted")
+    assert "log digest mismatch" in evidence.validate(root, output)
+
+
+@pytest.mark.parametrize("code", ["raise SystemExit(7)", "import time; time.sleep(5)"])
+def test_failure_and_timeout_retained(checkout, code):
+    receipt = run(checkout, code, timeout=1)
+    assert receipt["returncode"] != 0
+    assert "command did not pass" in evidence.validate(*checkout)
+
+
+@pytest.mark.parametrize("field,value,error", [
+    ("revision", "0" * 40, "source revision mismatch"),
+    ("dirty", True, "dirty source snapshot"),
+    ("observed_packages", {}, "installed versions differ"),
+    ("returncode", None, "command did not pass"),
+])
+def test_mismatch(checkout, field, value, error):
+    receipt = run(checkout)
+    receipt[field] = value
+    (checkout[1] / "receipt.json").write_text(json.dumps(receipt))
+    assert error in evidence.validate(*checkout)
+
+
+def test_missing_and_changed_manifest(checkout):
+    assert evidence.validate(*checkout)
+    run(checkout)
+    (checkout[0] / "integrations/demo/integration.yaml").write_text("tested_against: {}")
+    assert "manifest changed" in evidence.validate(*checkout)
+
+
+def test_reject_foreign_interpreter_and_path(checkout):
+    root, output = checkout
+    with pytest.raises(ValueError):
+        evidence.capture(root, "integrations/demo", output, ["sh", "-c", "true"], "synthetic")
+    with pytest.raises(ValueError):
+        evidence.capture(root, "../outside", output, [sys.executable, "-c", "pass"], "synthetic")