From 8388924adfb25bae045136946c70f063f9c99cd2 Mon Sep 17 00:00:00 2001 From: Srinivasa Consultants <78673153+dinakarjs@users.noreply.github.com> Date: Thu, 22 Oct 2026 01:45:30 +0530 Subject: [PATCH 1/7] feat(evidence): retain execution receipts with version and log binding Signed-off-by: Srinivasa Consultants --- scripts/test_evidence.py | 133 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 scripts/test_evidence.py diff --git a/scripts/test_evidence.py b/scripts/test_evidence.py new file mode 100644 index 00000000..13064838 --- /dev/null +++ b/scripts/test_evidence.py @@ -0,0 +1,133 @@ +"""Opt-in execution receipts. Local evidence is not tier approval or attestation.""" +from __future__ import annotations + +import argparse +import hashlib +import importlib.metadata +import json +import os +from pathlib import Path +import subprocess +import sys +from datetime import datetime, timezone + +import yaml + +ROOT = Path(__file__).resolve().parents[1] + + +def digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def capture(root: Path, integration: str, output: Path, command: list[str], + evidence_class: str, timeout: int = 600) -> dict: + """Run an explicitly selected command in this interpreter's environment.""" + if timeout <= 0: + raise ValueError("Timeout must be positive") + manifest = (root / integration / "integration.yaml").resolve() + manifest.relative_to(root.resolve()) + declared = yaml.safe_load(manifest.read_text()).get("tested_against") or {} + observed = {} + for package in declared: + try: + observed[package] = importlib.metadata.version(package) + except importlib.metadata.PackageNotFoundError: + observed[package] = None + revision = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=root, text=True, + ).strip() + dirty = bool(subprocess.check_output( + ["git", "status", "--porcelain", "--untracked-files=all"], cwd=root, text=True, + ).strip()) + if not command or command[0] != sys.executable: + raise ValueError("Command must start with this Python interpreter (use -- python ...)") + output.mkdir(parents=True, exist_ok=False) + started = datetime.now(timezone.utc).isoformat() + log = output / "output.log" + timed_out = False + with log.open("wb") as stream: + try: + result = subprocess.run(command, cwd=root, stdout=stream, + stderr=subprocess.STDOUT, timeout=timeout, check=False) + returncode = result.returncode + except subprocess.TimeoutExpired: + returncode = None + timed_out = True + receipt = { + "version": 1, "integration": integration, "revision": revision, + "dirty": dirty, "manifest_sha256": digest(manifest), + "declared_packages": declared, "observed_packages": observed, + "command": command, "evidence_class": evidence_class, + "started_at": started, "finished_at": datetime.now(timezone.utc).isoformat(), + "returncode": returncode, "timed_out": timed_out, + "log_sha256": digest(log), + "run_url": (f"https://github.com/{os.environ['GITHUB_REPOSITORY']}/actions/runs/" + f"{os.environ['GITHUB_RUN_ID']}" if os.environ.get("GITHUB_ACTIONS") == "true" + else None), + } + (output / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") + return receipt + + +def validate(root: Path, directory: Path) -> list[str]: + """Check a retained receipt against the current checkout and log bytes.""" + try: + receipt = json.loads((directory / "receipt.json").read_text()) + manifest = (root / receipt["integration"] / "integration.yaml").resolve() + manifest.relative_to(root.resolve()) + declared = yaml.safe_load(manifest.read_text()).get("tested_against") or {} + revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip() + checks = { + "unsupported receipt version": receipt["version"] == 1, + "source revision mismatch": receipt["revision"] == revision, + "dirty source snapshot": receipt["dirty"] is False, + "current checkout is dirty": not subprocess.check_output( + ["git", "status", "--porcelain", "--untracked-files=all"], cwd=root, text=True, + ).strip(), + "manifest changed": receipt["manifest_sha256"] == digest(manifest), + "declared versions changed": receipt["declared_packages"] == declared, + "missing tested_against declaration": bool(declared), + "installed versions differ": receipt["observed_packages"] == declared, + "command did not pass": receipt["returncode"] == 0 and receipt["timed_out"] is False, + "log digest mismatch": receipt["log_sha256"] == digest(directory / "output.log"), + "invalid evidence class": receipt["evidence_class"] in {"synthetic", "captured", "live"}, + "missing command": isinstance(receipt["command"], list) and bool(receipt["command"]), + } + for field in ("started_at", "finished_at"): + if datetime.fromisoformat(receipt[field]).tzinfo is None: + raise ValueError("Timestamp requires timezone") + if datetime.fromisoformat(receipt["finished_at"]) < datetime.fromisoformat(receipt["started_at"]): + raise ValueError("Finish precedes start") + return [message for message, passed in checks.items() if not passed] + except (KeyError, ValueError, TypeError, OSError) as exc: + return [f"invalid or missing evidence: {exc}"] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="mode", required=True) + record = sub.add_parser("capture") + record.add_argument("--integration", required=True) + record.add_argument("--output", type=Path, required=True) + record.add_argument("--evidence-class", choices=["synthetic", "captured", "live"], required=True) + record.add_argument("--timeout", type=int, default=600) + record.add_argument("command", nargs=argparse.REMAINDER) + check = sub.add_parser("check") + check.add_argument("directory", type=Path) + args = parser.parse_args() + if args.mode == "check": + errors = validate(ROOT, args.directory) + print(json.dumps({"status": "inconsistent" if errors else "consistent", "errors": errors})) + return int(bool(errors)) + command = args.command + if command[:1] == ["--"]: + command = command[1:] + if command[:1] == ["python"]: + command[0] = sys.executable + receipt = capture(ROOT, args.integration, args.output, command, args.evidence_class, args.timeout) + return int(receipt["returncode"] != 0 or receipt["timed_out"]) + + +if __name__ == "__main__": + raise SystemExit(main()) From 88756fd59bb3501e74f14c4388f525bfd544d0bf Mon Sep 17 00:00:00 2001 From: Srinivasa Consultants <78673153+dinakarjs@users.noreply.github.com> Date: Thu, 22 Oct 2026 01:45:36 +0530 Subject: [PATCH 2/7] feat(evidence): retain execution receipts with version and log binding Signed-off-by: Srinivasa Consultants --- tests/test_test_evidence.py | 76 +++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 tests/test_test_evidence.py diff --git a/tests/test_test_evidence.py b/tests/test_test_evidence.py new file mode 100644 index 00000000..020446b1 --- /dev/null +++ b/tests/test_test_evidence.py @@ -0,0 +1,76 @@ +"""Receipts distinguish execution, consistency and missing evidence.""" +import importlib.util +import json +from pathlib import Path +import subprocess +import sys + +import pytest + +spec = importlib.util.spec_from_file_location("test_evidence", Path(__file__).parents[1] / "scripts/test_evidence.py") +evidence = importlib.util.module_from_spec(spec) +spec.loader.exec_module(evidence) + + +@pytest.fixture +def checkout(tmp_path): + root = tmp_path / "repo" + root.mkdir() + subprocess.run(["git", "init", "-q", str(root)], check=True) + directory = root / "integrations/demo" + directory.mkdir(parents=True) + (directory / "integration.yaml").write_text( + f"tested_against:\n pytest: '{evidence.importlib.metadata.version('pytest')}'\n") + subprocess.run(["git", "add", "."], cwd=root, check=True) + subprocess.run(["git", "-c", "user.name=Test", "-c", "user.email=test@example.invalid", + "commit", "-qm", "fixture"], cwd=root, check=True) + return root, tmp_path / "receipt" + + +def run(checkout, code="print('executed')", timeout=600): + root, output = checkout + return evidence.capture(root, "integrations/demo", output, + [sys.executable, "-c", code], "synthetic", timeout) + + +def test_success_and_log_binding(checkout): + run(checkout) + root, output = checkout + assert evidence.validate(root, output) == [] + (output / "output.log").write_text("substituted") + assert "log digest mismatch" in evidence.validate(root, output) + + +@pytest.mark.parametrize("code", ["raise SystemExit(7)", "import time; time.sleep(5)"]) +def test_failure_and_timeout_retained(checkout, code): + receipt = run(checkout, code, timeout=1) + assert receipt["returncode"] != 0 + assert "command did not pass" in evidence.validate(*checkout) + + +@pytest.mark.parametrize("field,value,error", [ + ("revision", "0" * 40, "source revision mismatch"), + ("dirty", True, "dirty source snapshot"), + ("observed_packages", {}, "installed versions differ"), + ("returncode", None, "command did not pass"), +]) +def test_mismatch(checkout, field, value, error): + receipt = run(checkout) + receipt[field] = value + (checkout[1] / "receipt.json").write_text(json.dumps(receipt)) + assert error in evidence.validate(*checkout) + + +def test_missing_and_changed_manifest(checkout): + assert evidence.validate(*checkout) + run(checkout) + (checkout[0] / "integrations/demo/integration.yaml").write_text("tested_against: {}") + assert "manifest changed" in evidence.validate(*checkout) + + +def test_reject_foreign_interpreter_and_path(checkout): + root, output = checkout + with pytest.raises(ValueError): + evidence.capture(root, "integrations/demo", output, ["sh", "-c", "true"], "synthetic") + with pytest.raises(ValueError): + evidence.capture(root, "../outside", output, [sys.executable, "-c", "pass"], "synthetic") From 718ede35c242b2794d176b4296aa0867b2a685de Mon Sep 17 00:00:00 2001 From: Srinivasa Consultants <78673153+dinakarjs@users.noreply.github.com> Date: Thu, 22 Oct 2026 01:45:43 +0530 Subject: [PATCH 3/7] feat(evidence): retain execution receipts with version and log binding Signed-off-by: Srinivasa Consultants --- docs/test-evidence.md | 51 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 docs/test-evidence.md diff --git a/docs/test-evidence.md b/docs/test-evidence.md new file mode 100644 index 00000000..8652b408 --- /dev/null +++ b/docs/test-evidence.md @@ -0,0 +1,51 @@ +# Retained execution evidence + +`tested_against` is a declaration. A version floor check alone does not show +that a command ran against those versions. The opt-in recorder links an +explicit command to a source revision, installed distribution versions, +manifest digest, timestamps, exit status and the digest of its retained log. + +Run from a clean checkout in the same environment that runs the test: + +```sh +python scripts/test_evidence.py capture \ + --integration integrations/ \ + --output /tmp/integration-evidence \ + --evidence-class synthetic \ + -- python -m pytest integrations//tests -q +python scripts/test_evidence.py check /tmp/integration-evidence +``` + +The output directory must be new and outside the checkout (or ignored by Git). +Use a unique directory per run. Only explicitly requested Python commands run; +the tool does not discover or execute vendor code. Dependencies are installed +by the caller from existing released-package locks. It records installed +metadata, so tests that switch environments or install dependencies internally +are outside its supported model. Use the same interpreter throughout. + +In an integration's existing GitHub workflow, wrap its current Python test +command, then upload the entire directory with `if: always()` and an explicit +retention period. Both `receipt.json` and `output.log` are needed. The recorder +captures the Actions run URL when running in Actions. A timeout or failed test +still produces a receipt and a failing command status. Do not upload logs that +contain credentials or sensitive workload data. + +The check returns `consistent` only for a clean matching revision, unchanged +manifest, exact declared/installed versions, successful command and matching +log digest. Missing evidence, undeclared dependencies, failed runs and stale +or mismatched content fail the check. A declaration without a receipt remains +declaration-only; absence does not prove the integration was never tested. +Evidence expiry after artifact retention likewise requires a new run. + +Evidence class is explicitly chosen: synthetic fixtures, captured evidence, +or live operation. The tool cannot independently verify that classification. +Receipts are unsigned local observations, not attestations: a digest catches +accidental substitution but not someone rewriting both receipt and log. +Review command coverage and trusted CI provenance separately. A successful +command can run zero tests; the tool does not infer conformance or completeness. +Neither a consistent receipt nor a passing test changes community/verified +tiers, compatibility floors, or maintainer approval requirements. + +The recorder tests exercise real subprocess success, failure and timeout plus +missing evidence, changed manifests, mismatched versions/revisions and log +substitution. Run `python -m pytest tests/test_test_evidence.py -q`. From 3e0261524d78b5bddfbe50c756a45c3cfe1334c0 Mon Sep 17 00:00:00 2001 From: Srinivasa Consultants <78673153+dinakarjs@users.noreply.github.com> Date: Thu, 22 Oct 2026 01:45:50 +0530 Subject: [PATCH 4/7] feat(evidence): retain execution receipts with version and log binding Signed-off-by: Srinivasa Consultants --- .github/workflows/validate.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 684b6c44..326a8423 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -36,6 +36,8 @@ jobs: run: | python scripts/validate_compatibility.py python -m pytest tests/test_validate_compatibility.py -q + - name: Test execution evidence receipts + run: python -m pytest tests/test_test_evidence.py -q - name: Check generated integration index run: python scripts/generate_integration_index.py --check - name: Test and check generated Marketplace catalog From 044ca19ceeda253cf6f1ef749293fe6f8d6d2569 Mon Sep 17 00:00:00 2001 From: Srinivasa Consultants <78673153+dinakarjs@users.noreply.github.com> Date: Thu, 22 Oct 2026 01:45:57 +0530 Subject: [PATCH 5/7] feat(evidence): retain execution receipts with version and log binding Signed-off-by: Srinivasa Consultants --- .github/workflows/trace-adapters-tests.yml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/trace-adapters-tests.yml b/.github/workflows/trace-adapters-tests.yml index 0d53e742..7ba826e7 100644 --- a/.github/workflows/trace-adapters-tests.yml +++ b/.github/workflows/trace-adapters-tests.yml @@ -11,6 +11,7 @@ on: - "integrations/google-adk/**" - "integrations/openshell/**" - "integrations/openai-agents/**" + - "scripts/test_evidence.py" - "requirements/adapters*.in" - "requirements/adapters*.txt" - "requirements/sdk-*.in" @@ -27,6 +28,7 @@ on: - "integrations/google-adk/**" - "integrations/openshell/**" - "integrations/openai-agents/**" + - "scripts/test_evidence.py" - "requirements/adapters*.in" - "requirements/adapters*.txt" - "requirements/sdk-*.in" @@ -169,7 +171,23 @@ jobs: - name: Test released LangChain and LangGraph interoperability run: | pip install --require-hashes -r requirements/sdk-langchain.txt - python -m pytest integrations/langchain/test_langgraph_interop.py -q + python scripts/test_evidence.py capture \ + --integration integrations/langchain \ + --output "$RUNNER_TEMP/langchain-evidence" \ + --evidence-class synthetic \ + -- python -m pytest integrations/langchain/test_langgraph_interop.py -q + - name: Report declaration and execution consistency + if: always() + continue-on-error: true + run: python scripts/test_evidence.py check "$RUNNER_TEMP/langchain-evidence" + - name: Retain LangChain execution receipt and log + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: langchain-execution-evidence + path: ${{ runner.temp }}/langchain-evidence/ + retention-days: 30 + if-no-files-found: warn llamaindex-workflow-adapter: runs-on: ubuntu-latest From 8c4f77c0e7430b1a9793691b34d2e4456bc3b3e8 Mon Sep 17 00:00:00 2001 From: Srinivasa Consultants <78673153+dinakarjs@users.noreply.github.com> Date: Thu, 22 Oct 2026 01:49:01 +0530 Subject: [PATCH 6/7] fix(evidence): align LangChain declaration with observed CI and enforce receipt Signed-off-by: Srinivasa Consultants --- integrations/langchain/integration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrations/langchain/integration.yaml b/integrations/langchain/integration.yaml index 8d883ec9..3ae4b8d8 100644 --- a/integrations/langchain/integration.yaml +++ b/integrations/langchain/integration.yaml @@ -19,4 +19,4 @@ trace_roles: - record-producer trace_conformance_level: 0 tested_against: - agentrust-trace: "0.9.0" + agentrust-trace: "0.11.0" From d213b840da8e601951dbaa3e7c2bf79eb538090f Mon Sep 17 00:00:00 2001 From: Srinivasa Consultants <78673153+dinakarjs@users.noreply.github.com> Date: Thu, 22 Oct 2026 01:49:08 +0530 Subject: [PATCH 7/7] fix(evidence): align LangChain declaration with observed CI and enforce receipt Signed-off-by: Srinivasa Consultants --- .github/workflows/trace-adapters-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/trace-adapters-tests.yml b/.github/workflows/trace-adapters-tests.yml index 7ba826e7..76ab4b6d 100644 --- a/.github/workflows/trace-adapters-tests.yml +++ b/.github/workflows/trace-adapters-tests.yml @@ -178,7 +178,6 @@ jobs: -- python -m pytest integrations/langchain/test_langgraph_interop.py -q - name: Report declaration and execution consistency if: always() - continue-on-error: true run: python scripts/test_evidence.py check "$RUNNER_TEMP/langchain-evidence" - name: Retain LangChain execution receipt and log if: always()