From 8074bdbfdde7087da9606aad226f5f63595b4c2d Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 19 Oct 2026 14:56:58 -0700 Subject: [PATCH 1/9] feat(task): play an outside agent through a stand-in agent in its place An episode's outside agent is now played through a model-less stand-in agent that the task's own deploy_agent or install_agent deploys in its place, so the steps that load into, run in, collect from or check the agent's sandbox run unchanged, and the loop's bash runs there as the agent's user in its folder. A task that uses no agent sandbox gets the same stand-in served in-process. Co-Authored-By: Claude Opus 5.5 --- .../agentenv_protocol/a2a_agent/__init__.py | 22 ++ .../agentenv_protocol/a2a_agent/extensions.py | 88 ++++++ .../agentenv_protocol/a2a_agent/stand_in.py | 214 +++++++++++++++ .../tests/test_a2a_stand_in.py | 185 +++++++++++++ src/agent_env/a2a_agent/a2a_agent.py | 3 +- src/agent_env/a2a_agent/stand_in/Dockerfile | 13 + src/agent_env/a2a_agent/stand_in/__init__.py | 67 +++++ .../a2a_agent/stand_in/a2a_server.py | 37 +++ src/agent_env/task/episode/rules.py | 57 ++-- src/agent_env/task/episode/run.py | 250 +++++++++++++----- src/agent_env/task/episode/spec.py | 38 ++- src/agent_env/task/episode/stand_in.py | 89 +++++++ src/agent_env/task/episode/steps.py | 91 +++---- .../task_step/task_steps/install_agent.py | 45 ++++ .../agent/stand_in_modal_vm_test.py | 81 ++++++ .../env/envs/server_env_local_test.py | 51 +++- .../env/envs/server_env_modal_test.py | 2 +- tst/unit/a2a_agent/stand_in_test.py | 38 +++ .../test_reachability_contributions.py | 2 +- tst/unit/store/aws/test_shared_credentials.py | 12 +- tst/unit/task/episode/conftest.py | 12 + tst/unit/task/episode/run_test.py | 111 +++++++- tst/unit/task/episode/spec_test.py | 126 +++++---- tst/unit/task_step/install_agent_test.py | 80 ++++++ 24 files changed, 1475 insertions(+), 239 deletions(-) create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/stand_in.py create mode 100644 packages/agentenv-protocol/tests/test_a2a_stand_in.py create mode 100644 src/agent_env/a2a_agent/stand_in/Dockerfile create mode 100644 src/agent_env/a2a_agent/stand_in/__init__.py create mode 100644 src/agent_env/a2a_agent/stand_in/a2a_server.py create mode 100644 src/agent_env/task/episode/stand_in.py create mode 100644 tst/integration/agent/stand_in_modal_vm_test.py create mode 100644 tst/unit/a2a_agent/stand_in_test.py create mode 100644 tst/unit/task/episode/conftest.py diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py index 93141797..605fe39d 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py @@ -5,6 +5,7 @@ from .extensions import ( AGENT_CONFIG_V1, ATTRIBUTION_PROBE_V1, + DRIVER_V1, INSTALL_V1, MCP_CONFIG_V1, PEER_AGENTS_V1, @@ -19,6 +20,16 @@ ContextEventsTrajectoryRequest, ContextObjectTrajectoryRequest, ContextTrajectoryRequest, + DriverClaimRequest, + DriverExecRequest, + DriverExecResponse, + DriverNextRequest, + DriverNextResponse, + DriverOutputRequest, + DriverOutputResponse, + DriverReplyRequest, + DriverSkill, + DriverTurn, ExtensionActivation, ExtensionConfiguration, ExtensionDefinition, @@ -117,6 +128,7 @@ __all__ = [ "AGENT_CONFIG_V1", "ATTRIBUTION_PROBE_V1", + "DRIVER_V1", "INSTALL_V1", "MCP_CONFIG_V1", "PEER_AGENTS_V1", @@ -139,6 +151,16 @@ "ContextEventsTrajectoryRequest", "ContextObjectTrajectoryRequest", "ContextTrajectoryRequest", + "DriverClaimRequest", + "DriverExecRequest", + "DriverExecResponse", + "DriverNextRequest", + "DriverNextResponse", + "DriverOutputRequest", + "DriverOutputResponse", + "DriverReplyRequest", + "DriverSkill", + "DriverTurn", "DataPart", "DefaultExtensionHandlers", "ExtensionActivation", diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py index a35c19bb..82df2bd9 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py @@ -287,6 +287,71 @@ class TriggerDecideRequest(ExtensionRequest): env_triggers: dict[str, dict[str, Any]] | None = None +DRIVER_TOKEN_MIN_LENGTH = 22 + + +class DriverClaimRequest(ExtensionRequest): + token: str = Field(min_length=DRIVER_TOKEN_MIN_LENGTH) + + +class DriverNextRequest(ExtensionRequest): + token: str + after: int = Field(default=0, ge=0) + + +class DriverSkill(ExtensionResponse): + name: str + description: str + path: str | None = None + + +class DriverTurn(ExtensionResponse): + task_id: str + context_id: str + text: str + system_prompt: str | None = None + skills: list[DriverSkill] = Field(default_factory=list) + + +class DriverNextResponse(ExtensionResponse): + version: int + turns: list[DriverTurn] + + +class DriverReplyRequest(ExtensionRequest): + token: str + task_id: str + text: str | None = None + error: str | None = None + trajectory: list[dict[str, Any]] = Field(default_factory=list) + tool_call_count: int = Field(default=0, ge=0) + + @model_validator(mode="after") + def _text_or_error(self) -> DriverReplyRequest: + if (self.text is None) == (self.error is None): + raise ValueError("a reply carries exactly one of text and error") + return self + + +class DriverExecRequest(ExtensionRequest): + token: str + command: str + + +class DriverExecResponse(ExtensionResponse): + exec_id: str + + +class DriverOutputRequest(ExtensionRequest): + token: str + exec_id: str + + +class DriverOutputResponse(ExtensionResponse): + done: bool + output: str | None = None + + def _tuple(values: Iterable[str]) -> tuple[str, ...]: return tuple(dict.fromkeys(values)) @@ -1167,6 +1232,28 @@ def _configure_trajectory(*, live: bool = False) -> ExtensionConfiguration: configuration_validator=_configure_install, ) +DRIVER_V1 = ExtensionDefinition( + uri="urn:agentenv:driver/v1", + description="Hand the agent's turns to a driver outside its sandbox, and run its commands where the agent works.", + endpoint="/ext/driver", + core_operations={ + "claim": _op("claim", "POST", "/ext/driver/claim", RUNTIME, request_model=DriverClaimRequest), + "next": _op( + "next", "POST", "/ext/driver/next", RUNTIME, + request_model=DriverNextRequest, response_model=DriverNextResponse, + ), + "reply": _op("reply", "POST", "/ext/driver/reply", RUNTIME, request_model=DriverReplyRequest), + "exec": _op( + "exec", "POST", "/ext/driver/exec", RUNTIME, + request_model=DriverExecRequest, response_model=DriverExecResponse, + ), + "output": _op( + "output", "POST", "/ext/driver/output", RUNTIME, + request_model=DriverOutputRequest, response_model=DriverOutputResponse, + ), + }, +) + STANDARD_EXTENSIONS: Mapping[str, ExtensionDefinition] = MappingProxyType( { extension.uri: extension @@ -1180,6 +1267,7 @@ def _configure_trajectory(*, live: bool = False) -> ExtensionConfiguration: ATTRIBUTION_PROBE_V1, TRIGGERS_V1, INSTALL_V1, + DRIVER_V1, ) } ) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/stand_in.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/stand_in.py new file mode 100644 index 00000000..6d5b5eb5 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/stand_in.py @@ -0,0 +1,214 @@ +"""The stand-in: an agent with no model, played by a driver outside its sandbox. + +Each message it gets is an open turn until the driver replies to it. Through ``urn:agentenv:driver/v1`` the driver +reads the open turns, replies to them and runs commands where the stand-in works, as it; its first ``claim`` sets the +token every later call carries. A turn its sender cancels closes without a reply. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import hmac +import os +import signal +import uuid +from dataclasses import dataclass, field +from pathlib import Path + +from starlette.exceptions import HTTPException + +from .extensions import ( + DRIVER_V1, + MCP_CONFIG_V1, + SKILL_CONFIG_V1, + TRAJECTORY_V1, + BundleSkillRequest, + DriverClaimRequest, + DriverExecRequest, + DriverExecResponse, + DriverNextRequest, + DriverNextResponse, + DriverOutputRequest, + DriverOutputResponse, + DriverReplyRequest, + DriverSkill, + DriverTurn, + InlineSkillRequest, + extension, +) +from .framework import AgentEnvAgent, AgentIdentity, a2a_agent +from .tasks.v1 import AgentConfig, TaskRequest, TaskResult, TextPart, Usage +from ..transfers import download + +LONG_POLL_SECONDS = 25.0 +COMMAND_TIMEOUT_SECONDS = 3600 +SKILLS_DIR = Path.home() / "skills" +TRAJECTORY_FORMAT = "agentenv-stand-in/v1" + + +class StandInConfig(AgentConfig): + system_prompt: str | None = None + + +@dataclass +class _Turn: + task_id: str + context_id: str + text: str + system_prompt: str | None + skills: list[DriverSkill] + reply: asyncio.Future[DriverReplyRequest] = field( + default_factory=lambda: asyncio.get_running_loop().create_future() + ) + + +IDENTITY = AgentIdentity( + name="agentenv-stand-in", + description="Stands in for an agent played from outside AgentEnv: a driver answers its turns.", + version="1.0.0", +) +EXTENSIONS = (MCP_CONFIG_V1, TRAJECTORY_V1, DRIVER_V1) + + +@a2a_agent(identity=IDENTITY, config=StandInConfig, extensions=EXTENSIONS) +class StandInAgent(AgentEnvAgent): + """``shell=False`` is a stand-in for an agent that works in no sandbox: it keeps no skills and runs no commands. + ``long_poll_seconds`` bounds how long ``next`` and ``output`` wait for something to report.""" + + def __init__(self, *, shell: bool = True, long_poll_seconds: float = LONG_POLL_SECONDS): + self._shell = shell + self._long_poll_seconds = long_poll_seconds + self._token: str | None = None + self._turns: dict[str, _Turn] = {} + self._version = 0 + self._changed: asyncio.Future[None] | None = None + self._commands: dict[str, asyncio.Task[str]] = {} + + async def run(self, request: TaskRequest[StandInConfig]) -> TaskResult: + turn = _Turn( + task_id=request.task_id, + context_id=request.context_id, + text="\n".join(part.text for part in request.parts if isinstance(part, TextPart)), + system_prompt=request.config.system_prompt, + skills=[DriverSkill(name=skill["name"], description=skill.get("description") or "", + path=str(SKILLS_DIR / skill["name"]) if self._shell else None) + for skill in request.skills], + ) + self._turns[turn.task_id] = turn + self._bump() + try: + reply = await turn.reply + finally: + del self._turns[turn.task_id] + self._bump() + request.trajectory.set_format(TRAJECTORY_FORMAT) + for span in reply.trajectory: + request.trajectory.append(span) + if reply.error is not None: + return TaskResult.failure("stand_in.driver_error", reply.error) + return TaskResult.text(reply.text, usage=Usage(tool_call_count=reply.tool_call_count)) + + @extension(DRIVER_V1.claim) + async def claim(self, request: DriverClaimRequest) -> None: + if self._token is None: + self._token = request.token + elif not hmac.compare_digest(self._token, request.token): + raise HTTPException(status_code=409, detail="this stand-in is already claimed") + + @extension(DRIVER_V1.next) + async def next_turns(self, request: DriverNextRequest) -> DriverNextResponse: + self._check(request.token) + if self._version <= request.after: + with contextlib.suppress(asyncio.TimeoutError): + await asyncio.wait_for(asyncio.shield(self._next_change()), self._long_poll_seconds) + return DriverNextResponse(version=self._version, turns=[ + DriverTurn(task_id=turn.task_id, context_id=turn.context_id, text=turn.text, + system_prompt=turn.system_prompt, skills=turn.skills) + for turn in self._turns.values() if not turn.reply.done() + ]) + + @extension(DRIVER_V1.reply) + async def reply(self, request: DriverReplyRequest) -> None: + self._check(request.token) + turn = self._turns.get(request.task_id) + if turn is None or turn.reply.done(): + raise HTTPException(status_code=404, detail=f"no open turn {request.task_id!r}") + turn.reply.set_result(request) + + @extension(DRIVER_V1.exec) + async def exec_command(self, request: DriverExecRequest) -> DriverExecResponse: + self._check(request.token) + if not self._shell: + raise HTTPException(status_code=409, detail="this stand-in works in no sandbox, so it runs no commands") + command_id = uuid.uuid4().hex + self._commands[command_id] = asyncio.create_task(_run(request.command)) + return DriverExecResponse(exec_id=command_id) + + @extension(DRIVER_V1.output) + async def output(self, request: DriverOutputRequest) -> DriverOutputResponse: + self._check(request.token) + if (command := self._commands.get(request.exec_id)) is None: + raise HTTPException(status_code=404, detail=f"no command {request.exec_id!r}") + await asyncio.wait((command,), timeout=self._long_poll_seconds) + if not command.done(): + return DriverOutputResponse(done=False) + self._commands.pop(request.exec_id, None) + return DriverOutputResponse(done=True, output=command.result()) + + @extension(SKILL_CONFIG_V1.add.inline) + async def add_inline_skill(self, request: InlineSkillRequest) -> dict[str, str]: + folder = self._skill_folder(request.name) + folder.mkdir(parents=True) + (folder / "SKILL.md").write_text(request.skill_md) + return {"name": request.name} + + @extension(SKILL_CONFIG_V1.add.bundle) + async def add_bundle_skill(self, request: BundleSkillRequest) -> dict[str, str]: + folder = self._skill_folder(request.name) + for item in request.skill_bundle.files: + await download(item.object, folder / item.path) + return {"name": request.name} + + def _check(self, token: str) -> None: + if self._token is None or not hmac.compare_digest(self._token, token): + raise HTTPException(status_code=401, detail="the driver's token is missing or wrong") + + def _skill_folder(self, name: str) -> Path: + if not self._shell: + raise HTTPException(status_code=409, detail="this stand-in works in no sandbox, so it keeps no skills") + return SKILLS_DIR / name + + def _next_change(self) -> asyncio.Future[None]: + if self._changed is None: + self._changed = asyncio.get_running_loop().create_future() + return self._changed + + def _bump(self) -> None: + self._version += 1 + if self._changed is not None: + self._changed.set_result(None) + self._changed = None + + +async def _run(command: str) -> str: + """``command``'s stdout then its stderr, as verifiers' bash tool returns them. It runs in a process group of its + own, stopped with whatever it started if it outlives ``COMMAND_TIMEOUT_SECONDS``.""" + process = await asyncio.create_subprocess_exec( + "bash", "-c", command, stdin=asyncio.subprocess.DEVNULL, stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, start_new_session=True, + ) + try: + stdout, stderr = await asyncio.wait_for(process.communicate(), COMMAND_TIMEOUT_SECONDS) + except asyncio.TimeoutError: + return f"error: the command timed out after {COMMAND_TIMEOUT_SECONDS}s" + finally: + if process.returncode is None: + with contextlib.suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGKILL) + await process.wait() + return (stdout + stderr).decode(errors="replace") + + +if __name__ == "__main__": + StandInAgent().serve() diff --git a/packages/agentenv-protocol/tests/test_a2a_stand_in.py b/packages/agentenv-protocol/tests/test_a2a_stand_in.py new file mode 100644 index 00000000..1908c685 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_a2a_stand_in.py @@ -0,0 +1,185 @@ +from __future__ import annotations + +import os +import time +from typing import Any + +import pytest +from starlette.testclient import TestClient + +from agentenv_protocol.a2a_agent import stand_in +from agentenv_protocol.a2a_agent.stand_in import StandInAgent + +TOKEN = "t" * 32 + + +def _send(client: TestClient, text: str, *, context_id: str = "context-1") -> str: + response = client.post("/a2a", json={ + "jsonrpc": "2.0", "id": "1", "method": "message/send", + "params": { + "message": {"kind": "message", "messageId": f"message-{text}", "role": "user", "contextId": context_id, + "parts": [{"kind": "text", "text": text}]}, + "configuration": {"blocking": False}, + }, + }) + return response.json()["result"]["id"] + + +def _task(client: TestClient, task_id: str) -> dict[str, Any]: + return client.post("/a2a", json={"jsonrpc": "2.0", "id": "1", "method": "tasks/get", + "params": {"id": task_id}}).json()["result"] + + +def _settled(client: TestClient, task_id: str) -> dict[str, Any]: + deadline = time.monotonic() + 10 + while (task := _task(client, task_id))["status"]["state"] not in ("completed", "failed", "canceled"): + assert time.monotonic() < deadline, task + time.sleep(0.02) + return task + + +def _driver(client: TestClient, operation: str, **body: Any): + return client.post(f"/ext/driver/{operation}", json={"token": TOKEN, **body}) + + +def _turns(client: TestClient, after: int = 0) -> dict[str, Any]: + response = _driver(client, "next", after=after) + assert response.status_code == 200, response.text + return response.json() + + +def _ran(client: TestClient, command: str) -> str: + exec_id = _driver(client, "exec", command=command).json()["exec_id"] + while not (result := _driver(client, "output", exec_id=exec_id).json())["done"]: + pass + return result["output"] + + +@pytest.fixture +def client(monkeypatch, tmp_path): + monkeypatch.setattr(stand_in, "SKILLS_DIR", tmp_path / "skills") + monkeypatch.chdir(tmp_path) + with TestClient(StandInAgent(long_poll_seconds=0.2).create_app()) as client: + assert client.post("/ext/driver/claim", json={"token": TOKEN}).status_code == 200 + yield client + + +def test_the_card_advertises_the_driver_and_takes_a_system_prompt(): + card = StandInAgent().create_app().state.agentenv_a2a.card.model_dump(mode="json") + extensions = {extension["uri"]: extension for extension in card["capabilities"]["extensions"]} + + assert sorted(extensions["urn:agentenv:driver/v1"]["params"]["methods"]) == [ + "claim", "exec", "next", "output", "reply"] + assert "system_prompt" in extensions["urn:agentenv:agent-config/v1"]["params"]["methods"]["set"]["request"][ + "supported"] + + +def test_the_first_claim_sets_the_token_that_every_driver_call_needs(): + with TestClient(StandInAgent(long_poll_seconds=0.2).create_app()) as client: + assert _driver(client, "next").status_code == 401 + assert client.post("/ext/driver/claim", json={"token": "short"}).status_code == 400 + assert client.post("/ext/driver/claim", json={"token": TOKEN}).status_code == 200 + assert client.post("/ext/driver/claim", json={"token": TOKEN}).status_code == 200 + assert client.post("/ext/driver/claim", json={"token": "u" * 32}).status_code == 409 + assert client.post("/ext/driver/next", json={"token": "u" * 32}).status_code == 401 + assert _driver(client, "next").status_code == 200 + + +def test_a_message_is_an_open_turn_until_the_driver_replies(client): + client.post("/ext/agent-config", json={"system_prompt": "be brief"}) + task_id = _send(client, "fix the build") + + seen = _turns(client) + assert seen["turns"] == [{"task_id": task_id, "context_id": "context-1", "text": "fix the build", + "system_prompt": "be brief", "skills": []}] + span = {"name": "bash", "attributes": {"gen_ai.operation.name": "execute_tool"}} + assert _driver(client, "reply", task_id=task_id, text="fixed", trajectory=[span], tool_call_count=1).status_code \ + == 200 + + task = _settled(client, task_id) + assert task["status"]["state"] == "completed" + parts = task["status"]["message"]["parts"] + assert parts[0] == {"kind": "text", "text": "fixed"} + assert parts[1]["data"]["usage"]["tool_call_count"] == 1 + assert client.post("/ext/trajectory", json={"task_id": task_id}).json()["trajectory"] == [span] + assert _turns(client, after=seen["version"])["turns"] == [] + + +def test_an_error_reply_fails_the_turn(client): + task_id = _send(client, "do it") + + _driver(client, "reply", task_id=task_id, error="the driver gave up") + + task = _settled(client, task_id) + assert task["status"]["state"] == "failed" + assert "the driver gave up" in task["status"]["message"]["parts"][0]["text"] + + +def test_a_reply_names_exactly_one_open_turn(client): + task_id = _send(client, "do it") + + assert _driver(client, "reply", task_id=task_id).status_code == 400 + assert _driver(client, "reply", task_id="nope", text="done").status_code == 404 + _driver(client, "reply", task_id=task_id, text="done") + _settled(client, task_id) + assert _driver(client, "reply", task_id=task_id, text="again").status_code == 404 + + +def test_a_turn_its_sender_cancels_is_withdrawn(client): + task_id = _send(client, "do it") + assert [turn["task_id"] for turn in _turns(client)["turns"]] == [task_id] + + client.post("/a2a", json={"jsonrpc": "2.0", "id": "1", "method": "tasks/cancel", "params": {"id": task_id}}) + + assert _settled(client, task_id)["status"]["state"] == "canceled" + assert _turns(client)["turns"] == [] + + +def test_next_waits_for_a_change_after_the_version_it_was_given(client): + version = _turns(client)["version"] + + started = time.monotonic() + unchanged = _turns(client, after=version) + + assert unchanged["version"] == version + assert time.monotonic() - started >= 0.2 + + +def test_exec_returns_stdout_then_stderr_from_the_folder_it_works_in(client, tmp_path): + (tmp_path / "inputs").mkdir() + + assert _ran(client, "ls; echo out; echo err >&2; exit 3") == "inputs\nout\nerr\n" + + +def test_a_command_that_outlives_its_timeout_is_stopped_with_what_it_started(client, monkeypatch, tmp_path): + monkeypatch.setattr(stand_in, "COMMAND_TIMEOUT_SECONDS", 0.5) + + output = _ran(client, "sleep 60 & echo $! > child; wait") + + assert output == "error: the command timed out after 0.5s" + with pytest.raises(ProcessLookupError): + os.kill(int((tmp_path / "child").read_text()), 0) + + +def test_output_names_a_command_it_started(client): + assert _driver(client, "output", exec_id="nope").status_code == 404 + + +def test_a_skill_is_written_where_its_turns_say_it_is(client, tmp_path): + response = client.post("/ext/skill-config", json={"name": "deploy", "description": "How to deploy.", + "skill_md": "# Deploy\n"}) + assert response.status_code == 200, response.text + _send(client, "go") + + skill = _turns(client)["turns"][0]["skills"] + assert skill == [{"name": "deploy", "description": "How to deploy.", "path": str(tmp_path / "skills" / "deploy")}] + assert (tmp_path / "skills" / "deploy" / "SKILL.md").read_text() == "# Deploy\n" + + +def test_a_stand_in_without_a_shell_keeps_no_skills_and_runs_no_commands(): + with TestClient(StandInAgent(shell=False, long_poll_seconds=0.2).create_app()) as client: + client.post("/ext/driver/claim", json={"token": TOKEN}) + + assert _driver(client, "exec", command="true").status_code == 409 + assert client.post("/ext/skill-config", json={"name": "x", "description": "d", "skill_md": "#"}).status_code \ + == 409 diff --git a/src/agent_env/a2a_agent/a2a_agent.py b/src/agent_env/a2a_agent/a2a_agent.py index 4642fd3a..d54a5794 100644 --- a/src/agent_env/a2a_agent/a2a_agent.py +++ b/src/agent_env/a2a_agent/a2a_agent.py @@ -63,6 +63,7 @@ _DIND_CONTAINER = "agent-dind" _DIND_NETWORK = "agent-docker-net" _DIND_PORT = 2375 +SHARE_STORE_CREDENTIALS = "share_store_credentials" def unwritten_local_agent(agent_id: str, field: str) -> str: @@ -549,7 +550,7 @@ def _build_merged_env(self, resolved_env: dict[str, str], a2a_port: int) -> dict merged_env.update(resolved_env) merged_env["A2A_PORT"] = str(a2a_port) shared = get_config().get_object_store().shared_credentials_env() - if shared.keys().isdisjoint(merged_env): + if self.metadata.get(SHARE_STORE_CREDENTIALS, True) and shared.keys().isdisjoint(merged_env): merged_env.update(shared) merged_env.update(all_sandbox_container_env()) diff --git a/src/agent_env/a2a_agent/stand_in/Dockerfile b/src/agent_env/a2a_agent/stand_in/Dockerfile new file mode 100644 index 00000000..67edcb2a --- /dev/null +++ b/src/agent_env/a2a_agent/stand_in/Dockerfile @@ -0,0 +1,13 @@ +FROM mirror.gcr.io/library/python:3.12-slim@sha256:423ed6ab25b1921a477529254bfeeabf5855151dc2c3141699a1bfc852199fbf +RUN apt-get update \ + && apt-get install -y --no-install-recommends bash ca-certificates coreutils curl findutils git procps \ + && rm -rf /var/lib/apt/lists/* +COPY requirements.txt /opt/agentenv-stand-in/ +RUN pip install --no-cache-dir -r /opt/agentenv-stand-in/requirements.txt && useradd --create-home agent +COPY agentenv_protocol/ /opt/agentenv-stand-in/agentenv_protocol/ +ENV PYTHONPATH=/opt/agentenv-stand-in +WORKDIR /app +COPY a2a_server.py . +RUN chown agent:agent /app +USER agent +CMD ["python", "/app/a2a_server.py"] diff --git a/src/agent_env/a2a_agent/stand_in/__init__.py b/src/agent_env/a2a_agent/stand_in/__init__.py new file mode 100644 index 00000000..cbf38df1 --- /dev/null +++ b/src/agent_env/a2a_agent/stand_in/__init__.py @@ -0,0 +1,67 @@ +"""The stand-in agent (``agentenv_protocol.a2a_agent.stand_in``) as the Episode API deploys it, in the place of an agent +a loop plays: ``@local/agentenv-stand-in``, an image registered by its build context alone and built where each deploy +runs. Its context is this folder, the installed protocol package, and that package's requirements for serving an +agent, pinned to the versions installed here.""" + +from __future__ import annotations + +import shutil +import tempfile +import threading +from importlib.metadata import requires, version +from pathlib import Path + +import agentenv_protocol +from packaging.requirements import Requirement + +from agent_env.a2a_agent.a2a_agent import SHARE_STORE_CREDENTIALS, A2AAgent +from agent_env.artifact.artifacts.docker_image import DockerImageArtifact +from agent_env.store.base import NotFoundError + +STAND_IN_AGENT_ID = "@local/agentenv-stand-in" +_IMAGE_ID = f"{STAND_IN_AGENT_ID}/image" +_PROTOCOL_DISTRIBUTION = "agentenv-framework-protocol" +_HERE = Path(__file__).parent +_writing = threading.Lock() + + +def stand_in_agent() -> A2AAgent: + """The stand-in, written to the store first when it isn't there or its build context has changed.""" + with _writing, tempfile.TemporaryDirectory() as staged: + context = _stage(Path(staged)) + dockerfile = str(context / "Dockerfile") + digest, _ = DockerImageArtifact.context_identity(_IMAGE_ID, str(context), dockerfile, platform=None) + try: + agent = A2AAgent.get(STAND_IN_AGENT_ID) + if agent.docker_image_artifact.source_digest == digest: + return agent + except NotFoundError: + pass + image = DockerImageArtifact.put_context( + _IMAGE_ID, description="The stand-in agent: a driver outside its sandbox answers its turns.", + context_path=str(context), dockerfile_path=dockerfile, platform=None, + ) + return A2AAgent.put( + id=STAND_IN_AGENT_ID, docker_image_artifact=image, + default_env_vars={"LITELLM_API_KEY": "unused", "LITELLM_BASE_URL": "http://unused.invalid"}, + metadata={SHARE_STORE_CREDENTIALS: False}, + ) + + +def _stage(context: Path) -> Path: + for name in ("Dockerfile", "a2a_server.py"): + shutil.copy2(_HERE / name, context / name) + shutil.copytree(Path(agentenv_protocol.__file__).parent, context / "agentenv_protocol", + ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) + (context / "requirements.txt").write_text("".join(f"{line}\n" for line in _requirements())) + return context + + +def _requirements() -> list[str]: + pinned = [] + for spec in requires(_PROTOCOL_DISTRIBUTION) or []: + requirement = Requirement(spec) + if requirement.marker is None or requirement.marker.evaluate({"extra": "agent"}): + extras = f"[{','.join(sorted(requirement.extras))}]" if requirement.extras else "" + pinned.append(f"{requirement.name}{extras}=={version(requirement.name)}") + return sorted(pinned) diff --git a/src/agent_env/a2a_agent/stand_in/a2a_server.py b/src/agent_env/a2a_agent/stand_in/a2a_server.py new file mode 100644 index 00000000..dadb0b38 --- /dev/null +++ b/src/agent_env/a2a_agent/stand_in/a2a_server.py @@ -0,0 +1,37 @@ +"""The stand-in agent as AgentEnv deploys it: served from ``/app`` in its image, or installed on a VM host as +``install_agent`` installs it, in the place of the agent it replaces (its ``{workdir}``, as its ``{user}``). +``AGENT_CARD`` is what ``install_agent`` reads; importing this module serves nothing. ``install_agent`` formats each +command with ``str.format``, so a literal brace is doubled.""" + +from agentenv_protocol.a2a_agent import INSTALL_V1, a2a_agent, enable +from agentenv_protocol.a2a_agent.stand_in import EXTENSIONS, IDENTITY, StandInAgent, StandInConfig + +HOME = "/opt/agentenv-stand-in-{agent_name}" +INSTALL_COMMANDS_HOST = [ + "bash -c 'set -e; python3 -c \"import ensurepip, venv\" 2>/dev/null || {{ export DEBIAN_FRONTEND=noninteractive; " + "apt-get update -qq; apt-get install -y -qq --no-install-recommends python3 python3-venv; }}'", + f"rm -rf {HOME} && mkdir -p {HOME} && tar -xzf {{agent_ctx_tar}} -C {HOME}", + f"python3 -m venv {HOME}/venv && {HOME}/venv/bin/pip install -q -r {HOME}/requirements.txt", + "id -u {user} >/dev/null 2>&1 || useradd -m {user}; mkdir -p {workdir} && chown -R {user}: {workdir}", + f"nohup runuser -u {{user}} -- env -C {{workdir}} A2A_PORT={{a2a_port}} PYTHONPATH={HOME} {HOME}/venv/bin/python " + f"{HOME}/a2a_server.py > /tmp/agentenv-stand-in-{{agent_name}}.log 2>&1 < /dev/null &", +] + + +@a2a_agent( + identity=IDENTITY, + config=StandInConfig, + extensions=( + *EXTENSIONS, + enable(INSTALL_V1, install_commands_host=INSTALL_COMMANDS_HOST, + required_params_host=["agent_ctx_tar", "a2a_port", "agent_name", "workdir", "user"], a2a_port=8000), + ), +) +class InstalledStandIn(StandInAgent): + pass + + +AGENT_CARD = InstalledStandIn().create_app().state.agentenv_a2a.card + +if __name__ == "__main__": + InstalledStandIn().serve() diff --git a/src/agent_env/task/episode/rules.py b/src/agent_env/task/episode/rules.py index 8d766b42..176ce491 100644 --- a/src/agent_env/task/episode/rules.py +++ b/src/agent_env/task/episode/rules.py @@ -1,8 +1,9 @@ """Which steps a task may have when some of its agents are played from outside AgentEnv, and why the others are refused. -Every registered step type is either checked here (``CHECKS``) or listed as untouched by an outside agent -(``UNAFFECTED``). A test holds the registry to the two, so a new step type is classified before an episode runs it. -Plugin step types aren't inspected. +An outside agent is played through a stand-in agent deployed in its place (``agentenv_protocol.a2a_agent.stand_in``), +so steps that act on the agent, or in its sandbox, act on the stand-in. Every registered step type is either checked +here (``CHECKS``) or listed as untouched by an outside agent (``UNAFFECTED``). A test holds the registry to the two, so +a new step type is classified before an episode runs it. Plugin step types aren't inspected. """ from __future__ import annotations @@ -15,6 +16,7 @@ from agent_env.task_step.task_step import TaskStep from agent_env.task_step.task_steps.collect_artifacts import CollectArtifactsTaskStep from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep + from agent_env.task_step.task_steps.install_agent import InstallAgentTaskStep from agent_env.task_step.task_steps.peer_agents import PeerAgentsTaskStep from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep from agent_env.task_step.task_steps.register_env_triggers import RegisterEnvTriggersStep @@ -33,10 +35,10 @@ def problems(steps: list[TaskStep], outside: list[str]) -> list[str]: """Why each outside agent, and each step, can't run with those agents played from outside; empty when it can.""" found = [] for name in outside: - deploys = [step for step in steps if step.type == "deploy_agent" and step.agent_name == name] + deploys = [step for step in steps if step.type in DEPLOYS and step.agent_name == name] if len(deploys) != 1: - found.append(f"agent {name!r} is played from outside, so it needs exactly one deploy_agent step, " - f"not {len(deploys)}") + found.append(f"agent {name!r} is played from outside, so it needs exactly one deploy_agent or " + f"install_agent step, not {len(deploys)}") scope = Outside(frozenset(outside), { step.id: step.prompt_id for step in steps if step.type == "prompt_agent" and step.agent_name in outside }) @@ -48,6 +50,17 @@ def problems(steps: list[TaskStep], outside: list[str]) -> list[str]: return found +def works_in_sandbox(steps: list[TaskStep], agent: str) -> bool: + """Whether ``steps`` use ``agent``'s sandbox, so its stand-in is deployed there rather than served in this + process.""" + return any(step.agent_name == agent and ( + step.type in ("install_agent", "load_artifact", "run_code", "add_skills") + or step.type == "deploy_agent" and step.skills + or step.type == "verify_sandbox" and not step.sandbox_name + or step.type == "collect_artifacts" and not (step.env_id or step.container_name or step.sandbox_name) + ) for step in steps if getattr(step, "agent_name", None)) + + def _prompt_agent(step: PromptAgentTaskStep, outside: Outside) -> list[str]: if step.agent_name in outside.agents: return [ @@ -69,7 +82,6 @@ def _deploy_agent(step: DeployAgentTaskStep, outside: Outside) -> list[str]: return [] return [ f"an outside agent can't have {what}" for what, present in ( - ("skills, instructions its player wouldn't get", step.skills), ("agent_snapshot_files_artifact_id", step.agent_snapshot_files_artifact_id), ("agent_changelog_object_url", step.agent_changelog_object_url), ("enable_agent_changelog", step.enable_agent_changelog), @@ -77,6 +89,13 @@ def _deploy_agent(step: DeployAgentTaskStep, outside: Outside) -> list[str]: ] +def _install_agent(step: InstallAgentTaskStep, outside: Outside) -> list[str]: + if step.agent_name not in outside.agents or step.container_name is None: + return [] + return [f"an outside agent's stand-in replaces an agent installed on the VM host, not in container " + f"{step.container_name!r}"] + + def _deploys_an_agent(step: TaskStep, outside: Outside) -> list[str]: if step.agent_name not in outside.agents: return [] @@ -94,12 +113,9 @@ def _validates_an_agent(step: TaskStep, outside: Outside) -> list[str]: def _collect_artifacts(step: CollectArtifactsTaskStep, outside: Outside) -> list[str]: - why = [] - if step.agent_name in outside.agents and not (step.env_id or step.container_name or step.sandbox_name): - why.append(f"collects from {step.agent_name!r}'s sandbox, and it is played from outside") if step.manifest_step_id in outside.prompts: - why.append(f"its manifest is outside prompt {step.manifest_step_id!r}") - return why + return [f"its manifest is outside prompt {step.manifest_step_id!r}"] + return [] def _register_env_triggers(step: RegisterEnvTriggersStep, outside: Outside) -> list[str]: @@ -116,25 +132,18 @@ def _peer_agents(step: PeerAgentsTaskStep, outside: Outside) -> list[str]: def _rubrics_verifier(step: RubricsVerifierTaskStep, outside: Outside) -> list[str]: - why = [] - if step.use_trajectory and step.prompt_id in outside.prompts.values(): - why.append("grades an outside prompt's trajectory, which an outside player doesn't hand back") if step.use_agent_judge and step.agent_name in outside.agents: - why.append(f"judges in {step.agent_name!r}'s sandbox, and it is played from outside") - return why + return [f"judges in {step.agent_name!r}'s sandbox, and it is played from outside"] + return [] CHECKS: dict[str, Callable[..., list[str]]] = { "prompt_agent": _prompt_agent, "deploy_agent": _deploy_agent, - "install_agent": _deploys_an_agent, + "install_agent": _install_agent, "deploy_human_agent": _deploys_an_agent, - "add_skills": _needs_the_agent, "register_agent_triggers": _needs_the_agent, "snapshot_agent_state": _needs_the_agent, - "load_artifact": _needs_the_agent, - "verify_sandbox": _needs_the_agent, - "run_code": _needs_the_agent, "collect_artifacts": _collect_artifacts, "register_env_triggers": _register_env_triggers, "peer_agents": _peer_agents, @@ -157,4 +166,8 @@ def _rubrics_verifier(step: RubricsVerifierTaskStep, outside: Outside) -> list[s "agent_prompt_response_verifier", "verify_mcp_env_assessment", # It skips an agent that isn't deployed. "teardown_sandboxes", + # They act on an outside agent's stand-in, in its sandbox, as on the agent. + "add_skills", "load_artifact", "run_code", "verify_sandbox", }) + +DEPLOYS = ("deploy_agent", "install_agent") diff --git a/src/agent_env/task/episode/run.py b/src/agent_env/task/episode/run.py index 8ee4ddfd..86abb281 100644 --- a/src/agent_env/task/episode/run.py +++ b/src/agent_env/task/episode/run.py @@ -1,37 +1,59 @@ from __future__ import annotations import asyncio +import json import logging from collections import deque -from typing import TYPE_CHECKING, Any +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any +import httpx from agentenv_protocol import EnvironmentTool, ToolResult, ToolSession from agent_env.providers.sandbox_providers.sandbox_provider import sandbox_request_headers_for_url +from agent_env.store.routing import namespace_routing +from agent_env.task.episode.stand_in import InProcessStandIn, StandInClient from agent_env.task.store import record_task_cancelled from agent_env.task.task import Task, recorded_scores, substitute_seed_in_steps from agent_env.task.teardown import teardown_run -from agent_env.task_step.context import TaskStepContext - -if TYPE_CHECKING: - from agent_env.task.episode.steps import _EpisodeTurnStep +from agent_env.task_step.context import DeployedAgent, TaskStepContext logger = logging.getLogger(__name__) _live: dict[str, Episode] = {} +_DRAIN_RETRY_SECONDS = 1.0 +_CARD_TIMEOUT_SECONDS = 10.0 + +BASH = EnvironmentTool( + name="bash", + description="Run a bash command where the agent works, as its user. Returns its stdout, then its stderr.", + inputSchema={"type": "object", "properties": {"command": {"type": "string", "description": "The command to run."}}, + "required": ["command"]}, +) class EpisodeError(RuntimeError): """A task that can't run as an episode, or an episode asked for what it can't give.""" +@dataclass(frozen=True) +class Skill: + """A skill the task gave the agent: its instructions are ``path``'s ``SKILL.md``, in the agent's sandbox.""" + + name: str + description: str + path: str | None + + class Turn: - """An outside agent's turn. Its model reads ``system_prompt`` and ``prompt``, acts through ``tools()`` (or an MCP - client of its own on ``mcp_urls``, sending ``headers`` and ``role``), and hands its reply back with ``end()``.""" + """An outside agent's turn. Its model reads ``system_prompt``, ``prompt`` and ``skills``, acts through ``tools()`` + (or an MCP client of its own on ``mcp_urls``, sending ``headers`` and ``role``), and hands its reply back with + ``end()``.""" def __init__( self, *, agent: str, step_id: str, prompt: str, system_prompt: str | None, role: str | None, - mcp_urls: dict[str, str], + mcp_urls: dict[str, str], skills: list[Skill], shell: bool, stand_in: StandInClient, task_id: str, ): self.agent = agent self.step_id = step_id @@ -40,43 +62,50 @@ def __init__( self.role = role self.mcp_urls = mcp_urls self.headers = {env_id: sandbox_request_headers_for_url(url) for env_id, url in mcp_urls.items()} - self._reply: asyncio.Future[str] = asyncio.get_running_loop().create_future() + self.skills = skills + self._shell = shell + self._stand_in = stand_in + self._task_id = task_id + self._spans: list[dict[str, Any]] = [] + self._started = _now() + self._open = True def tools(self, *, timeout: float = 600.0) -> TurnTools: """A client for the tools of the envs this agent acts in, calling them as its role with the headers each URL - needs; use it with ``async with``. ``timeout`` bounds each call, in seconds.""" + needs, and ``bash`` when it works in a sandbox; use it with ``async with``. ``timeout`` bounds each env tool's + call, in seconds.""" return TurnTools(self, timeout) async def end(self, response: str | None = None) -> None: """Hand the turn back, and the run carries on. No reply records an empty one, as an agent that answers with nothing leaves. Once the turn is over, ended or not, this does nothing.""" + if not self._open: + return + self._open = False + reply = response or "" + spans = [*self._spans, _span("turn", "chain", self._started, completion={"content": [ + {"type": "text", "text": reply}]})] + await self._stand_in.reply(self._task_id, text=reply, trajectory=spans, tool_call_count=len(self._spans)) + + async def _fail(self, error: Exception) -> None: if self._open: - self._reply.set_result(response or "") - - @property - def _open(self) -> bool: - return not self._reply.done() - - async def _replied_within(self, seconds: float) -> str: - return await asyncio.wait_for(self._reply, seconds) - - def _fail(self, error: Exception) -> None: - if self._open: - self._reply.set_exception(error) + self._open = False + await self._stand_in.reply(self._task_id, error=str(error)) class TurnTools: - """The tools of a turn's envs, listed once when it opens: ``list()`` names every one, and ``call()`` routes a - call to the env that offers it.""" + """The tools of a turn, listed once when it opens: its envs', and ``bash`` when its agent works in a sandbox. + ``list()`` names every one, ``call()`` routes a call to what offers it, and each call goes in the turn's + trajectory.""" def __init__(self, turn: Turn, timeout: float): - self._step_id = turn.step_id + self._turn = turn self._sessions = { env_id: ToolSession(mcp_url=url, role=turn.role, headers=turn.headers[env_id], timeout=timeout) for env_id, url in turn.mcp_urls.items() } self._tools: list[EnvironmentTool] = [] - self._offered_by: dict[str, str] = {} + self._offered_by: dict[str, str | None] = {} async def __aenter__(self) -> TurnTools: try: @@ -87,6 +116,12 @@ async def __aenter__(self) -> TurnTools: raise EpisodeError(f"tool {tool.name!r} is offered by both env {other!r} and env {env_id!r}") self._offered_by[tool.name] = env_id self._tools.append(tool) + if self._turn._shell: + if (env_id := self._offered_by.get(BASH.name)) is not None: + raise EpisodeError(f"tool {BASH.name!r} is offered by env {env_id!r}, and the agent's sandbox " + "offers its own") + self._offered_by[BASH.name] = None + self._tools.append(BASH) except BaseException: await self._close() raise @@ -101,8 +136,21 @@ def list(self) -> list[EnvironmentTool]: async def call(self, name: str, arguments: dict[str, Any] | None = None) -> ToolResult: if name not in self._offered_by: raise EpisodeError( - f"no env of turn {self._step_id!r} offers tool {name!r}; they offer {sorted(self._offered_by)}") - return await self._sessions[self._offered_by[name]].call_tool(name, arguments) + f"no env of turn {self._turn.step_id!r} offers tool {name!r}; they offer {sorted(self._offered_by)}") + started = _now() + env_id = self._offered_by[name] + if env_id is None: + result = await self._bash(arguments or {}) + else: + result = await self._sessions[env_id].call_tool(name, arguments) + self._turn._spans.append(_span(name, "execute_tool", started, prompt={"input": arguments or {}}, + completion={"output": result.content})) + return result + + async def _bash(self, arguments: dict[str, Any]) -> ToolResult: + if not isinstance(command := arguments.get("command"), str): + raise EpisodeError(f"tool {BASH.name!r} takes a command string, not {command!r}") + return ToolResult(content=[{"type": "text", "text": await self._turn._stand_in.run(command)}]) async def _close(self) -> None: await asyncio.gather(*(session.close() for session in self._sessions.values())) @@ -119,12 +167,14 @@ def __init__(self, id: str, task: Task, context: TaskStepContext): self._task = task self._context = context self._run: asyncio.Task[None] - self._bindings: dict[str, tuple[str | None, dict[str, str]]] = {} self._waiting: deque[Turn] = deque() self._turns: set[Turn] = set() self._changed: asyncio.Future[None] = asyncio.get_running_loop().create_future() self._tearing_down = False self._abandoned: str | None = None + self._stand_ins: list[StandInClient] = [] + self._drains: list[asyncio.Task[None]] = [] + self._in_process: list[InProcessStandIn] = [] async def next_turn(self) -> Turn | None: """The next outside turn the run reaches, or None once the run has finished. Raises the run's error if it @@ -151,8 +201,7 @@ async def result(self) -> dict[str, float]: ) await self._wait(changed) self._raise_unless_completed() - seeded = substitute_seed_in_steps(self._task.steps, self._context.metadata.get("seed") or {}) - return recorded_scores(seeded, self._context) + return recorded_scores(self._seeded_steps(), self._context) async def abandon(self, reason: str) -> None: """Stop the run and wait for its teardown; it is recorded cancelled with ``reason``. A run a step has @@ -160,8 +209,8 @@ async def abandon(self, reason: str) -> None: Once the run has ended, or while it tears down, this only waits.""" if not (self._run.done() or self._tearing_down or self._abandoned is not None): if any(failure.get("is_fatal") for failure in self._context.metadata.get("failed_steps") or []): - for turn in self._turns: - turn._fail(EpisodeError(f"episode abandoned: {reason}")) + for turn in list(self._turns): + await turn._fail(EpisodeError(f"episode abandoned: {reason}")) else: self._abandoned = reason self._run.cancel(msg=reason) @@ -181,19 +230,24 @@ async def _launch(self) -> None: raise async def _drive(self, running: asyncio.Event) -> None: + """Run the task with namespace routing on, as ``agent-env run`` does, so the stand-in, an ``@local`` agent, + resolves.""" running.set() - try: - await self._task.run(context=self._context, instance_id=self.id) - finally: - self._tearing_down = True + with namespace_routing(): try: - report = await teardown_run(self._context) - if report.still_up: - logger.warning("episode %s: %s may still be up", self.id, [s.sandbox_id for s in report.still_up]) - if self._abandoned is not None: - await asyncio.to_thread(record_task_cancelled, self.id, self._abandoned, Task._utc_now_str()) + await self._task.run(context=self._context, instance_id=self.id) finally: - del _live[self.id] + self._tearing_down = True + try: + await self._release_stand_ins() + report = await teardown_run(self._context) + if report.still_up: + logger.warning("episode %s: %s may still be up", self.id, + [s.sandbox_id for s in report.still_up]) + if self._abandoned is not None: + await asyncio.to_thread(record_task_cancelled, self.id, self._abandoned, Task._utc_now_str()) + finally: + del _live[self.id] async def _wait(self, changed: asyncio.Future[None]) -> None: """Until the run ends or what was ``changed`` when the caller last looked changes again.""" @@ -205,29 +259,93 @@ def _raise_unless_completed(self) -> None: if (error := self._run.exception()) is not None: raise error - def _bind(self, agent: str, role: str | None, mcp_urls: dict[str, str]) -> None: - self._bindings[agent] = (role, mcp_urls) - - async def _play(self, step: _EpisodeTurnStep) -> str: - if step.agent_name not in self._bindings: - raise RuntimeError( - f"agent {step.agent_name!r} isn't deployed yet; turn {step.id!r} must depend on its deploy_agent step" - ) - role, mcp_urls = self._bindings[step.agent_name] - turn = Turn(agent=step.agent_name, step_id=step.id, prompt=step.prompt, system_prompt=step.system_prompt, - role=role, mcp_urls=dict(mcp_urls)) - self._waiting.append(turn) - self._turns.add(turn) - self._changed.set_result(None) - self._changed = asyncio.get_running_loop().create_future() - try: - return await turn._replied_within(step.timeout_seconds) - except TimeoutError: - raise TimeoutError( - f"turn {step.id!r} of agent {step.agent_name!r} wasn't ended within {step.timeout_seconds}s" - ) from None - finally: - self._turns.discard(turn) + async def _serve_in_process(self, agent: str, role: str | None, mcp_urls: dict[str, str]) -> DeployedAgent: + """A stand-in served in this process for ``agent``, which works in no sandbox, played as the others are.""" + server = InProcessStandIn() + self._in_process.append(server) + url = await server.start() + async with httpx.AsyncClient(timeout=_CARD_TIMEOUT_SECONDS) as client: + response = await client.get(f"{url}/.well-known/agent.json") + response.raise_for_status() + deployed = DeployedAgent(agent_name=agent, api_url=url, a2a_url=url, a2a_card=response.json(), role=role) + await self._attach(agent, deployed, role, mcp_urls, shell=False) + return deployed + + async def _attach( + self, agent: str, deployed: DeployedAgent, role: str | None, mcp_urls: dict[str, str], *, shell: bool, + ) -> None: + """Claim ``agent``'s stand-in and hand out the turns it opens.""" + stand_in = StandInClient(deployed.a2a_url, deployed.a2a_card or {}) + self._stand_ins.append(stand_in) + await stand_in.claim() + self._drains.append(asyncio.create_task(self._drain(agent, stand_in, role, mcp_urls, shell))) + + async def _drain( + self, agent: str, stand_in: StandInClient, role: str | None, mcp_urls: dict[str, str], shell: bool, + ) -> None: + """Hand out each turn the stand-in opens, and close each one it drops, until the run ends.""" + after, open_turns = 0, {} + while True: + try: + answer = await stand_in.next(after) + except httpx.HTTPError as e: + logger.warning("episode %s: reading agent %r's turns failed, retrying: %s", self.id, agent, e) + await asyncio.sleep(_DRAIN_RETRY_SECONDS) + continue + after = answer["version"] + listed = {item["task_id"]: item for item in answer["turns"]} + for task_id in [task_id for task_id in open_turns if task_id not in listed]: + turn = open_turns.pop(task_id) + turn._open = False + self._turns.discard(turn) + for task_id, item in listed.items(): + if task_id not in open_turns: + open_turns[task_id] = turn = Turn( + agent=agent, step_id=self._step_of(agent, item["context_id"]), prompt=item["text"], + system_prompt=item.get("system_prompt"), role=role, mcp_urls=dict(mcp_urls), + skills=[Skill(**skill) for skill in item.get("skills", [])], shell=shell, stand_in=stand_in, + task_id=task_id, + ) + self._waiting.append(turn) + self._turns.add(turn) + self._changed.set_result(None) + self._changed = asyncio.get_running_loop().create_future() + + def _step_of(self, agent: str, context_id: str) -> str: + """The ``prompt_agent`` step whose conversation with ``agent`` a message on ``context_id`` is in.""" + conversations = self._context.metadata.get("a2a_conversations", {}) + for step in self._seeded_steps(): + if step.type == "prompt_agent" and step.agent_name == agent and ( + step.context_id or conversations.get(step.id)) == context_id: + return step.id + return context_id + + def _seeded_steps(self) -> list: + return substitute_seed_in_steps(self._task.steps, self._context.metadata.get("seed") or {}) + + async def _release_stand_ins(self) -> None: + for drain in self._drains: + drain.cancel() + await asyncio.gather(*self._drains, return_exceptions=True) + for turn in self._turns: + turn._open = False + for server in self._in_process: + server.stop() + await asyncio.gather(*(stand_in.close() for stand_in in self._stand_ins), return_exceptions=True) + + +def _span(name: str, operation: str, started: str, *, prompt: Any = None, completion: Any = None) -> dict[str, Any]: + """An OTel GenAI span, the trajectory format rubrics read.""" + attributes = {"gen_ai.operation.name": operation} + if prompt is not None: + attributes["gen_ai.prompt"] = json.dumps(prompt, default=str) + if completion is not None: + attributes["gen_ai.completion"] = json.dumps(completion, default=str) + return {"name": name, "start_time": started, "end_time": _now(), "attributes": attributes} + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat() def _episode_of(context: TaskStepContext) -> Episode: diff --git a/src/agent_env/task/episode/spec.py b/src/agent_env/task/episode/spec.py index 0ddddda0..c8b0c026 100644 --- a/src/agent_env/task/episode/spec.py +++ b/src/agent_env/task/episode/spec.py @@ -1,12 +1,13 @@ from __future__ import annotations +import math import uuid from collections.abc import Mapping, Sequence from typing import Any -from agent_env.task.episode.rules import problems +from agent_env.task.episode.rules import DEPLOYS, problems, works_in_sandbox from agent_env.task.episode.run import Episode, EpisodeError -from agent_env.task.episode.steps import _EpisodeAgentStep, _EpisodeTurnStep +from agent_env.task.episode.steps import _EpisodeAgentStep from agent_env.task.task import Task, substitute_seed_in_steps from agent_env.task_step.context import TaskStepContext from agent_env.task_step.task_step import TaskStep @@ -28,18 +29,15 @@ def from_task( turn_timeout: float | None = None, ) -> EpisodeSpec: """``trainer_agents`` are the agents the loop plays, by default the task's only prompted agent. - ``turn_timeout`` is every turn's deadline in seconds, in place of its prompt's own ``timeout_seconds``, - which budgets an agent's run rather than the loop playing it.""" + ``turn_timeout`` is every turn's deadline in seconds, rounded up to a whole one, in place of its prompt's own + ``timeout_seconds``, which budgets an agent's run rather than the loop playing it.""" seed = dict(seed) if seed else None seeded = substitute_seed_in_steps(task.steps, seed or {}) outside = _trainer_agents(task, seeded, trainer_agents) if found := problems(seeded, outside): raise EpisodeError(f"task {task.id!r} can't run as an episode: " + "; ".join(found)) - deploys = { - as_run.agent_name: step for step, as_run in zip(task.steps, seeded) - if step.type == "deploy_agent" and as_run.agent_name in outside - } - steps = [_stand_in(step, as_run, deploys, turn_timeout) for step, as_run in zip(task.steps, seeded)] + sandboxed = {name: works_in_sandbox(seeded, name) for name in outside} + steps = [_stand_in(step, as_run, sandboxed, turn_timeout) for step, as_run in zip(task.steps, seeded)] return cls(copy=Task(id=task.id, version=task.version, steps=steps), trainer_agents=tuple(outside), seed=seed) async def start(self) -> Episode: @@ -71,21 +69,15 @@ def _trainer_agents(task: Task, steps: list[TaskStep], named: Sequence[str] | No return outside -def _stand_in(step: TaskStep, as_run: TaskStep, deploys: dict[str, TaskStep], turn_timeout: float | None) -> TaskStep: +def _stand_in(step: TaskStep, as_run: TaskStep, sandboxed: dict[str, bool], turn_timeout: float | None) -> TaskStep: """The step the episode's copy of the task runs in place of ``step``, judged by ``as_run``, the step as the - seed makes it; ``deploys`` are the outside agents', by name. A stand-in keeps the step's own fields, which the - run seeds.""" - if as_run.type == "deploy_agent" and as_run.agent_name in deploys: + seed makes it; ``sandboxed`` says, for each outside agent by name, whether its stand-in is deployed in its + sandbox. The copy keeps every step's own fields, which the run seeds.""" + if as_run.type in DEPLOYS and as_run.agent_name in sandboxed: return _EpisodeAgentStep( - id=step.id, version=step.version, depends_on=step.depends_on, fail_task_on_error=step.fail_task_on_error, - agent_name=step.agent_name, env_ids=step.env_ids, env_step_id=step.env_step_id, role=step.role, - ) - if as_run.type == "prompt_agent" and as_run.agent_name in deploys: - timeout = step.timeout_seconds if turn_timeout is None else turn_timeout - return _EpisodeTurnStep( - id=step.id, version=step.version, depends_on=step.depends_on, fail_task_on_error=step.fail_task_on_error, - agent_name=step.agent_name, prompt_id=step.prompt_id, prompt=step.prompt, - system_prompt=step.system_prompt or deploys[as_run.agent_name].system_prompt, - timeout_seconds=PromptAgentTaskStep.DEFAULT_PROMPT_TIMEOUT_SECONDS if timeout is None else timeout, + id=step.id, version=step.version, step=step, sandboxed=sandboxed[as_run.agent_name], + depends_on=step.depends_on, fail_task_on_error=step.fail_task_on_error, ) + if as_run.type == "prompt_agent" and as_run.agent_name in sandboxed and turn_timeout is not None: + return PromptAgentTaskStep.from_dict({**step.to_dict(), "timeout_seconds": math.ceil(turn_timeout)}) return step diff --git a/src/agent_env/task/episode/stand_in.py b/src/agent_env/task/episode/stand_in.py new file mode 100644 index 00000000..dbff4ecb --- /dev/null +++ b/src/agent_env/task/episode/stand_in.py @@ -0,0 +1,89 @@ +"""The episode's end of a stand-in agent: the driver calls of ``urn:agentenv:driver/v1``, and a stand-in served in this +process for an outside agent that works in no sandbox.""" + +from __future__ import annotations + +import asyncio +import secrets +import threading +from typing import Any + +import httpx +import uvicorn +from agentenv_protocol.a2a_agent import DRIVER_V1 +from agentenv_protocol.a2a_agent.stand_in import StandInAgent + +from agent_env.a2a_agent.a2a_agent import A2AAgent + +_CALL_TIMEOUT = httpx.Timeout(60.0, connect=10.0) +_SERVE_TIMEOUT_SECONDS = 30.0 +# Short, so a long-poll in flight when the episode ends finishes within the server's shutdown. +_IN_PROCESS_LONG_POLL_SECONDS = 1.0 + + +class StandInClient: + """One stand-in, claimed with a token of this episode's, at the endpoints its card names.""" + + def __init__(self, a2a_url: str, card: dict[str, Any]): + driver = A2AAgent.find_extension(card, DRIVER_V1.uri) + if driver is None: + raise RuntimeError(f"the agent at {a2a_url} isn't a stand-in: its card has no {DRIVER_V1.uri}") + self._endpoints = {operation: a2a_url + A2AAgent.operation(driver, operation)[1] + for operation in DRIVER_V1.operations} + self._token = secrets.token_urlsafe(24) + self._client = httpx.AsyncClient(timeout=_CALL_TIMEOUT) + + async def claim(self) -> None: + await self._call("claim") + + async def next(self, after: int) -> dict[str, Any]: + return await self._call("next", after=after) + + async def reply(self, task_id: str, **reply: Any) -> bool: + """Whether the reply reached an open turn: a turn its sender stopped waiting for is gone.""" + response = await self._client.post(self._endpoints["reply"], json={"token": self._token, "task_id": task_id, + **reply}) + if response.status_code == 404: + return False + response.raise_for_status() + return True + + async def run(self, command: str) -> str: + exec_id = (await self._call("exec", command=command))["exec_id"] + while not (output := await self._call("output", exec_id=exec_id))["done"]: + pass + return output["output"] + + async def close(self) -> None: + await self._client.aclose() + + async def _call(self, operation: str, **body: Any) -> dict[str, Any]: + response = await self._client.post(self._endpoints[operation], json={"token": self._token, **body}) + response.raise_for_status() + return response.json() + + +class InProcessStandIn: + """A stand-in served on loopback in a thread of its own, with no shell: one for an outside agent that works in no + sandbox, which only this process's steps message.""" + + def __init__(self) -> None: + agent = StandInAgent(shell=False, long_poll_seconds=_IN_PROCESS_LONG_POLL_SECONDS) + self._server = uvicorn.Server(uvicorn.Config(agent.create_app(), host="127.0.0.1", port=0, log_level="warning", + timeout_graceful_shutdown=2 * _IN_PROCESS_LONG_POLL_SECONDS)) + self._thread = threading.Thread(target=self._server.run, name="agentenv-stand-in", daemon=True) + + async def start(self) -> str: + """Serve it, and return its URL once it answers.""" + self._thread.start() + async with asyncio.timeout(_SERVE_TIMEOUT_SECONDS): + while not self._server.started: + if not self._thread.is_alive(): + raise RuntimeError("the in-process stand-in agent stopped as it started") + await asyncio.sleep(0.01) + port = self._server.servers[0].sockets[0].getsockname()[1] + return f"http://127.0.0.1:{port}" + + def stop(self) -> None: + """Have it stop; its thread ends once the requests in flight have.""" + self._server.should_exit = True diff --git a/src/agent_env/task/episode/steps.py b/src/agent_env/task/episode/steps.py index f6f55d33..f6df0f33 100644 --- a/src/agent_env/task/episode/steps.py +++ b/src/agent_env/task/episode/steps.py @@ -1,80 +1,63 @@ from __future__ import annotations +import asyncio from typing import Any, ClassVar, Optional -from agent_env.entity.refs import EntityRef +from agent_env.a2a_agent import protocol +from agent_env.a2a_agent.a2a_agent import A2AAgent +from agent_env.a2a_agent.stand_in import STAND_IN_AGENT_ID, stand_in_agent from agent_env.task.episode.run import _episode_of -from agent_env.task_step.context import PromptResponse, TaskStepContext +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.registry import get_task_step_registry from agent_env.task_step.task_step import TaskStep from agent_env.task_step.task_steps.deploy_agent import bound_deployment, live_mcp_url class _EpisodeAgentStep(TaskStep): - """An outside agent's ``deploy_agent``: deploys nothing, and binds the agent to its envs where it stood.""" + """An outside agent's ``deploy_agent`` or ``install_agent``. In a task that uses the agent's sandbox it runs that + step with the stand-in agent in the agent's place; otherwise it serves a stand-in in this process. Either way the + episode then plays the stand-in, which the agent's ``prompt_agent`` steps message as they would the agent.""" type: ClassVar[str] = "episode_agent" - entity_refs = (EntityRef.env("env_ids[]"),) + entity_refs = () def __init__( - self, id: str, version: Optional[int], *, agent_name: str, env_ids: list[str], env_step_id: str | None, - role: str | None, depends_on: Optional[list] = None, fail_task_on_error: bool = True, + self, id: str, version: Optional[int], *, step: TaskStep, sandboxed: bool, + depends_on: Optional[list] = None, fail_task_on_error: bool = True, ): super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) - self.agent_name = agent_name - self.env_ids = list(env_ids) - self.env_step_id = env_step_id - self.role = role + self.step = step + self.sandboxed = sandboxed def to_dict(self) -> dict[str, Any]: - return {**super().to_dict(), "agent_name": self.agent_name, "env_ids": self.env_ids, - "env_step_id": self.env_step_id, "role": self.role} + return {**super().to_dict(), "step": self.step.to_dict(), "sandboxed": self.sandboxed} @classmethod def from_dict(cls, data: dict[str, Any]) -> _EpisodeAgentStep: - return cls(**cls._base_from_dict(data), agent_name=data["agent_name"], env_ids=data["env_ids"], - env_step_id=data["env_step_id"], role=data["role"]) + step = get_task_step_registry()[data["step"]["type"]].from_dict(data["step"]) + return cls(**cls._base_from_dict(data), step=step, sandboxed=data["sandboxed"]) async def execute(self, context: TaskStepContext) -> TaskStepContext: + episode = _episode_of(context) + agent = self.step.agent_name + role = getattr(self.step, "role", None) mcp_urls = {} - for env_id in self.env_ids: - deployed = bound_deployment(context, env_id, env_step_id=self.env_step_id, step_id=self.id) + for env_id in getattr(self.step, "env_ids", None) or []: + deployed = bound_deployment(context, env_id, env_step_id=self.step.env_step_id, step_id=self.id) mcp_urls[env_id] = deployed.mcp_url if deployed is not None else live_mcp_url(env_id) - _episode_of(context)._bind(self.agent_name, self.role, mcp_urls) - return context - - -class _EpisodeTurnStep(TaskStep): - """An outside agent's ``prompt_agent``: hands the turn to the caller and records the reply it gets back.""" - - type: ClassVar[str] = "episode_turn" - entity_refs = () - - def __init__( - self, id: str, version: Optional[int], *, agent_name: str, prompt_id: str, prompt: str, - system_prompt: str | None, timeout_seconds: float, depends_on: Optional[list] = None, - fail_task_on_error: bool = True, - ): - super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) - self.agent_name = agent_name - self.prompt_id = prompt_id - self.prompt = prompt - self.system_prompt = system_prompt - self.timeout_seconds = timeout_seconds - - def to_dict(self) -> dict[str, Any]: - return {**super().to_dict(), "agent_name": self.agent_name, "prompt_id": self.prompt_id, - "prompt": self.prompt, "system_prompt": self.system_prompt, "timeout_seconds": self.timeout_seconds} - - @classmethod - def from_dict(cls, data: dict[str, Any]) -> _EpisodeTurnStep: - return cls(**cls._base_from_dict(data), agent_name=data["agent_name"], prompt_id=data["prompt_id"], - prompt=data["prompt"], system_prompt=data["system_prompt"], - timeout_seconds=data["timeout_seconds"]) - - async def execute(self, context: TaskStepContext) -> TaskStepContext: - reply = await _episode_of(context)._play(self) - context.prompt_responses.append(PromptResponse( - prompt_id=self.prompt_id, response=reply, prompt_text=self.prompt, agent_name=self.agent_name, - step_id=self.id, - )) + if not self.sandboxed: + deployed = await episode._serve_in_process(agent, role, mcp_urls) + context.deployed_agents.append(deployed) + if system_prompt := getattr(self.step, "system_prompt", None): + config = A2AAgent.negotiate_agent_config(deployed.a2a_card, {"system_prompt": system_prompt}) + await protocol.post_agent_config(deployed.a2a_url + config.endpoint, config.fields) + return context + await asyncio.to_thread(stand_in_agent) + override = {"a2a_agent_id": STAND_IN_AGENT_ID} if self.step.type == "deploy_agent" else { + "stand_in": STAND_IN_AGENT_ID} + step_params = context.metadata.setdefault("user_overrides", {}).setdefault("step_params", {}) + step_params[self.step.id] = {**step_params.get(self.step.id, {}), **override} + context = await self.step.execute(context) + deployed = next(deployed for deployed in context.deployed_agents if deployed.agent_name == agent) + await episode._attach(agent, deployed, role, mcp_urls, shell=True) return context diff --git a/src/agent_env/task_step/task_steps/install_agent.py b/src/agent_env/task_step/task_steps/install_agent.py index 40df1eb2..07d2e554 100644 --- a/src/agent_env/task_step/task_steps/install_agent.py +++ b/src/agent_env/task_step/task_steps/install_agent.py @@ -43,6 +43,8 @@ REQUIRED_PARAMS_KEY = "required_params" REQUIRED_PARAMS_HOST_KEY = "required_params_host" A2A_PORT_KEY = "a2a_port" +WORKDIR_HOST_KEY = "workdir_host" +USER_HOST_KEY = "user_host" def _install_problem(agent: A2AAgent) -> str | None: @@ -66,6 +68,7 @@ def _build_param_resolvers( litellm_api_key_override: Optional[str] = None, sandbox_type: Optional[str] = None, trust_lines: str = "", + place: Optional[dict[str, str]] = None, ) -> dict[str, Callable[[], str]]: """Per-install registry: required_params name -> zero-arg resolver returning the raw value. @@ -86,6 +89,14 @@ def _resolve_workspace_dir() -> str: ) return workspace_dir + def _placed(name: str) -> str: + if place is None: + raise KeyError( + f"agent declares required_param {name!r}, which only an install in another agent's place " + "(the step's `stand_in` param) gives" + ) + return place[name] + def _resolve_container() -> str: if container_name is None: raise KeyError( @@ -102,6 +113,9 @@ def _resolve_container() -> str: "a2a_port": lambda: str(a2a_port), "agent_name": lambda: agent_name, "workspace_dir": _resolve_workspace_dir, + # Where the agent an install replaces works on the VM host, and as which user. + "workdir": lambda: _placed("workdir"), + "user": lambda: _placed("user"), # Config-sourced (from agent-env config) "litellm_api_key": lambda: (config.resolve_reference(litellm_api_key_override) if litellm_api_key_override else config.get_litellm_api_key()), @@ -225,6 +239,12 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: agent = A2AAgent.get(a2a_agent_id, self.a2a_agent_version) if problem := _install_problem(agent): raise RuntimeError(problem) + place = None + if stand_in := self.step_param_overrides(context).get("stand_in"): + place = await self._place_of(sandbox, agent) + agent = A2AAgent.get(stand_in) + if problem := _install_problem(agent): + raise RuntimeError(problem) logger.info(f"Loading agent image {agent.docker_image_artifact.image_name} onto VM (needed to read agent card)...") await sandbox.load_docker_images([agent.docker_image_artifact]) @@ -283,6 +303,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: # A host install on a local sandbox runs on this machine itself, not in a container. sandbox_type=None if host_mode else sandbox.type, trust_lines=trust_lines, + place=place, ) resolved: dict[str, str] = {} for name in required_params: @@ -356,6 +377,30 @@ async def _trust_lines(sandbox, container: Optional[str], work_dir: str) -> str: env = trust_env(directory) return "".join(f"{name}={shlex.quote(value)}\n" for name, value in env.items()) + async def _place_of(self, sandbox, agent: A2AAgent) -> dict[str, str]: + """Where ``agent``, which a stand-in replaces, works on the VM host and as which user: its install/v1 params + ``workdir_host`` and ``user_host``, templates of ``{agent_name}`` as its install commands are.""" + if self.container_name is not None: + raise RuntimeError( + f"install_agent '{self.id}': a stand-in replaces an agent installed on the VM host, not in a container" + ) + if sandbox.ON_THIS_MACHINE: + raise RuntimeError( + f"install_agent '{self.id}': a stand-in isn't installed on a sandbox that runs on this machine, where " + "its player's commands would run here; deploy the sandbox on a remote VM provider" + ) + logger.info(f"Loading agent image {agent.docker_image_artifact.image_name} onto VM (to read where it works)...") + await sandbox.load_docker_images([agent.docker_image_artifact]) + install_ext = await self._extract_install_extension(sandbox, agent.docker_image_artifact.image_name) + params = install_ext.get("params") or {} + place = {"workdir": params.get(WORKDIR_HOST_KEY), "user": params.get(USER_HOST_KEY)} + if not all(isinstance(value, str) and value for value in place.values()): + raise RuntimeError( + f"Agent {agent.id} v{agent.version} install/v1 params don't say where it works on the VM host: a " + f"stand-in installed in its place needs '{WORKDIR_HOST_KEY}' and '{USER_HOST_KEY}'" + ) + return {name: value.format(agent_name=self.agent_name) for name, value in place.items()} + def _register_instance(self, agent, a2a_url: str, ds, agent_card: dict): """Register the installed agent in the instance store, same as A2AAgent.deploy(), so store-resolving consumers (AddSkillsTaskStep, the a2a-agent CLIs) can find it.""" diff --git a/tst/integration/agent/stand_in_modal_vm_test.py b/tst/integration/agent/stand_in_modal_vm_test.py new file mode 100644 index 00000000..425671b6 --- /dev/null +++ b/tst/integration/agent/stand_in_modal_vm_test.py @@ -0,0 +1,81 @@ +"""An agent installed on a Modal VM's host, played from this machine through the stand-in installed in its place: the +shape of a task that loads its inputs onto the VM, installs a harness there and checks the outputs it leaves.""" + +import asyncio +import tempfile +import uuid +from pathlib import Path + +import modal +import pytest + +from agent_env.a2a_agent.a2a_agent import A2AAgent +from agent_env.artifact import DockerImageArtifact, FileArtifact +from agent_env.task import EpisodeSpec, Task +from tst.util.capabilities import skip_without_remote_sandbox + +pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow, skip_without_remote_sandbox("modal_vm")] + +# An agent the stand-in replaces needs only a card declaring where it works on a VM host. +REPLACED_SERVER = """ +class _Card: + def model_dump(self): + return {"name": "replaced", "capabilities": {"extensions": [{"uri": "urn:agentenv:install/v1", "params": { + "workdir_host": "/opt/agent-{agent_name}/workspace", "user_host": "agent-{agent_name}", + "install_commands_host": ["false"], "a2a_port": 8000}}]}} + + +AGENT_CARD = _Card() +""" +REPLACED_DOCKERFILE = """FROM mirror.gcr.io/library/python:3.12-slim +WORKDIR /app +COPY a2a_server.py . +""" + + +@pytest.mark.asyncio +async def test_an_agent_installed_on_a_vm_host_is_played_through_a_stand_in_installed_in_its_place(): + uid = uuid.uuid4().hex[:8] + greeting = FileArtifact.put_bytes(id=f"greeting-vm-{uid}", description="stand-in on a VM", filename="greeting.txt", + content=b"hello", content_type="text/plain") + task = Task.from_dict({"id": f"copy-vm-{uid}", "steps": [ + {"id": "vm", "type": "deploy_sandbox", "sandbox_name": "vm", "sandbox_mode": "vm", "sandbox_type": "modal_vm", + "exposed_ports": [8000], "cpu": 2, "memory_mb": 8192, "ttl_seconds": 1800}, + {"id": "load", "type": "load_artifact", "sandbox_name": "vm", "artifact_id": greeting.id, + "destination_path": "/opt/agent-solver/workspace"}, + {"id": "install", "type": "install_agent", "sandbox_name": "vm", "a2a_agent_id": _replaced(uid).id, + "agent_name": "solver"}, + {"id": "solve", "type": "prompt_agent", "agent_name": "solver", "prompt_id": "solve", + "prompt": "Copy greeting.txt to outputs/answer.txt."}, + {"id": "check", "type": "verify_sandbox", "sandbox_name": "vm", "base_dir": "/opt/agent-solver/workspace", + "criteria": [{"type": "probe_file_contains", "criterion": "the greeting is copied", + "paths": ["outputs/answer.txt"], "expected": "^hello$"}]}, + ]}) + + episode = await EpisodeSpec.from_task(task).start() + try: + turn = await episode.next_turn() + async with turn.tools() as tools: + assert [tool.name for tool in tools.list()] == ["bash"] + assert (await tools.call("bash", {"command": "pwd; whoami; cat greeting.txt"})).text == ( + "/opt/agent-solver/workspace\nagent-solver\nhello") + await tools.call("bash", {"command": "mkdir -p outputs && cp greeting.txt outputs/answer.txt"}) + await turn.end("Copied.") + assert await episode.result() == {"check": 1.0} + finally: + await episode.abandon("the test ended") + [vm] = Task.get_instance(episode.id).context["deployed_sandboxes"] + sandbox = await modal.Sandbox.from_id.aio(vm["sandbox_id"]) + async with asyncio.timeout(60): + while await sandbox.poll.aio() is None: + await asyncio.sleep(2) + + +def _replaced(uid: str) -> A2AAgent: + with tempfile.TemporaryDirectory() as build_dir: + context = Path(build_dir) + (context / "a2a_server.py").write_text(REPLACED_SERVER) + (context / "Dockerfile").write_text(REPLACED_DOCKERFILE) + image = DockerImageArtifact.put_context(f"replaced-agent-image-{uid}", description="stand-in on a VM", + context_path=str(context), dockerfile_path="Dockerfile", platform=None) + return A2AAgent.put(id=f"replaced-agent-{uid}", docker_image_artifact=image) diff --git a/tst/integration/env/envs/server_env_local_test.py b/tst/integration/env/envs/server_env_local_test.py index 2481076c..e15f4c1b 100644 --- a/tst/integration/env/envs/server_env_local_test.py +++ b/tst/integration/env/envs/server_env_local_test.py @@ -211,7 +211,8 @@ async def verify(mcp_url): @pytest.mark.asyncio async def test_a_tasks_agent_is_played_from_a_loop_of_ones_own_as_an_episode(local_stack, tmp_path): """A trainer's view of a seeded task: the env deployed and loaded and the echo agent prompted around the outside - agent, which plays its turn through the env's tools as its role and is never deployed; its reply recorded and + agent, which plays its turn through the env's tools as its role through a stand-in served in this process; its + reply recorded and scored under the verifier's step; and every episode, played, not played, abandoned or forgotten past its turn's deadline, leaving nothing behind.""" uid = uuid.uuid4().hex[:8] @@ -237,7 +238,9 @@ async def test_a_tasks_agent_is_played_from_a_loop_of_ones_own_as_an_episode(loc async def play(episode, add): turn = await episode.next_turn() assert turn.prompt == "Add a crate." and turn.role == "alice@example.com" - assert [agent["agent_name"] for agent in Task.get_instance(episode.id).context["deployed_agents"]] == ["echo"] + assert [(agent["agent_name"], bool(agent["sandbox_id"])) + for agent in Task.get_instance(episode.id).context["deployed_agents"]] == [("policy", False), + ("echo", True)] if add: async with turn.tools() as tools: assert {"items_add_item", "items_whoami"} <= {tool.name for tool in tools.list()} @@ -252,7 +255,7 @@ async def play(episode, add): assert list(sandboxes.glob("agent-env-*")) == [] record = Task.get_instance(played.id) assert record.status == "completed" and record.context["metadata"]["episode"]["trainer_agents"] == ["policy"] - assert [agent["agent_name"] for agent in record.context["deployed_agents"]] == ["echo"] + assert [agent["agent_name"] for agent in record.context["deployed_agents"]] == ["policy", "echo"] replies = {response["step_id"]: response["response"] for response in record.context["prompt_responses"]} assert replies["play"] == "Added a crate." and replies["ask-echo"].startswith("Echo:") @@ -265,7 +268,7 @@ async def play(episode, add): forgotten = await EpisodeSpec.from_task(task, trainer_agents=["policy"], seed=seed, turn_timeout=3).start() await forgotten.next_turn() - with pytest.raises(TimeoutError, match="turn 'play' of agent 'policy' wasn't ended within 3s"): + with pytest.raises(TimeoutError, match="did not complete within 3s"): await forgotten.next_turn() assert Task.get_instance(forgotten.id).status == "failed" assert list(sandboxes.glob("agent-env-*")) == [] @@ -299,3 +302,43 @@ def _items_artifact(uid: str) -> EnvironmentArtifact: file_artifact=FileArtifact.put_bytes(id=f"server-items-file-{uid}", description="server env e2e", filename="items.json", content=json.dumps({"items": ["snap-x", "snap-y"]}).encode(), content_type="application/json"), ) + + +@pytest.mark.asyncio +async def test_an_agent_that_works_in_its_sandbox_is_played_through_a_stand_in_deployed_there(local_stack, tmp_path): + """A task that loads files into its agent and checks what the agent left there: the agent is never deployed, + a stand-in is, in its place, and the loop's ``bash`` runs in it, as its user and in its folder, with no model key + of its own; the check scores what the commands left, and nothing is left running.""" + uid = uuid.uuid4().hex[:8] + greeting = FileArtifact.put_bytes(id=f"greeting-{uid}", description="stand-in e2e", filename="greeting.txt", + content=b"hello", content_type="text/plain") + task = Task.from_dict({"id": f"copy-{uid}", "steps": [ + {"id": "agent", "type": "deploy_agent", "agent_name": "solver", "a2a_agent_id": f"never-deployed-{uid}"}, + {"id": "load", "type": "load_artifact", "agent_name": "solver", "artifact_id": greeting.id, + "destination_path": "/app/inputs"}, + {"id": "solve", "type": "prompt_agent", "agent_name": "solver", "prompt_id": "solve", + "prompt": "Copy inputs/greeting.txt to outputs/answer.txt.", "poll_interval_seconds": 0.5}, + {"id": "check", "type": "verify_sandbox", "agent_name": "solver", "criteria": [ + {"type": "probe_file_contains", "criterion": "the greeting is copied", "paths": ["outputs/answer.txt"], + "expected": "^hello$"}]}, + ]}) + + episode = await EpisodeSpec.from_task(task).start() + turn = await episode.next_turn() + async with turn.tools() as tools: + assert [tool.name for tool in tools.list()] == ["bash"] + assert (await tools.call("bash", {"command": "pwd; whoami; cat inputs/greeting.txt"})).text == ( + "/app\nagent\nhello") + assert (await tools.call("bash", {"command": "printenv LITELLM_API_KEY"})).text == "unused\n" + await tools.call("bash", {"command": "mkdir -p outputs && cp inputs/greeting.txt outputs/answer.txt"}) + await turn.end("Copied.") + + assert await episode.result() == {"check": 1.0} + record = Task.get_instance(episode.id) + [solver] = record.context["deployed_agents"] + assert solver["agent_name"] == "solver" and solver["sandbox_id"] + [response] = record.context["prompt_responses"] + assert (response["response"], response["tool_call_count"]) == ("Copied.", 3) + assert response["agent_trajectory_object_url"] + assert list((tmp_path / "sandboxes").glob("agent-env-*")) == [] + assert _docker("ps", "-aq", "--filter", f"name=^/agent-{solver['sandbox_id']}$").stdout.strip() == "" diff --git a/tst/integration/env/envs/server_env_modal_test.py b/tst/integration/env/envs/server_env_modal_test.py index ed7ef874..2eb2c002 100644 --- a/tst/integration/env/envs/server_env_modal_test.py +++ b/tst/integration/env/envs/server_env_modal_test.py @@ -112,7 +112,7 @@ async def start(spec): forgotten = await start(EpisodeSpec.from_task(task, turn_timeout=3)) await forgotten.next_turn() - with pytest.raises(TimeoutError, match="wasn't ended within 3s"): + with pytest.raises(TimeoutError, match="did not complete within 3s"): await forgotten.next_turn() for episode in (played, abandoned, forgotten): diff --git a/tst/unit/a2a_agent/stand_in_test.py b/tst/unit/a2a_agent/stand_in_test.py new file mode 100644 index 00000000..84db9d24 --- /dev/null +++ b/tst/unit/a2a_agent/stand_in_test.py @@ -0,0 +1,38 @@ +"""The stand-in agent the Episode API deploys in place of an agent played from outside: written to the store as an +image's build context, and written again only when that context changes.""" + +from agent_env.a2a_agent import stand_in +from agent_env.a2a_agent.a2a_agent import SHARE_STORE_CREDENTIALS +from agent_env.a2a_agent.stand_in import STAND_IN_AGENT_ID, stand_in_agent + + +def test_the_stand_in_is_a_build_context_given_no_model_key_or_store_credentials(local_stores, cli_routing): + agent = stand_in_agent() + + assert agent.id == STAND_IN_AGENT_ID + assert agent.docker_image_artifact.context_only + assert agent.default_env_vars == {"LITELLM_API_KEY": "unused", "LITELLM_BASE_URL": "http://unused.invalid"} + assert agent.metadata == {SHARE_STORE_CREDENTIALS: False} + + +def test_an_unchanged_context_reuses_the_stand_in_and_a_changed_one_writes_it_again( + local_stores, cli_routing, monkeypatch): + first = stand_in_agent() + assert stand_in_agent().version == first.version + + monkeypatch.setattr(stand_in, "_requirements", lambda: ["httpx==0.0.1"]) + + assert stand_in_agent().version == first.version + 1 + + +def test_its_context_holds_the_server_the_protocol_package_and_the_pinned_requirements_to_serve(tmp_path): + context = stand_in._stage(tmp_path) + + assert sorted(path.name for path in context.iterdir()) == [ + "Dockerfile", "a2a_server.py", "agentenv_protocol", "requirements.txt"] + assert (context / "agentenv_protocol" / "a2a_agent" / "stand_in.py").is_file() + assert not list(context.rglob("__pycache__")) + requirements = (context / "requirements.txt").read_text().splitlines() + assert {line.split("==")[0] for line in requirements} == { + "a2a-sdk[http-server]", "httpx", "pydantic", "python-multipart", "regex", "starlette", "uvicorn"} + assert all(line.split("==")[1] for line in requirements) diff --git a/tst/unit/providers/sandbox_providers/test_reachability_contributions.py b/tst/unit/providers/sandbox_providers/test_reachability_contributions.py index e22b6e5c..2b931d90 100644 --- a/tst/unit/providers/sandbox_providers/test_reachability_contributions.py +++ b/tst/unit/providers/sandbox_providers/test_reachability_contributions.py @@ -245,7 +245,7 @@ def test_a2a_merged_env_carries_contributions_and_they_win_over_caller_env(cf_cr from agent_env.a2a_agent.a2a_agent import A2AAgent - record = SimpleNamespace(default_env_vars={"AWS_ACCESS_KEY_ID": "preset"}) + record = SimpleNamespace(default_env_vars={"AWS_ACCESS_KEY_ID": "preset"}, metadata={}) merged = A2AAgent._build_merged_env(record, {_CF_ID: "caller-supplied"}, 8000) assert merged[_CF_ID] == "cf-id" assert merged[_CF_SECRET] == "cf-secret" diff --git a/tst/unit/store/aws/test_shared_credentials.py b/tst/unit/store/aws/test_shared_credentials.py index e67c7390..65efc460 100644 --- a/tst/unit/store/aws/test_shared_credentials.py +++ b/tst/unit/store/aws/test_shared_credentials.py @@ -5,7 +5,7 @@ import pytest from agentenv_protocol import client as protocol_v1 -from agent_env.a2a_agent.a2a_agent import A2AAgent +from agent_env.a2a_agent.a2a_agent import SHARE_STORE_CREDENTIALS, A2AAgent from agent_env.config import set_object_store from agent_env.store import routing from agent_env.store.object_store import s3_object_store @@ -33,8 +33,9 @@ def host_credentials(monkeypatch): monkeypatch.setattr(s3_object_store.boto3, "Session", lambda: SimpleNamespace(get_credentials=lambda: creds)) -def _merged_env(caller_env: dict[str, str] | None = None) -> dict[str, str]: - return A2AAgent._build_merged_env(SimpleNamespace(default_env_vars={}), caller_env or {}, 8000) +def _merged_env(caller_env: dict[str, str] | None = None, metadata: dict | None = None) -> dict[str, str]: + return A2AAgent._build_merged_env(SimpleNamespace(default_env_vars={}, metadata=metadata or {}), caller_env or {}, + 8000) @pytest.mark.asyncio @@ -80,3 +81,8 @@ def test_a_caller_supplied_key_suppresses_the_whole_shared_set(host_credentials) merged = _merged_env({"AWS_ACCESS_KEY_ID": "AKIA-CALLER", "AWS_SECRET_ACCESS_KEY": "caller-secret"}) assert merged["AWS_ACCESS_KEY_ID"] == "AKIA-CALLER" assert "AWS_SESSION_TOKEN" not in merged + + +def test_an_agent_that_opts_out_gets_none_of_the_shared_set(host_credentials): + set_object_store(S3ObjectStore(object(), "bucket", region="us-west-2", share_credentials=True)) + assert not any(key.startswith("AWS_") for key in _merged_env(metadata={SHARE_STORE_CREDENTIALS: False})) diff --git a/tst/unit/task/episode/conftest.py b/tst/unit/task/episode/conftest.py new file mode 100644 index 00000000..15854fe4 --- /dev/null +++ b/tst/unit/task/episode/conftest.py @@ -0,0 +1,12 @@ +"""An episode's turns reach a stand-in agent this process serves on loopback, so these tests allow loopback +connections and nothing else.""" + +import pytest +from pytest_socket import enable_socket, socket_allow_hosts + + +@pytest.fixture(autouse=True) +def _disable_network(): + socket_allow_hosts(["127.0.0.1"], allow_unix_socket=True) + yield + enable_socket() diff --git a/tst/unit/task/episode/run_test.py b/tst/unit/task/episode/run_test.py index 4cd5ccdb..f5ac223d 100644 --- a/tst/unit/task/episode/run_test.py +++ b/tst/unit/task/episode/run_test.py @@ -1,12 +1,14 @@ """The run of an episode, on fake steps: its turns, its scores, abandoning it, teardown and its record.""" import asyncio +import json import pytest import pytest_asyncio from agentenv_protocol import EnvironmentTool, ToolResult import agent_env.task.episode.run as episode_run +from agent_env.config import get_config from agent_env.config.runtime import Config from agent_env.env.env import DeployedEnv from agent_env.task import EpisodeError, EpisodeSpec, Task @@ -156,7 +158,7 @@ async def teardowns(monkeypatch, local_stores): ALICE = {"id": "alice", "type": "deploy_agent", "agent_name": "alice", "env_ids": ["items"], "role": "alice@example.com", "system_prompt": "You keep the items."} TURN = {"id": "turn", "type": "prompt_agent", "agent_name": "alice", "prompt_id": "turn", "prompt": "Add a crate.", - "timeout_seconds": 5} + "timeout_seconds": 5, "poll_interval_seconds": 0.05} REPLIES = {"id": "replies", "type": "episode_test_replies", "verifier_id": "replies"} @@ -194,7 +196,8 @@ async def test_the_run_waits_at_an_outside_turn_and_carries_on_once_it_ends(tear assert record.context["metadata"]["episode"] == {"id": episode.id, "trainer_agents": ["alice"]} assert [(r["step_id"], r["prompt_text"], r["response"]) for r in record.context["prompt_responses"]] == [ ("turn", "Add a crate.", "Added a crate.")] - assert record.context["deployed_agents"] == [] + assert [(agent["agent_name"], agent["sandbox_id"]) for agent in record.context["deployed_agents"]] == [ + ("alice", None)] @pytest.mark.asyncio @@ -322,32 +325,32 @@ async def test_a_setup_failure_reaches_the_caller_waiting_for_a_turn(teardowns): @pytest.mark.asyncio async def test_a_turn_not_taken_by_its_deadline_fails_the_run_and_is_never_handed_out(teardowns): - episode = await _spec(ITEMS, ALICE, {**TURN, "timeout_seconds": 0.05}, REPLIES).start() + episode = await _spec(ITEMS, ALICE, {**TURN, "timeout_seconds": 1}, REPLIES).start() await _ended(episode) - with pytest.raises(TimeoutError, match="turn 'turn' of agent 'alice' wasn't ended within 0.05s"): + with pytest.raises(TimeoutError, match="did not complete within 1s"): await episode.next_turn() assert Task.get_instance(episode.id).status == "failed" @pytest.mark.asyncio async def test_a_turn_ended_after_its_deadline_changes_nothing(teardowns): - episode = await _spec(ITEMS, ALICE, {**TURN, "timeout_seconds": 0.5}, REPLIES).start() + episode = await _spec(ITEMS, ALICE, {**TURN, "timeout_seconds": 1}, REPLIES).start() turn = await episode.next_turn() await _ended(episode) await turn.end("too late") - with pytest.raises(TimeoutError, match="wasn't ended within 0.5s"): + with pytest.raises(TimeoutError, match="did not complete within 1s"): await episode.result() record = Task.get_instance(episode.id) - assert record.status == "failed" and "wasn't ended within 0.5s" in record.error + assert record.status == "failed" and "did not complete within 1s" in record.error assert record.context["prompt_responses"] == [] @pytest.mark.asyncio async def test_a_tolerant_turn_past_its_deadline_is_scored_without_a_reply(teardowns): - tolerant = {**TURN, "timeout_seconds": 0.05, "fail_task_on_error": False} + tolerant = {**TURN, "timeout_seconds": 1, "fail_task_on_error": False} episode = await _spec(ITEMS, ALICE, tolerant, REPLIES).start() await _ended(episode) @@ -586,3 +589,95 @@ async def test_a_cancelled_start_takes_its_run_down_with_it(teardowns): assert not episode_run._live assert teardowns.finished == teardowns.started and len(teardowns.finished) == 1 + + +class _SandboxedStandIn: + """Stands in for the client of a stand-in deployed in a sandbox: runs each command as ``ran ``, and + records each reply.""" + + def __init__(self): + self.commands = [] + self.replies = [] + + async def run(self, command): + self.commands.append(command) + return f"ran {command}" + + async def reply(self, task_id, **reply): + self.replies.append((task_id, reply)) + return True + + +def _sandboxed_turn(stand_in): + return episode_run.Turn(agent="alice", step_id="turn", prompt="Add a crate.", system_prompt=None, role=None, + mcp_urls={"items": ITEMS_MCP}, skills=[], shell=True, stand_in=stand_in, task_id="t-1") + + +@pytest.mark.asyncio +async def test_a_turn_in_the_agents_sandbox_offers_bash_beside_its_envs_tools(monkeypatch): + _tool_sessions(monkeypatch, {ITEMS_MCP: ["items_add_item"]}) + stand_in = _SandboxedStandIn() + + async with _sandboxed_turn(stand_in).tools() as tools: + assert [tool.name for tool in tools.list()] == ["items_add_item", "bash"] + assert (await tools.call("bash", {"command": "ls inputs"})).text == "ran ls inputs" + with pytest.raises(EpisodeError, match="tool 'bash' takes a command string, not 3"): + await tools.call("bash", {"command": 3}) + + assert stand_in.commands == ["ls inputs"] + + +@pytest.mark.asyncio +async def test_an_env_tool_named_bash_clashes_with_the_sandboxes_own(monkeypatch): + _tool_sessions(monkeypatch, {ITEMS_MCP: ["bash"]}) + + with pytest.raises(EpisodeError, match="tool 'bash' is offered by env 'items', and the agent's sandbox offers"): + async with _sandboxed_turn(_SandboxedStandIn()).tools(): + pass + + +@pytest.mark.asyncio +async def test_a_turns_calls_and_its_reply_are_handed_back_as_its_trajectory(monkeypatch): + _tool_sessions(monkeypatch, {ITEMS_MCP: ["items_add_item"]}) + stand_in = _SandboxedStandIn() + turn = _sandboxed_turn(stand_in) + + async with turn.tools() as tools: + await tools.call("items_add_item", {"name": "crate"}) + await tools.call("bash", {"command": "ls"}) + await turn.end("Added a crate.") + + [(task_id, reply)] = stand_in.replies + assert (task_id, reply["text"], reply["tool_call_count"]) == ("t-1", "Added a crate.", 2) + spans = reply["trajectory"] + assert [(span["name"], span["attributes"]["gen_ai.operation.name"]) for span in spans] == [ + ("items_add_item", "execute_tool"), ("bash", "execute_tool"), ("turn", "chain")] + assert json.loads(spans[0]["attributes"]["gen_ai.prompt"]) == {"input": {"name": "crate"}} + assert json.loads(spans[1]["attributes"]["gen_ai.completion"]) == {"output": [{"type": "text", "text": "ran ls"}]} + assert json.loads(spans[2]["attributes"]["gen_ai.completion"]) == { + "content": [{"type": "text", "text": "Added a crate."}]} + assert all(span["start_time"] <= span["end_time"] for span in spans) + + +@pytest.mark.asyncio +async def test_the_turns_trajectory_is_stored_as_any_agents_is(teardowns, monkeypatch): + _tool_sessions(monkeypatch, {ITEMS_MCP: ["items_add_item"]}) + episode = await _spec(ITEMS, ALICE, TURN, REPLIES).start() + turn = await episode.next_turn() + + async with turn.tools() as tools: + await tools.call("items_add_item", {"name": "crate"}) + await turn.end("Added a crate.") + await episode.result() + + [response] = Task.get_instance(episode.id).context["prompt_responses"] + stored = json.loads(get_config().get_object_store().get(response["agent_trajectory_object_url"])) + assert [span["name"] for span in stored] == ["items_add_item", "turn"] + assert response["tool_call_count"] == 1 + + +@pytest.mark.asyncio +async def test_a_prompts_own_system_prompt_wins_over_the_agents(teardowns): + episode = await _spec(ITEMS, ALICE, {**TURN, "system_prompt": "Count first."}, REPLIES).start() + + assert (await episode.next_turn()).system_prompt == "Count first." diff --git a/tst/unit/task/episode/spec_test.py b/tst/unit/task/episode/spec_test.py index 14e4d4f2..1246b673 100644 --- a/tst/unit/task/episode/spec_test.py +++ b/tst/unit/task/episode/spec_test.py @@ -5,7 +5,7 @@ from agent_env.task import EpisodeError, EpisodeSpec, Task from agent_env.task.episode.rules import CHECKS, UNAFFECTED -from agent_env.task.episode.steps import _EpisodeAgentStep, _EpisodeTurnStep +from agent_env.task.episode.steps import _EpisodeAgentStep from agent_env.task.task import substitute_seed_in_steps from agent_env.task_step.registry import _builtin_registry from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep @@ -53,20 +53,7 @@ def test_an_agent_named_by_the_seed_is_the_one_played_from_outside(): assert spec.trainer_agents == ("planner",) assert [step.type for step in spec._copy.steps] == [ - "deploy_env", "load_artifact", "episode_agent", "episode_turn", "env_outcome_verifier"] - - -def test_a_prompts_own_system_prompt_wins_over_the_agents(): - spec = EpisodeSpec.from_task(_portsim(play={**PLAY, "system_prompt": "Plan only the berths."})) - - assert spec._copy.steps[3].system_prompt == "Plan only the berths." - - -def test_each_prompt_of_a_trainer_agent_becomes_a_turn(): - again = {**PLAY, "id": "replan", "prompt_id": "replan", "prompt": "Replan.", "depends_on": ["play"]} - spec = EpisodeSpec.from_task(_portsim(again)) - - assert [step.id for step in spec._copy.steps if step.type == "episode_turn"] == ["play", "replan"] + "deploy_env", "load_artifact", "episode_agent", "prompt_agent", "env_outcome_verifier"] def test_a_task_prompting_several_agents_needs_the_ones_to_play_named(): @@ -97,60 +84,97 @@ def test_the_copy_stands_in_for_the_outside_agents_steps_and_keeps_everything_el assert (copy.id, copy.version) == ("dock", 4) assert [step.type for step in copy.steps] == [ - "deploy_env", "load_artifact", "episode_agent", "episode_turn", "env_outcome_verifier", "deploy_agent", + "deploy_env", "load_artifact", "episode_agent", "prompt_agent", "env_outcome_verifier", "deploy_agent", "prompt_agent", ] assert [copy.steps[i] is task.steps[i] for i in range(len(task.steps))] == [ - True, True, False, False, True, True, True] + True, True, False, True, True, True, True] for original, stand_in in zip(task.steps, copy.steps): assert (stand_in.id, stand_in.version, stand_in.depends_on, stand_in.fail_task_on_error) == ( original.id, original.version, original.depends_on, original.fail_task_on_error) -def test_the_stand_ins_carry_what_the_run_needs_unseeded_and_round_trip(): - agent, turn = EpisodeSpec.from_task(_portsim(), seed=SEED)._copy.steps[2:4] +def test_the_outside_agents_deploy_runs_in_a_stand_in_unseeded_and_round_trips(): + task = _portsim() + agent = EpisodeSpec.from_task(task, seed=SEED)._copy.steps[2] - assert isinstance(agent, _EpisodeAgentStep) and isinstance(turn, _EpisodeTurnStep) - assert (agent.agent_name, agent.env_ids, agent.env_step_id, agent.role) == ( - "default-agent", ["portsim"], None, "planner@") - assert (turn.agent_name, turn.prompt_id, turn.prompt, turn.system_prompt, turn.timeout_seconds) == ( - "default-agent", "play", "Plan week .", "You plan .", 600) - for step in (agent, turn): - assert type(step).from_dict(step.to_dict()).to_dict() == step.to_dict() + assert isinstance(agent, _EpisodeAgentStep) + assert (agent.step, agent.sandboxed) == (task.steps[2], False) + assert agent.step.role == "planner@" + assert _EpisodeAgentStep.from_dict(agent.to_dict()).to_dict() == agent.to_dict() -@pytest.mark.parametrize( - ("timeout", "deadline"), [(30, 30), (None, PromptAgentTaskStep.DEFAULT_PROMPT_TIMEOUT_SECONDS)]) -def test_a_turns_deadline_is_its_prompts_timeout(timeout, deadline): - turn = EpisodeSpec.from_task(_portsim(play={**PLAY, "timeout_seconds": timeout}))._copy.steps[3] +def test_without_a_turn_timeout_a_turns_deadline_is_its_prompts_own(): + task = _portsim(play={**PLAY, "timeout_seconds": 30}) - assert turn.timeout_seconds == deadline + assert EpisodeSpec.from_task(task)._copy.steps[3] is task.steps[3] -def test_a_turn_timeout_replaces_every_outside_prompts_own(): +@pytest.mark.parametrize(("turn_timeout", "deadline"), [(90, 90), (0.5, 1)]) +def test_a_turn_timeout_replaces_every_outside_prompts_own_in_whole_seconds(turn_timeout, deadline): again = {**PLAY, "id": "replan", "prompt_id": "replan", "timeout_seconds": 60} - copy = EpisodeSpec.from_task(_portsim(again, play={**PLAY, "timeout_seconds": 7200}), turn_timeout=90)._copy + task = _portsim(again, play={**PLAY, "timeout_seconds": 7200}) + copy = EpisodeSpec.from_task(task, turn_timeout=turn_timeout)._copy - assert [step.timeout_seconds for step in copy.steps if step.type == "episode_turn"] == [90, 90] + assert [step.timeout_seconds for step in copy.steps if step.type == "prompt_agent"] == [deadline, deadline] + assert copy.steps[3].to_dict() == {**task.steps[3].to_dict(), "timeout_seconds": deadline} def test_an_inside_agent_keeps_its_own_steps(): copy = EpisodeSpec.from_task(_portsim(BOB, ASK_BOB), trainer_agents=["bob"])._copy assert type(copy.steps[2]) is DeployAgentTaskStep and type(copy.steps[3]) is PromptAgentTaskStep - assert [step.type for step in copy.steps[5:]] == ["episode_agent", "episode_turn"] + assert [step.type for step in copy.steps[5:]] == ["episode_agent", "prompt_agent"] + + +INSTALL = {"id": "agent", "type": "install_agent", "sandbox_name": "box", "a2a_agent_id": "codex"} +IN_THE_SANDBOX = { + "an install": ({}, {"agent": INSTALL}), + "skills on the deploy": ({}, {"agent": {**AGENT, "skills": [{"name": "s", "skill_md": "Plan berths first."}]}}), + "add_skills": (({"id": "s", "type": "add_skills", "skills": [{"name": "s", "description": "d", "body": "x"}]},), + {}), + "a load into the agent": (({"id": "s", "type": "load_artifact", "artifact_id": "a", + "agent_name": "default-agent"},), {}), + "a collect from the agent": (({"id": "s", "type": "collect_artifacts", "artifact_paths": ["/out"]},), {}), + "run_code in the agent": (({"id": "s", "type": "run_code", "script_artifact_id": "s", "entrypoint": "main.py"},), + {}), + "verify_sandbox": (({"id": "s", "type": "verify_sandbox", "criteria": []},), {}), +} +OUTSIDE_THE_SANDBOX = { + "env tools alone": ((), {}), + "a load into the env": (({**LOAD, "id": "load-2"},), {}), + "a collect from the env": (({"id": "s", "type": "collect_artifacts", "env_id": "portsim", + "artifact_paths": ["/out"]},), {}), + "run_code in the env": (({"id": "s", "type": "run_code", "script_artifact_id": "s", "entrypoint": "main.py", + "env_id": "portsim"},), {}), + "verify_sandbox on a sandbox": (({"id": "s", "type": "verify_sandbox", "sandbox_name": "box", "criteria": []},), + {}), +} + + +@pytest.mark.parametrize(("extra", "replace"), IN_THE_SANDBOX.values(), ids=IN_THE_SANDBOX.keys()) +def test_a_task_using_the_outside_agents_sandbox_deploys_its_stand_in_there(extra, replace): + assert EpisodeSpec.from_task(_portsim(*extra, **replace))._copy.steps[2].sandboxed + + +@pytest.mark.parametrize(("extra", "replace"), OUTSIDE_THE_SANDBOX.values(), ids=OUTSIDE_THE_SANDBOX.keys()) +def test_a_task_using_only_the_outside_agents_envs_serves_its_stand_in_here(extra, replace): + assert not EpisodeSpec.from_task(_portsim(*extra, **replace))._copy.steps[2].sandboxed AT_THE_AGENT = {"agent_name": "default-agent"} REFUSED = { "no deploy_agent": ({}, {"agent": None}, "agent 'default-agent' is played from outside, so it needs exactly one " - "deploy_agent step, not 0"), - "two deploy_agents": (({**AGENT, "id": "agent-2"},), {}, "needs exactly one deploy_agent step, not 2"), + "deploy_agent or install_agent step, not 0"), + "two deploy_agents": (({**AGENT, "id": "agent-2"},), {}, + "needs exactly one deploy_agent or install_agent step, not 2"), + "a deploy and an install": (({**INSTALL, "id": "install"},), {}, + "needs exactly one deploy_agent or install_agent step, not 2"), "deploy_human_agent": (({"id": "human", "type": "deploy_human_agent", **AT_THE_AGENT},), {"agent": None}, "step 'human': deploy_human_agent deploys 'default-agent', which is played from outside"), - "install_agent": (({"id": "install", "type": "install_agent", "sandbox_name": "box", "a2a_agent_id": "x", - **AT_THE_AGENT},), {}, - "step 'install': install_agent deploys 'default-agent'"), + "an install into a container": ({}, {"agent": {**INSTALL, "container_name": "task"}}, + "step 'agent': an outside agent's stand-in replaces an agent installed on the VM " + "host, not in container 'task'"), "parts": ({}, {"play": {**PLAY, "prompt": None, "parts": [{"kind": "text", "text": "Plan."}]}}, "step 'play': an outside agent's prompt can't have parts, only a text prompt"), "user simulator": ({}, {"play": {**PLAY, "max_conversation_turns": 2}}, @@ -162,28 +186,16 @@ def test_an_inside_agent_keeps_its_own_steps(): "step 'play': an outside agent's prompt can't have agentenv_tools"), "snapshot_config": ({}, {"play": {**PLAY, "snapshot_config": {"env_id": "portsim"}}}, "step 'play': an outside agent's prompt can't have snapshot_config"), - "skills": ({}, {"agent": {**AGENT, "skills": [{"name": "s", "skill_md": "Plan berths first."}]}}, - "step 'agent': an outside agent can't have skills, instructions its player wouldn't get"), "snapshot files": ({}, {"agent": {**AGENT, "agent_snapshot_files_artifact_id": "files"}}, "step 'agent': an outside agent can't have agent_snapshot_files_artifact_id"), "changelog url": ({}, {"agent": {**AGENT, "agent_changelog_object_url": "file:///changelog"}}, "step 'agent': an outside agent can't have agent_changelog_object_url"), "changelog": ({}, {"agent": {**AGENT, "enable_agent_changelog": True}}, "step 'agent': an outside agent can't have enable_agent_changelog"), - "add_skills": (({"id": "s", "type": "add_skills", "skills": [{"name": "s", "description": "d", "body": "x"}]},), {}, - "step 's': add_skills needs 'default-agent' as a real agent, and it is played from outside"), "register_agent_triggers": (({"id": "s", "type": "register_agent_triggers", "triggers": [], **AT_THE_AGENT},), {}, "step 's': register_agent_triggers needs 'default-agent' as a real agent"), "snapshot_agent_state": (({"id": "s", "type": "snapshot_agent_state", "artifact_id": "a", "prompt_id": "play"},), {}, "step 's': snapshot_agent_state needs 'default-agent' as a real agent"), - "load_artifact": (({"id": "s", "type": "load_artifact", "artifact_id": "a", **AT_THE_AGENT},), {}, - "step 's': load_artifact needs 'default-agent' as a real agent"), - "verify_sandbox": (({"id": "s", "type": "verify_sandbox", "criteria": []},), {}, - "step 's': verify_sandbox needs 'default-agent' as a real agent"), - "run_code": (({"id": "s", "type": "run_code", "script_artifact_id": "s", "entrypoint": "main.py"},), {}, - "step 's': run_code needs 'default-agent' as a real agent"), - "collect from the agent": (({"id": "s", "type": "collect_artifacts", "artifact_paths": ["/out"]},), {}, - "step 's': collects from 'default-agent''s sandbox, and it is played from outside"), "collect a manifest": (({"id": "s", "type": "collect_artifacts", "env_id": "portsim", "manifest_step_id": "play"},), {}, "step 's': its manifest is outside prompt 'play'"), "trigger executor": (({"id": "s", "type": "register_env_triggers", "env_id": "portsim", "triggers": [], @@ -192,10 +204,6 @@ def test_an_inside_agent_keeps_its_own_steps(): "peered": ((BOB, {"id": "s", "type": "peer_agents", "peerings": [ {"source_agent_name": "bob", "peer_agent_names": ["default-agent"]}]}), {}, "step 's': peers 'default-agent', which is played from outside"), - "rubric on the trajectory": (({"id": "s", "type": "rubrics_verifier", "prompt_id": "play", "criteria": [], - "use_agent_judge": False},), {}, - "step 's': grades an outside prompt's trajectory, which an outside player doesn't " - "hand back"), "rubric judged in the agent": (({"id": "s", "type": "rubrics_verifier", "prompt_id": "other", "criteria": [], "use_trajectory": False, **AT_THE_AGENT},), {}, "step 's': judges in 'default-agent''s sandbox, and it is played from outside"), @@ -238,6 +246,12 @@ def test_every_refusal_is_reported_at_once(): "use_trajectory": False, "use_agent_judge": False}},), "collecting from an env": ({"collect": {"id": "collect", "type": "collect_artifacts", "env_id": "portsim", "artifact_paths": ["/out"]}},), + "a rubric of the trajectory": ( + {"verify": {"id": "rubric", "type": "rubrics_verifier", "prompt_id": "play", "criteria": [], + "use_agent_judge": False}},), + "installing the outside agent on a VM host": ({"agent": INSTALL},), + **{name: ({"agent": replace["agent"]} if replace else {"extra": extra[0]},) + for name, (extra, replace) in IN_THE_SANDBOX.items() if name != "an install"}, "tearing down an outside agent": ({"down": {"id": "down", "type": "teardown_sandboxes", "agent_names": ["default-agent"]}},), } diff --git a/tst/unit/task_step/install_agent_test.py b/tst/unit/task_step/install_agent_test.py index 17ba0691..c2525076 100644 --- a/tst/unit/task_step/install_agent_test.py +++ b/tst/unit/task_step/install_agent_test.py @@ -129,6 +129,8 @@ def test_a2a_port_roundtrip(): class _InstallSandbox: + ON_THIS_MACHINE = False + def __init__(self): self.scripts: list[str] = [] @@ -310,3 +312,81 @@ def test_the_trust_lines_resolve_as_the_trust_env_param(): resolvers = _build_param_resolvers(container_name="c", work_dir="/w", agent_ctx_tar="/w/a.tgz", a2a_port=8000, config=None, agent_name="a", trust_lines="SSL_CERT_FILE=/x\n") assert resolvers["trust_env"]() == "SSL_CERT_FILE=/x\n" +STAND_IN = "@local/agentenv-stand-in" +CODEX_INSTALL = {"params": {"workdir_host": "/opt/agent-{agent_name}/workspace", "user_host": "agent-{agent_name}", + "install_commands_host": ["install codex"], "a2a_port": 8000}} +STAND_IN_INSTALL = {"params": {"install_commands_host": ["serve as {user} in {workdir} on {a2a_port}"], + "required_params_host": ["user", "workdir", "a2a_port"], "a2a_port": 8000}} + + +def _installing_in_place(monkeypatch, *, codex=CODEX_INSTALL, sandbox=None, container_name=None): + """An install of ``agent-x`` that the step's ``stand_in`` param replaces with the stand-in.""" + sandbox = sandbox or _InstallSandbox() + monkeypatch.setattr(sandbox_provider, "build_sandbox_provider", + lambda name: SimpleNamespace(get_sandbox=_resolved(sandbox))) + agents = {agent_id: SimpleNamespace(id=agent_id, version=1, metadata={}, docker_image_artifact=SimpleNamespace( + image_name=f"{agent_id}-image", build_context_object_url=f"s3://bucket/{agent_id}.tar.gz")) + for agent_id in ("agent-x", STAND_IN)} + monkeypatch.setattr(A2AAgent, "get", staticmethod(lambda agent_id, version=None: agents[agent_id])) + step = InstallAgentTaskStep(id="i", version=None, sandbox_name="vm", a2a_agent_id="agent-x", agent_name="solver", + container_name=container_name) + cards = {"agent-x-image": codex, f"{STAND_IN}-image": STAND_IN_INSTALL} + monkeypatch.setattr(step, "_extract_install_extension", lambda sandbox, image: _resolved(cards[image])()) + monkeypatch.setattr(step, "_wait_for_agent_card", _resolved({})) + monkeypatch.setattr(step, "_register_instance", lambda *args: SimpleNamespace(instance_id="solver-1")) + context = TaskStepContext( + deployed_sandboxes=[DeployedSandbox( + sandbox_name="vm", sandbox_id="sb-1", sandbox_mode="vm", sandbox_type="modal_vm", + tunnel_urls={"8000": "https://vm-8000.example"})], + metadata={"user_overrides": {"step_params": {"i": {"stand_in": STAND_IN}}}, + "deployed_docker_containers": [{"container_name": "task", "sandbox_name": "vm"}]}, + ) + return step, context, sandbox + + +@pytest.mark.asyncio +async def test_a_stand_in_is_installed_where_the_agent_it_replaces_says_it_works(monkeypatch, local_stores): + step, context, sandbox = _installing_in_place(monkeypatch) + + await step.execute(context) + + assert [s for s in sandbox.scripts if s.startswith(("serve", "install"))] == [ + "serve as agent-solver in /opt/agent-solver/workspace on 8000"] + assert [agent.agent_name for agent in context.deployed_agents] == ["solver"] + + +@pytest.mark.asyncio +async def test_a_stand_in_needs_the_replaced_agent_to_say_where_it_works(monkeypatch, local_stores): + step, context, _ = _installing_in_place(monkeypatch, codex={"params": {"install_commands_host": ["install codex"], + "workdir_host": "/opt/agent"}}) + + with pytest.raises(RuntimeError, match="needs 'workdir_host' and 'user_host'"): + await step.execute(context) + + +@pytest.mark.asyncio +async def test_a_stand_in_isnt_installed_into_a_container(monkeypatch, local_stores): + step, context, _ = _installing_in_place(monkeypatch, container_name="task") + + with pytest.raises(RuntimeError, match="installed on the VM host, not in a container"): + await step.execute(context) + + +@pytest.mark.asyncio +async def test_a_stand_in_isnt_installed_on_a_sandbox_that_runs_on_this_machine(monkeypatch, local_stores): + here = _InstallSandbox() + here.ON_THIS_MACHINE = True + step, context, _ = _installing_in_place(monkeypatch, sandbox=here) + + with pytest.raises(RuntimeError, match="runs on this machine"): + await step.execute(context) + + +def test_only_an_install_in_another_agents_place_gives_its_workdir_and_user(): + with pytest.raises(KeyError, match="stand_in"): + _resolvers(None)["workdir"]() + placed = _build_param_resolvers( + container_name=None, work_dir="/tmp/w", agent_ctx_tar="/tmp/w/c.tgz", a2a_port=8000, config=object(), + agent_name="solver", place={"workdir": "/opt/agent-solver/workspace", "user": "agent-solver"}, + ) + assert (placed["workdir"](), placed["user"]()) == ("/opt/agent-solver/workspace", "agent-solver") From f0c609b002a0543994e7f5695a8374eddda6db91 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 19 Oct 2026 15:12:37 -0700 Subject: [PATCH 2/9] docs: an episode plays its outside agents through a stand-in agent Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 25ae1d16..22a0da6f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -202,11 +202,15 @@ orchestrators call it the same way. What callers rely on: - A runner that must hear from a long step puts a zero-argument callable at `context.metadata[HEARTBEAT_KEY]` (`task_step/context.py`); a step that blocks for long calls `context.heartbeat()`, a no-op without one. The key is in `_REDACTED_KEYS`, so it is never persisted. -- An episode (`EpisodeSpec`, `task/episode/`) is one `Task.run()` of a copy of the task in which - the `deploy_agent` and `prompt_agent` steps of each of its `trainer_agents` are the plain-data - `episode_agent` and `episode_turn`, which wait for the caller instead of deploying or prompting. Its - record is tagged `context.metadata["episode"]` and names the original task, so it isn't resumable - with `start_step`. The episode tears down what its run deployed, and an abandoned one is recorded +- An episode (`EpisodeSpec`, `task/episode/`) is one `Task.run()` of a copy of the task, with + namespace routing on, in which each of its `trainer_agents` is played through a stand-in agent + (`agentenv_protocol.a2a_agent.stand_in`, written as `@local/agentenv-stand-in` from its build context). + The agent's `deploy_agent` or `install_agent`, wrapped as `episode_agent`, deploys the stand-in in the + agent's place when the task uses the agent's sandbox and serves it in this process when it doesn't; + the agent's `prompt_agent` steps message it as they would the agent, and the caller plays each + message as a `Turn` through its `urn:agentenv:driver/v1`, where a turn's `bash` runs. Its record is + tagged `context.metadata["episode"]` and names the original task, so it isn't resumable with + `start_step`. The episode tears down what its run deployed, and an abandoned one is recorded `cancelled`. ## Tests From c7773c77240f26599c2433466949320b30c6b7c9 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 19 Oct 2026 15:13:40 -0700 Subject: [PATCH 3/9] fix(task): a turn's reply that can't reach its stand-in leaves the failure to the run Turn.end() never raised before the stand-in; a reply the stand-in can't take is logged, and the run's own failure reaches the caller through next_turn() and result(). Skill, the type of Turn.skills, is exported beside Turn. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/task/__init__.py | 3 ++- src/agent_env/task/episode/__init__.py | 4 ++-- src/agent_env/task/episode/run.py | 13 ++++++++++--- tst/unit/task/episode/run_test.py | 14 ++++++++++++++ 4 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/agent_env/task/__init__.py b/src/agent_env/task/__init__.py index f7d2f7c2..3c5b634d 100644 --- a/src/agent_env/task/__init__.py +++ b/src/agent_env/task/__init__.py @@ -1,7 +1,7 @@ """Task module for AgentEnv.""" from .task import Task -from .episode import Episode, EpisodeError, EpisodeSpec, Turn +from .episode import Episode, EpisodeError, EpisodeSpec, Skill, Turn from .store import ( TaskStore, TaskQuery, get_task_store, set_task_store, reset_task_store, TaskInstance, TaskInstanceStore, get_task_instance_store, set_task_instance_store, reset_task_instance_store, @@ -13,6 +13,7 @@ "Episode", "EpisodeError", "EpisodeSpec", + "Skill", "Turn", "TaskStore", "TaskQuery", diff --git a/src/agent_env/task/episode/__init__.py b/src/agent_env/task/episode/__init__.py index bd00530f..1b8fd59d 100644 --- a/src/agent_env/task/episode/__init__.py +++ b/src/agent_env/task/episode/__init__.py @@ -3,7 +3,7 @@ AgentEnv runs everything else exactly as ``Task.run`` does, so its scores are the ones ``agent-env run`` would report. """ -from .run import Episode, EpisodeError, Turn +from .run import Episode, EpisodeError, Skill, Turn from .spec import EpisodeSpec -__all__ = ["Episode", "EpisodeError", "EpisodeSpec", "Turn"] +__all__ = ["Episode", "EpisodeError", "EpisodeSpec", "Skill", "Turn"] diff --git a/src/agent_env/task/episode/run.py b/src/agent_env/task/episode/run.py index 86abb281..c6e4a464 100644 --- a/src/agent_env/task/episode/run.py +++ b/src/agent_env/task/episode/run.py @@ -78,19 +78,26 @@ def tools(self, *, timeout: float = 600.0) -> TurnTools: async def end(self, response: str | None = None) -> None: """Hand the turn back, and the run carries on. No reply records an empty one, as an agent that answers with - nothing leaves. Once the turn is over, ended or not, this does nothing.""" + nothing leaves. Once the turn is over, ended or not, this does nothing; a stand-in that can't be reached + fails the run, which ``next_turn()`` and ``result()`` report.""" if not self._open: return self._open = False reply = response or "" spans = [*self._spans, _span("turn", "chain", self._started, completion={"content": [ {"type": "text", "text": reply}]})] - await self._stand_in.reply(self._task_id, text=reply, trajectory=spans, tool_call_count=len(self._spans)) + await self._reply(text=reply, trajectory=spans, tool_call_count=len(self._spans)) async def _fail(self, error: Exception) -> None: if self._open: self._open = False - await self._stand_in.reply(self._task_id, error=str(error)) + await self._reply(error=str(error)) + + async def _reply(self, **reply: Any) -> None: + try: + await self._stand_in.reply(self._task_id, **reply) + except httpx.HTTPError as e: + logger.warning("turn %r of agent %r: its reply didn't reach the stand-in: %s", self.step_id, self.agent, e) class TurnTools: diff --git a/tst/unit/task/episode/run_test.py b/tst/unit/task/episode/run_test.py index f5ac223d..35035f81 100644 --- a/tst/unit/task/episode/run_test.py +++ b/tst/unit/task/episode/run_test.py @@ -3,6 +3,7 @@ import asyncio import json +import httpx import pytest import pytest_asyncio from agentenv_protocol import EnvironmentTool, ToolResult @@ -681,3 +682,16 @@ async def test_a_prompts_own_system_prompt_wins_over_the_agents(teardowns): episode = await _spec(ITEMS, ALICE, {**TURN, "system_prompt": "Count first."}, REPLIES).start() assert (await episode.next_turn()).system_prompt == "Count first." + + +@pytest.mark.asyncio +async def test_a_reply_that_cant_reach_the_stand_in_leaves_the_runs_own_failure_to_report(caplog): + class _Unreachable(_SandboxedStandIn): + async def reply(self, task_id, **reply): + raise httpx.ConnectError("the sandbox is gone") + + turn = _sandboxed_turn(_Unreachable()) + + await turn.end("done") + + assert "its reply didn't reach the stand-in: the sandbox is gone" in caplog.text From 4072d2c4f804b00044981e86696b1b7922d1a7a0 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 20 Oct 2026 14:20:59 -0700 Subject: [PATCH 4/9] test: the stand-in's pinned requirements follow the protocol package's agent extra Co-Authored-By: Claude Opus 5.5 --- tst/unit/a2a_agent/stand_in_test.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tst/unit/a2a_agent/stand_in_test.py b/tst/unit/a2a_agent/stand_in_test.py index 84db9d24..11eaebb8 100644 --- a/tst/unit/a2a_agent/stand_in_test.py +++ b/tst/unit/a2a_agent/stand_in_test.py @@ -1,6 +1,8 @@ """The stand-in agent the Episode API deploys in place of an agent played from outside: written to the store as an image's build context, and written again only when that context changes.""" +from importlib.metadata import version + from agent_env.a2a_agent import stand_in from agent_env.a2a_agent.a2a_agent import SHARE_STORE_CREDENTIALS from agent_env.a2a_agent.stand_in import STAND_IN_AGENT_ID, stand_in_agent @@ -32,7 +34,7 @@ def test_its_context_holds_the_server_the_protocol_package_and_the_pinned_requir "Dockerfile", "a2a_server.py", "agentenv_protocol", "requirements.txt"] assert (context / "agentenv_protocol" / "a2a_agent" / "stand_in.py").is_file() assert not list(context.rglob("__pycache__")) - requirements = (context / "requirements.txt").read_text().splitlines() - assert {line.split("==")[0] for line in requirements} == { - "a2a-sdk[http-server]", "httpx", "pydantic", "python-multipart", "regex", "starlette", "uvicorn"} - assert all(line.split("==")[1] for line in requirements) + pinned = dict(line.split("==") for line in (context / "requirements.txt").read_text().splitlines()) + assert {"a2a-sdk[http-server]", "httpx", "pydantic", "starlette", "uvicorn"} <= pinned.keys() + assert not pinned.keys() & {"fastmcp", "pytest", "pytest-asyncio"} + assert all(pinned[name] == version(name.split("[")[0]) for name in pinned) From 0ad10aca647aee9251a33c514cd7269717543948 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 20 Oct 2026 14:21:01 -0700 Subject: [PATCH 5/9] feat(agents): Claude Code declares where it works on a VM host, so a stand-in can take its place Co-Authored-By: Claude Opus 5.5 --- src/agent_env/agents/agents/claude-code/a2a_server.py | 3 +++ tst/unit/agents/claude_code/test_card_contract.py | 1 + 2 files changed, 4 insertions(+) diff --git a/src/agent_env/agents/agents/claude-code/a2a_server.py b/src/agent_env/agents/agents/claude-code/a2a_server.py index 7d13710c..f209bd43 100644 --- a/src/agent_env/agents/agents/claude-code/a2a_server.py +++ b/src/agent_env/agents/agents/claude-code/a2a_server.py @@ -375,6 +375,9 @@ async def _turns_stop_with_the_server(app: Starlette) -> AsyncIterator[None]: required_params=["container", "work_dir", "agent_ctx_tar", "a2a_port", *_ENDPOINT_PARAMS], install_commands_host=_INSTALL_COMMANDS_HOST, required_params_host=["work_dir", "agent_ctx_tar", "a2a_port", "agent_name", *_ENDPOINT_PARAMS], + # Where an install on the host works, and as whom, for a stand-in installed in this agent's place. + workdir_host="/app", + user_host="root", ), enable(TRIGGERS_V1, description=( "Register a typed trigger script and decide the stakeholder reply per turn " diff --git a/tst/unit/agents/claude_code/test_card_contract.py b/tst/unit/agents/claude_code/test_card_contract.py index bd964ead..1b48c910 100644 --- a/tst/unit/agents/claude_code/test_card_contract.py +++ b/tst/unit/agents/claude_code/test_card_contract.py @@ -141,6 +141,7 @@ def test_install_agent_reads_the_card_and_its_install_commands_from_the_image(): (install,) = [ext for ext in json.loads(out.stdout)["capabilities"]["extensions"] if ext["uri"] == "urn:agentenv:install/v1"] assert install["params"]["install_commands"] and install["params"]["install_commands_host"] + assert (install["params"]["workdir_host"], install["params"]["user_host"]) == ("/app", "root") def test_each_install_mode_asks_only_for_params_install_agent_supplies(app): From 89f819fd1de3ab6ab96304fc1ccd590b0125fdd8 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 20 Oct 2026 15:13:07 -0700 Subject: [PATCH 6/9] fix(episode): keep credentials from the stand-in, record failed calls, close turns their step gave up on - An agent whose metadata sets takes_credentials False is given no model key, endpoint or headers, from any source, and none of the object store's shared credentials. The stand-in sets it and carries no placeholder keys. - A tool call that raises is recorded in the turn's trajectory with its error. - A message on a conversation maps to the first of its prompt steps not yet played. - A command that times out or is cancelled has its whole process group killed, so a background child holding its output can't keep it waiting. - A turn whose prompt_agent step has failed is closed at once, rather than a long-poll later when the stand-in drops it, so it is never handed out once the run's record says failed; abandon() closes the turns of the run it cancels. Co-Authored-By: Claude Opus 5.5 --- .../agentenv_protocol/a2a_agent/stand_in.py | 17 +++---- .../tests/test_a2a_stand_in.py | 10 ++++ src/agent_env/a2a_agent/a2a_agent.py | 35 +++++++++----- src/agent_env/a2a_agent/stand_in/__init__.py | 8 +--- src/agent_env/task/episode/run.py | 48 +++++++++++++------ .../env/envs/server_env_local_test.py | 6 +-- tst/unit/a2a_agent/stand_in_test.py | 20 ++++++-- tst/unit/store/aws/test_shared_credentials.py | 4 +- tst/unit/task/episode/run_test.py | 48 +++++++++++++++++-- 9 files changed, 142 insertions(+), 54 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/stand_in.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/stand_in.py index 6d5b5eb5..00e915be 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/stand_in.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/stand_in.py @@ -193,20 +193,21 @@ def _bump(self) -> None: async def _run(command: str) -> str: """``command``'s stdout then its stderr, as verifiers' bash tool returns them. It runs in a process group of its - own, stopped with whatever it started if it outlives ``COMMAND_TIMEOUT_SECONDS``.""" + own, stopped with whatever it started if its output isn't done within ``COMMAND_TIMEOUT_SECONDS``, or if the call + is cancelled, even when ``bash`` has exited and a child it left holds the output open.""" process = await asyncio.create_subprocess_exec( "bash", "-c", command, stdin=asyncio.subprocess.DEVNULL, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, start_new_session=True, ) try: stdout, stderr = await asyncio.wait_for(process.communicate(), COMMAND_TIMEOUT_SECONDS) - except asyncio.TimeoutError: - return f"error: the command timed out after {COMMAND_TIMEOUT_SECONDS}s" - finally: - if process.returncode is None: - with contextlib.suppress(ProcessLookupError): - os.killpg(process.pid, signal.SIGKILL) - await process.wait() + except BaseException as e: + with contextlib.suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGKILL) + await process.wait() + if isinstance(e, asyncio.TimeoutError): + return f"error: the command timed out after {COMMAND_TIMEOUT_SECONDS}s" + raise return (stdout + stderr).decode(errors="replace") diff --git a/packages/agentenv-protocol/tests/test_a2a_stand_in.py b/packages/agentenv-protocol/tests/test_a2a_stand_in.py index 1908c685..50b7189d 100644 --- a/packages/agentenv-protocol/tests/test_a2a_stand_in.py +++ b/packages/agentenv-protocol/tests/test_a2a_stand_in.py @@ -161,6 +161,16 @@ def test_a_command_that_outlives_its_timeout_is_stopped_with_what_it_started(cli os.kill(int((tmp_path / "child").read_text()), 0) +def test_a_child_left_holding_the_output_is_stopped_when_the_command_times_out(client, monkeypatch, tmp_path): + monkeypatch.setattr(stand_in, "COMMAND_TIMEOUT_SECONDS", 0.5) + + output = _ran(client, "sleep 60 & echo $! > child") + + assert output == "error: the command timed out after 0.5s" + with pytest.raises(ProcessLookupError): + os.kill(int((tmp_path / "child").read_text()), 0) + + def test_output_names_a_command_it_started(client): assert _driver(client, "output", exec_id="nope").status_code == 404 diff --git a/src/agent_env/a2a_agent/a2a_agent.py b/src/agent_env/a2a_agent/a2a_agent.py index d54a5794..b65b3410 100644 --- a/src/agent_env/a2a_agent/a2a_agent.py +++ b/src/agent_env/a2a_agent/a2a_agent.py @@ -63,7 +63,10 @@ _DIND_CONTAINER = "agent-dind" _DIND_NETWORK = "agent-docker-net" _DIND_PORT = 2375 -SHARE_STORE_CREDENTIALS = "share_store_credentials" +# Metadata an agent whose commands its player runs sets False: it is given no model endpoint, key or headers, whatever +# a deploy passes, and none of the object store's shared credentials. +TAKES_CREDENTIALS = "takes_credentials" +MODEL_CREDENTIALS_ENV = ("LITELLM_API_KEY", "LITELLM_BASE_URL", MODEL_HEADERS_ENV) def unwritten_local_agent(agent_id: str, field: str) -> str: @@ -455,16 +458,7 @@ async def deploy( provider = None if sandbox is not None else ( build_sandbox_provider(sandbox_type) if sandbox_type else get_agent_sandbox_provider() ) - resolved_env = dict(env_vars) if env_vars else {} - if "LITELLM_API_KEY" not in resolved_env and "LITELLM_API_KEY" not in self.default_env_vars: - resolved_env["LITELLM_API_KEY"] = config.get_litellm_api_key() - if "LITELLM_BASE_URL" not in resolved_env and "LITELLM_BASE_URL" not in self.default_env_vars: - resolved_env["LITELLM_BASE_URL"] = config.get_litellm_base_url() - if MODEL_HEADERS_ENV not in resolved_env and MODEL_HEADERS_ENV not in self.default_env_vars: - headers = config.get_model_headers( - resolved_env.get("LITELLM_BASE_URL", self.default_env_vars.get("LITELLM_BASE_URL"))) - if headers: - resolved_env[MODEL_HEADERS_ENV] = json.dumps(headers) + resolved_env = self._with_model_credentials(env_vars or {}, config) if "LITELLM_BASE_URL" in resolved_env: resolved_env["LITELLM_BASE_URL"] = (provider or sandbox).url_from_sandbox(resolved_env["LITELLM_BASE_URL"]) @@ -545,12 +539,29 @@ def _refusal(self, image: DockerImageArtifact, link: SandboxProvider) -> str | N return f"{how}: {problem}" return None + def _with_model_credentials(self, env_vars: dict[str, str], config) -> dict[str, str]: + """``env_vars`` with the model endpoint, key and headers each taken from config when neither they nor + ``default_env_vars`` set it; for an agent that takes no credentials, with none of them, whatever set them.""" + if not self.metadata.get(TAKES_CREDENTIALS, True): + return {name: value for name, value in env_vars.items() if name not in MODEL_CREDENTIALS_ENV} + resolved_env = dict(env_vars) + if "LITELLM_API_KEY" not in resolved_env and "LITELLM_API_KEY" not in self.default_env_vars: + resolved_env["LITELLM_API_KEY"] = config.get_litellm_api_key() + if "LITELLM_BASE_URL" not in resolved_env and "LITELLM_BASE_URL" not in self.default_env_vars: + resolved_env["LITELLM_BASE_URL"] = config.get_litellm_base_url() + if MODEL_HEADERS_ENV not in resolved_env and MODEL_HEADERS_ENV not in self.default_env_vars: + headers = config.get_model_headers( + resolved_env.get("LITELLM_BASE_URL", self.default_env_vars.get("LITELLM_BASE_URL"))) + if headers: + resolved_env[MODEL_HEADERS_ENV] = json.dumps(headers) + return resolved_env + def _build_merged_env(self, resolved_env: dict[str, str], a2a_port: int) -> dict[str, str]: merged_env = dict(self.default_env_vars) merged_env.update(resolved_env) merged_env["A2A_PORT"] = str(a2a_port) shared = get_config().get_object_store().shared_credentials_env() - if self.metadata.get(SHARE_STORE_CREDENTIALS, True) and shared.keys().isdisjoint(merged_env): + if self.metadata.get(TAKES_CREDENTIALS, True) and shared.keys().isdisjoint(merged_env): merged_env.update(shared) merged_env.update(all_sandbox_container_env()) diff --git a/src/agent_env/a2a_agent/stand_in/__init__.py b/src/agent_env/a2a_agent/stand_in/__init__.py index cbf38df1..40f014b9 100644 --- a/src/agent_env/a2a_agent/stand_in/__init__.py +++ b/src/agent_env/a2a_agent/stand_in/__init__.py @@ -14,7 +14,7 @@ import agentenv_protocol from packaging.requirements import Requirement -from agent_env.a2a_agent.a2a_agent import SHARE_STORE_CREDENTIALS, A2AAgent +from agent_env.a2a_agent.a2a_agent import TAKES_CREDENTIALS, A2AAgent from agent_env.artifact.artifacts.docker_image import DockerImageArtifact from agent_env.store.base import NotFoundError @@ -41,11 +41,7 @@ def stand_in_agent() -> A2AAgent: _IMAGE_ID, description="The stand-in agent: a driver outside its sandbox answers its turns.", context_path=str(context), dockerfile_path=dockerfile, platform=None, ) - return A2AAgent.put( - id=STAND_IN_AGENT_ID, docker_image_artifact=image, - default_env_vars={"LITELLM_API_KEY": "unused", "LITELLM_BASE_URL": "http://unused.invalid"}, - metadata={SHARE_STORE_CREDENTIALS: False}, - ) + return A2AAgent.put(id=STAND_IN_AGENT_ID, docker_image_artifact=image, metadata={TAKES_CREDENTIALS: False}) def _stage(context: Path) -> Path: diff --git a/src/agent_env/task/episode/run.py b/src/agent_env/task/episode/run.py index c6e4a464..050df47b 100644 --- a/src/agent_env/task/episode/run.py +++ b/src/agent_env/task/episode/run.py @@ -144,16 +144,22 @@ async def call(self, name: str, arguments: dict[str, Any] | None = None) -> Tool if name not in self._offered_by: raise EpisodeError( f"no env of turn {self._turn.step_id!r} offers tool {name!r}; they offer {sorted(self._offered_by)}") - started = _now() - env_id = self._offered_by[name] - if env_id is None: - result = await self._bash(arguments or {}) - else: - result = await self._sessions[env_id].call_tool(name, arguments) - self._turn._spans.append(_span(name, "execute_tool", started, prompt={"input": arguments or {}}, - completion={"output": result.content})) + started, env_id = _now(), self._offered_by[name] + try: + if env_id is None: + result = await self._bash(arguments or {}) + else: + result = await self._sessions[env_id].call_tool(name, arguments) + except Exception as e: + self._record(name, arguments, started, {"error": f"{type(e).__name__}: {e}"}) + raise + self._record(name, arguments, started, {"output": result.content}) return result + def _record(self, name: str, arguments: dict[str, Any] | None, started: str, completion: dict[str, Any]) -> None: + self._turn._spans.append(_span(name, "execute_tool", started, prompt={"input": arguments or {}}, + completion=completion)) + async def _bash(self, arguments: dict[str, Any]) -> ToolResult: if not isinstance(command := arguments.get("command"), str): raise EpisodeError(f"tool {BASH.name!r} takes a command string, not {command!r}") @@ -181,6 +187,7 @@ def __init__(self, id: str, task: Task, context: TaskStepContext): self._abandoned: str | None = None self._stand_ins: list[StandInClient] = [] self._drains: list[asyncio.Task[None]] = [] + self._played: set[str] = set() self._in_process: list[InProcessStandIn] = [] async def next_turn(self) -> Turn | None: @@ -188,6 +195,7 @@ async def next_turn(self) -> Turn | None: failed first. End the turns you hold before waiting on one that depends on them.""" while True: changed = self._changed + self._close_turns_of_failed_steps() while self._waiting and self._abandoned is None: turn = self._waiting.popleft() if turn._open: @@ -202,6 +210,7 @@ async def result(self) -> dict[str, float]: over: what ``agent-env run`` reports. Making a reward of them is the caller's.""" while not self._run.done(): changed = self._changed + self._close_turns_of_failed_steps() if open_turn := next((turn for turn in self._turns if turn._open), None): raise EpisodeError( f"turn {open_turn.step_id!r} of agent {open_turn.agent!r} isn't ended, so the run can't finish" @@ -220,6 +229,8 @@ async def abandon(self, reason: str) -> None: await turn._fail(EpisodeError(f"episode abandoned: {reason}")) else: self._abandoned = reason + for turn in self._turns: + turn._open = False self._run.cancel(msg=reason) await asyncio.wait((self._run,)) @@ -260,6 +271,13 @@ async def _wait(self, changed: asyncio.Future[None]) -> None: """Until the run ends or what was ``changed`` when the caller last looked changes again.""" await asyncio.wait((self._run, changed), return_when=asyncio.FIRST_COMPLETED) + def _close_turns_of_failed_steps(self) -> None: + """A step that failed has given up on its turns, which its stand-in drops only a long-poll later.""" + failed = {failure.get("step_id") for failure in self._context.metadata.get("failed_steps") or []} + for turn in self._turns: + if turn.step_id in failed: + turn._open = False + def _raise_unless_completed(self) -> None: if self._abandoned is not None: raise EpisodeError(f"episode abandoned: {self._abandoned}") @@ -319,13 +337,15 @@ async def _drain( self._changed = asyncio.get_running_loop().create_future() def _step_of(self, agent: str, context_id: str) -> str: - """The ``prompt_agent`` step whose conversation with ``agent`` a message on ``context_id`` is in.""" + """The ``prompt_agent`` step a message to ``agent`` on ``context_id`` comes from: of its prompts in that + conversation, the first not yet played, since a prompt continuing a conversation runs after the one it + continues.""" conversations = self._context.metadata.get("a2a_conversations", {}) - for step in self._seeded_steps(): - if step.type == "prompt_agent" and step.agent_name == agent and ( - step.context_id or conversations.get(step.id)) == context_id: - return step.id - return context_id + steps = [step.id for step in self._seeded_steps() if step.type == "prompt_agent" and step.agent_name == agent + and (step.context_id or conversations.get(step.id)) == context_id] + step_id = next((step_id for step_id in steps if step_id not in self._played), steps[-1] if steps else context_id) + self._played.add(step_id) + return step_id def _seeded_steps(self) -> list: return substitute_seed_in_steps(self._task.steps, self._context.metadata.get("seed") or {}) diff --git a/tst/integration/env/envs/server_env_local_test.py b/tst/integration/env/envs/server_env_local_test.py index e15f4c1b..f5907ac9 100644 --- a/tst/integration/env/envs/server_env_local_test.py +++ b/tst/integration/env/envs/server_env_local_test.py @@ -307,8 +307,8 @@ def _items_artifact(uid: str) -> EnvironmentArtifact: @pytest.mark.asyncio async def test_an_agent_that_works_in_its_sandbox_is_played_through_a_stand_in_deployed_there(local_stack, tmp_path): """A task that loads files into its agent and checks what the agent left there: the agent is never deployed, - a stand-in is, in its place, and the loop's ``bash`` runs in it, as its user and in its folder, with no model key - of its own; the check scores what the commands left, and nothing is left running.""" + a stand-in is, in its place, and the loop's ``bash`` runs in it, as its user and in its folder, with no model + credentials; the check scores what the commands left, and nothing is left running.""" uid = uuid.uuid4().hex[:8] greeting = FileArtifact.put_bytes(id=f"greeting-{uid}", description="stand-in e2e", filename="greeting.txt", content=b"hello", content_type="text/plain") @@ -329,7 +329,7 @@ async def test_an_agent_that_works_in_its_sandbox_is_played_through_a_stand_in_d assert [tool.name for tool in tools.list()] == ["bash"] assert (await tools.call("bash", {"command": "pwd; whoami; cat inputs/greeting.txt"})).text == ( "/app\nagent\nhello") - assert (await tools.call("bash", {"command": "printenv LITELLM_API_KEY"})).text == "unused\n" + assert (await tools.call("bash", {"command": "env | grep -c ^LITELLM_ || true"})).text == "0\n" await tools.call("bash", {"command": "mkdir -p outputs && cp inputs/greeting.txt outputs/answer.txt"}) await turn.end("Copied.") diff --git a/tst/unit/a2a_agent/stand_in_test.py b/tst/unit/a2a_agent/stand_in_test.py index 11eaebb8..41d7c3ca 100644 --- a/tst/unit/a2a_agent/stand_in_test.py +++ b/tst/unit/a2a_agent/stand_in_test.py @@ -4,17 +4,29 @@ from importlib.metadata import version from agent_env.a2a_agent import stand_in -from agent_env.a2a_agent.a2a_agent import SHARE_STORE_CREDENTIALS +from agent_env.a2a_agent.a2a_agent import TAKES_CREDENTIALS, A2AAgent from agent_env.a2a_agent.stand_in import STAND_IN_AGENT_ID, stand_in_agent -def test_the_stand_in_is_a_build_context_given_no_model_key_or_store_credentials(local_stores, cli_routing): +def test_the_stand_in_is_a_build_context_that_takes_no_credentials(local_stores, cli_routing): agent = stand_in_agent() assert agent.id == STAND_IN_AGENT_ID assert agent.docker_image_artifact.context_only - assert agent.default_env_vars == {"LITELLM_API_KEY": "unused", "LITELLM_BASE_URL": "http://unused.invalid"} - assert agent.metadata == {SHARE_STORE_CREDENTIALS: False} + assert (agent.default_env_vars, agent.metadata) == ({}, {TAKES_CREDENTIALS: False}) + + +class _NoModelConfig: + def __getattr__(self, name): + raise AssertionError(f"an agent that takes no credentials asked config for {name}") + + +def test_an_agent_that_takes_no_credentials_gets_no_model_endpoint_key_or_headers_whoever_passes_them(): + agent = A2AAgent(id="a", version=1, docker_image_artifact=None, metadata={TAKES_CREDENTIALS: False}) + passed = {"LITELLM_API_KEY": "sk-from-the-step", "LITELLM_BASE_URL": "http://model", "LITELLM_EXTRA_HEADERS": "{}", + "GITHUB_TOKEN": "the task's own"} + + assert agent._with_model_credentials(passed, _NoModelConfig()) == {"GITHUB_TOKEN": "the task's own"} def test_an_unchanged_context_reuses_the_stand_in_and_a_changed_one_writes_it_again( diff --git a/tst/unit/store/aws/test_shared_credentials.py b/tst/unit/store/aws/test_shared_credentials.py index 65efc460..2b50bcab 100644 --- a/tst/unit/store/aws/test_shared_credentials.py +++ b/tst/unit/store/aws/test_shared_credentials.py @@ -5,7 +5,7 @@ import pytest from agentenv_protocol import client as protocol_v1 -from agent_env.a2a_agent.a2a_agent import SHARE_STORE_CREDENTIALS, A2AAgent +from agent_env.a2a_agent.a2a_agent import TAKES_CREDENTIALS, A2AAgent from agent_env.config import set_object_store from agent_env.store import routing from agent_env.store.object_store import s3_object_store @@ -85,4 +85,4 @@ def test_a_caller_supplied_key_suppresses_the_whole_shared_set(host_credentials) def test_an_agent_that_opts_out_gets_none_of_the_shared_set(host_credentials): set_object_store(S3ObjectStore(object(), "bucket", region="us-west-2", share_credentials=True)) - assert not any(key.startswith("AWS_") for key in _merged_env(metadata={SHARE_STORE_CREDENTIALS: False})) + assert not any(key.startswith("AWS_") for key in _merged_env(metadata={TAKES_CREDENTIALS: False})) diff --git a/tst/unit/task/episode/run_test.py b/tst/unit/task/episode/run_test.py index 35035f81..ded6a205 100644 --- a/tst/unit/task/episode/run_test.py +++ b/tst/unit/task/episode/run_test.py @@ -110,8 +110,9 @@ class _ToolSessions: """Stands in for ``ToolSession``: the session on each URL serves the tools ``offered`` names for it, and every session opened, call made and session closed is recorded.""" - def __init__(self, offered): + def __init__(self, offered, raises=None): self.offered = offered + self.raises = raises or {} self.opened = [] self.calls = [] self.closed = [] @@ -129,6 +130,8 @@ async def list_tools(self): async def call_tool(self, name, arguments=None): sessions.calls.append((mcp_url, name, arguments)) + if name in sessions.raises: + raise sessions.raises[name] return ToolResult(content=[{"type": "text", "text": f"{name} on {mcp_url}"}]) async def close(self): @@ -216,8 +219,8 @@ async def test_a_turn_carries_the_headers_each_url_needs(teardowns, monkeypatch) STOCKED = {**ALICE, "env_ids": ["items", "pantry"]} -def _tool_sessions(monkeypatch, offered): - sessions = _ToolSessions(offered) +def _tool_sessions(monkeypatch, offered, raises=None): + sessions = _ToolSessions(offered, raises) monkeypatch.setattr(episode_run, "ToolSession", sessions) monkeypatch.setattr(episode_run, "sandbox_request_headers_for_url", lambda url: {"X-Sandbox": url}) return sessions @@ -572,9 +575,10 @@ async def test_cancelling_the_caller_of_abandon_still_abandons_the_run(teardowns abandoning.cancel() with pytest.raises(asyncio.CancelledError): await abandoning - await _until(lambda: Task.get_instance(episode.id).status == "cancelled") + with pytest.raises(EpisodeError, match="episode abandoned: stop"): + await episode.result() - assert (Task.get_instance(episode.id).status,Task.get_instance(episode.id).error) == ("cancelled", "stop") + assert (Task.get_instance(episode.id).status, Task.get_instance(episode.id).error) == ("cancelled", "stop") assert teardowns.finished == [episode.id] @@ -660,6 +664,40 @@ async def test_a_turns_calls_and_its_reply_are_handed_back_as_its_trajectory(mon assert all(span["start_time"] <= span["end_time"] for span in spans) +@pytest.mark.asyncio +async def test_a_call_that_raises_is_handed_back_in_the_trajectory_too(monkeypatch): + _tool_sessions(monkeypatch, {ITEMS_MCP: ["items_add_item"]}, + raises={"items_add_item": TimeoutError("the env didn't answer")}) + stand_in = _SandboxedStandIn() + turn = _sandboxed_turn(stand_in) + + async with turn.tools() as tools: + with pytest.raises(TimeoutError, match="the env didn't answer"): + await tools.call("items_add_item", {"name": "crate"}) + await turn.end("Gave up.") + + [(_, reply)] = stand_in.replies + assert reply["tool_call_count"] == 1 + assert json.loads(reply["trajectory"][0]["attributes"]["gen_ai.completion"]) == { + "error": "TimeoutError: the env didn't answer"} + + +@pytest.mark.asyncio +async def test_a_prompt_continuing_a_conversation_gets_its_own_turn(teardowns): + first = {**TURN, "context_id": "chat"} + again = {**TURN, "id": "again", "prompt_id": "again", "prompt": "And another.", "depends_on": ["turn"], + "context_id": "chat"} + episode = await _spec(ITEMS, ALICE, first, again, REPLIES).start() + + turn = await episode.next_turn() + assert (turn.step_id, turn.prompt) == ("turn", "Add a crate.") + await turn.end("Added.") + turn = await episode.next_turn() + assert (turn.step_id, turn.prompt) == ("again", "And another.") + await turn.end("Added another.") + assert await episode.result() == {"replies": 1.0} + + @pytest.mark.asyncio async def test_the_turns_trajectory_is_stored_as_any_agents_is(teardowns, monkeypatch): _tool_sessions(monkeypatch, {ITEMS_MCP: ["items_add_item"]}) From ebb3a4442c4c32e43a85a933cf88e829e40dfb2e Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 20 Oct 2026 15:48:11 -0700 Subject: [PATCH 7/9] test(stand-in): count a killed child as stopped once it's a zombie A killed command's background child outlives its parent as a zombie until whatever adopted it reaps it, and signal 0 still reaches a zombie, so checking for its absence right after the timeout raced on Linux. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_a2a_stand_in.py | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/packages/agentenv-protocol/tests/test_a2a_stand_in.py b/packages/agentenv-protocol/tests/test_a2a_stand_in.py index 50b7189d..4fac062a 100644 --- a/packages/agentenv-protocol/tests/test_a2a_stand_in.py +++ b/packages/agentenv-protocol/tests/test_a2a_stand_in.py @@ -1,6 +1,6 @@ from __future__ import annotations -import os +import subprocess import time from typing import Any @@ -55,6 +55,18 @@ def _ran(client: TestClient, command: str) -> str: return result["output"] +def _stopped(pid: int) -> None: + """Until ``pid`` is gone, or a zombie: a killed child whose parent has gone stays one until what adopted it reaps it.""" + deadline = time.monotonic() + 10 + while (state := _state(pid)) and not state.startswith("Z"): + assert time.monotonic() < deadline, state + time.sleep(0.02) + + +def _state(pid: int) -> str: + return subprocess.run(["ps", "-o", "stat=", "-p", str(pid)], capture_output=True, text=True).stdout.strip() + + @pytest.fixture def client(monkeypatch, tmp_path): monkeypatch.setattr(stand_in, "SKILLS_DIR", tmp_path / "skills") @@ -157,8 +169,7 @@ def test_a_command_that_outlives_its_timeout_is_stopped_with_what_it_started(cli output = _ran(client, "sleep 60 & echo $! > child; wait") assert output == "error: the command timed out after 0.5s" - with pytest.raises(ProcessLookupError): - os.kill(int((tmp_path / "child").read_text()), 0) + _stopped(int((tmp_path / "child").read_text())) def test_a_child_left_holding_the_output_is_stopped_when_the_command_times_out(client, monkeypatch, tmp_path): @@ -167,8 +178,7 @@ def test_a_child_left_holding_the_output_is_stopped_when_the_command_times_out(c output = _ran(client, "sleep 60 & echo $! > child") assert output == "error: the command timed out after 0.5s" - with pytest.raises(ProcessLookupError): - os.kill(int((tmp_path / "child").read_text()), 0) + _stopped(int((tmp_path / "child").read_text())) def test_output_names_a_command_it_started(client): From 1196dd13ec4a6663ec001275f97adeb10028a065 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 21 Oct 2026 09:20:50 -0700 Subject: [PATCH 8/9] feat(protocol): StandInAgent, an agent with no model that a driver plays through driver/v1 urn:agentenv:driver/v1 hands an agent's turns to a driver outside its sandbox: claim sets the token, next long-polls for the open turns, reply ends one with its text or error and trajectory, and exec and output run a command where the agent works. StandInAgent serves it: each message is an open turn until the driver replies, a cancelled one drops out, skills go to ~/skills//, and commands run as bash -c in its folder, as its user, with a 3600 s timeout that kills their process group. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 25 ++ .../agentenv_protocol/a2a_agent/__init__.py | 22 ++ .../agentenv_protocol/a2a_agent/extensions.py | 88 +++++++ .../agentenv_protocol/a2a_agent/stand_in.py | 215 ++++++++++++++++++ .../tests/test_a2a_stand_in.py | 205 +++++++++++++++++ 5 files changed, 555 insertions(+) create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/stand_in.py create mode 100644 packages/agentenv-protocol/tests/test_a2a_stand_in.py diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index cc0b804b..366341a7 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -697,6 +697,31 @@ The answer is agent-wide rather than per caller, and like every extension route probe carries no access check of its own, so expose an agent only to the control plane that drives it. +`DRIVER_V1` hands an agent's turns to a driver outside its sandbox, and `StandInAgent` +(`agentenv_protocol.a2a_agent.stand_in`) serves it. The stand-in has no model: each message +it gets stays an open turn, its task working, until the driver replies. The driver's first +`POST /ext/driver/claim` (`{"token"}`, at least 22 characters) sets the token every later +call carries in its body. A claim with another token gets 409, and any other call with a +wrong one gets 401. The token is held in memory only, never in the environment a command +inherits. The other operations: + +- `POST /ext/driver/next` (`{"token", "after"}`) waits up to 25 s for the turns' version to + pass `after`, then returns `{"version", "turns": [{"task_id", "context_id", "text", + "system_prompt", "skills"}]}`. A turn its sender cancels drops out of the list. +- `POST /ext/driver/reply` (`{"token", "task_id", "text"}` or `"error"` in place of + `"text"`, plus `"trajectory"` and `"tool_call_count"`) ends a turn. A text is the task's + result, an error fails it with `stand_in.driver_error`, and the trajectory's spans become + the task's trajectory. +- `POST /ext/driver/exec` (`{"token", "command"}`) starts `bash -c` in the stand-in's working + folder, as its user, and returns `{"exec_id"}`. `POST /ext/driver/output` (`{"token", + "exec_id"}`) waits up to 25 s and returns `{"done", "output"}`. The output is stdout then + stderr, or an error line after 3600 s, when the command's process group is killed. + +Skills the stand-in is given are written to `~/skills//`, and a turn names each one's +folder. `StandInAgent(shell=False)` stands in for an agent that works in no sandbox: it keeps +no skills and refuses `exec`. Commands inherit the stand-in's environment, so run it with no +credentials in it. + Runnable, self-contained reference agents live in [`examples/`](https://github.com/scaleapi/agentenv-framework/tree/main/packages/agentenv-protocol/examples): the normal, streaming, and multimodal `run(request)` paths, single- and multi-operation custom extensions, and advanced ASGI-lifespan plus diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py index 93141797..605fe39d 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py @@ -5,6 +5,7 @@ from .extensions import ( AGENT_CONFIG_V1, ATTRIBUTION_PROBE_V1, + DRIVER_V1, INSTALL_V1, MCP_CONFIG_V1, PEER_AGENTS_V1, @@ -19,6 +20,16 @@ ContextEventsTrajectoryRequest, ContextObjectTrajectoryRequest, ContextTrajectoryRequest, + DriverClaimRequest, + DriverExecRequest, + DriverExecResponse, + DriverNextRequest, + DriverNextResponse, + DriverOutputRequest, + DriverOutputResponse, + DriverReplyRequest, + DriverSkill, + DriverTurn, ExtensionActivation, ExtensionConfiguration, ExtensionDefinition, @@ -117,6 +128,7 @@ __all__ = [ "AGENT_CONFIG_V1", "ATTRIBUTION_PROBE_V1", + "DRIVER_V1", "INSTALL_V1", "MCP_CONFIG_V1", "PEER_AGENTS_V1", @@ -139,6 +151,16 @@ "ContextEventsTrajectoryRequest", "ContextObjectTrajectoryRequest", "ContextTrajectoryRequest", + "DriverClaimRequest", + "DriverExecRequest", + "DriverExecResponse", + "DriverNextRequest", + "DriverNextResponse", + "DriverOutputRequest", + "DriverOutputResponse", + "DriverReplyRequest", + "DriverSkill", + "DriverTurn", "DataPart", "DefaultExtensionHandlers", "ExtensionActivation", diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py index a35c19bb..82df2bd9 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py @@ -287,6 +287,71 @@ class TriggerDecideRequest(ExtensionRequest): env_triggers: dict[str, dict[str, Any]] | None = None +DRIVER_TOKEN_MIN_LENGTH = 22 + + +class DriverClaimRequest(ExtensionRequest): + token: str = Field(min_length=DRIVER_TOKEN_MIN_LENGTH) + + +class DriverNextRequest(ExtensionRequest): + token: str + after: int = Field(default=0, ge=0) + + +class DriverSkill(ExtensionResponse): + name: str + description: str + path: str | None = None + + +class DriverTurn(ExtensionResponse): + task_id: str + context_id: str + text: str + system_prompt: str | None = None + skills: list[DriverSkill] = Field(default_factory=list) + + +class DriverNextResponse(ExtensionResponse): + version: int + turns: list[DriverTurn] + + +class DriverReplyRequest(ExtensionRequest): + token: str + task_id: str + text: str | None = None + error: str | None = None + trajectory: list[dict[str, Any]] = Field(default_factory=list) + tool_call_count: int = Field(default=0, ge=0) + + @model_validator(mode="after") + def _text_or_error(self) -> DriverReplyRequest: + if (self.text is None) == (self.error is None): + raise ValueError("a reply carries exactly one of text and error") + return self + + +class DriverExecRequest(ExtensionRequest): + token: str + command: str + + +class DriverExecResponse(ExtensionResponse): + exec_id: str + + +class DriverOutputRequest(ExtensionRequest): + token: str + exec_id: str + + +class DriverOutputResponse(ExtensionResponse): + done: bool + output: str | None = None + + def _tuple(values: Iterable[str]) -> tuple[str, ...]: return tuple(dict.fromkeys(values)) @@ -1167,6 +1232,28 @@ def _configure_trajectory(*, live: bool = False) -> ExtensionConfiguration: configuration_validator=_configure_install, ) +DRIVER_V1 = ExtensionDefinition( + uri="urn:agentenv:driver/v1", + description="Hand the agent's turns to a driver outside its sandbox, and run its commands where the agent works.", + endpoint="/ext/driver", + core_operations={ + "claim": _op("claim", "POST", "/ext/driver/claim", RUNTIME, request_model=DriverClaimRequest), + "next": _op( + "next", "POST", "/ext/driver/next", RUNTIME, + request_model=DriverNextRequest, response_model=DriverNextResponse, + ), + "reply": _op("reply", "POST", "/ext/driver/reply", RUNTIME, request_model=DriverReplyRequest), + "exec": _op( + "exec", "POST", "/ext/driver/exec", RUNTIME, + request_model=DriverExecRequest, response_model=DriverExecResponse, + ), + "output": _op( + "output", "POST", "/ext/driver/output", RUNTIME, + request_model=DriverOutputRequest, response_model=DriverOutputResponse, + ), + }, +) + STANDARD_EXTENSIONS: Mapping[str, ExtensionDefinition] = MappingProxyType( { extension.uri: extension @@ -1180,6 +1267,7 @@ def _configure_trajectory(*, live: bool = False) -> ExtensionConfiguration: ATTRIBUTION_PROBE_V1, TRIGGERS_V1, INSTALL_V1, + DRIVER_V1, ) } ) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/stand_in.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/stand_in.py new file mode 100644 index 00000000..00e915be --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/stand_in.py @@ -0,0 +1,215 @@ +"""The stand-in: an agent with no model, played by a driver outside its sandbox. + +Each message it gets is an open turn until the driver replies to it. Through ``urn:agentenv:driver/v1`` the driver +reads the open turns, replies to them and runs commands where the stand-in works, as it; its first ``claim`` sets the +token every later call carries. A turn its sender cancels closes without a reply. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import hmac +import os +import signal +import uuid +from dataclasses import dataclass, field +from pathlib import Path + +from starlette.exceptions import HTTPException + +from .extensions import ( + DRIVER_V1, + MCP_CONFIG_V1, + SKILL_CONFIG_V1, + TRAJECTORY_V1, + BundleSkillRequest, + DriverClaimRequest, + DriverExecRequest, + DriverExecResponse, + DriverNextRequest, + DriverNextResponse, + DriverOutputRequest, + DriverOutputResponse, + DriverReplyRequest, + DriverSkill, + DriverTurn, + InlineSkillRequest, + extension, +) +from .framework import AgentEnvAgent, AgentIdentity, a2a_agent +from .tasks.v1 import AgentConfig, TaskRequest, TaskResult, TextPart, Usage +from ..transfers import download + +LONG_POLL_SECONDS = 25.0 +COMMAND_TIMEOUT_SECONDS = 3600 +SKILLS_DIR = Path.home() / "skills" +TRAJECTORY_FORMAT = "agentenv-stand-in/v1" + + +class StandInConfig(AgentConfig): + system_prompt: str | None = None + + +@dataclass +class _Turn: + task_id: str + context_id: str + text: str + system_prompt: str | None + skills: list[DriverSkill] + reply: asyncio.Future[DriverReplyRequest] = field( + default_factory=lambda: asyncio.get_running_loop().create_future() + ) + + +IDENTITY = AgentIdentity( + name="agentenv-stand-in", + description="Stands in for an agent played from outside AgentEnv: a driver answers its turns.", + version="1.0.0", +) +EXTENSIONS = (MCP_CONFIG_V1, TRAJECTORY_V1, DRIVER_V1) + + +@a2a_agent(identity=IDENTITY, config=StandInConfig, extensions=EXTENSIONS) +class StandInAgent(AgentEnvAgent): + """``shell=False`` is a stand-in for an agent that works in no sandbox: it keeps no skills and runs no commands. + ``long_poll_seconds`` bounds how long ``next`` and ``output`` wait for something to report.""" + + def __init__(self, *, shell: bool = True, long_poll_seconds: float = LONG_POLL_SECONDS): + self._shell = shell + self._long_poll_seconds = long_poll_seconds + self._token: str | None = None + self._turns: dict[str, _Turn] = {} + self._version = 0 + self._changed: asyncio.Future[None] | None = None + self._commands: dict[str, asyncio.Task[str]] = {} + + async def run(self, request: TaskRequest[StandInConfig]) -> TaskResult: + turn = _Turn( + task_id=request.task_id, + context_id=request.context_id, + text="\n".join(part.text for part in request.parts if isinstance(part, TextPart)), + system_prompt=request.config.system_prompt, + skills=[DriverSkill(name=skill["name"], description=skill.get("description") or "", + path=str(SKILLS_DIR / skill["name"]) if self._shell else None) + for skill in request.skills], + ) + self._turns[turn.task_id] = turn + self._bump() + try: + reply = await turn.reply + finally: + del self._turns[turn.task_id] + self._bump() + request.trajectory.set_format(TRAJECTORY_FORMAT) + for span in reply.trajectory: + request.trajectory.append(span) + if reply.error is not None: + return TaskResult.failure("stand_in.driver_error", reply.error) + return TaskResult.text(reply.text, usage=Usage(tool_call_count=reply.tool_call_count)) + + @extension(DRIVER_V1.claim) + async def claim(self, request: DriverClaimRequest) -> None: + if self._token is None: + self._token = request.token + elif not hmac.compare_digest(self._token, request.token): + raise HTTPException(status_code=409, detail="this stand-in is already claimed") + + @extension(DRIVER_V1.next) + async def next_turns(self, request: DriverNextRequest) -> DriverNextResponse: + self._check(request.token) + if self._version <= request.after: + with contextlib.suppress(asyncio.TimeoutError): + await asyncio.wait_for(asyncio.shield(self._next_change()), self._long_poll_seconds) + return DriverNextResponse(version=self._version, turns=[ + DriverTurn(task_id=turn.task_id, context_id=turn.context_id, text=turn.text, + system_prompt=turn.system_prompt, skills=turn.skills) + for turn in self._turns.values() if not turn.reply.done() + ]) + + @extension(DRIVER_V1.reply) + async def reply(self, request: DriverReplyRequest) -> None: + self._check(request.token) + turn = self._turns.get(request.task_id) + if turn is None or turn.reply.done(): + raise HTTPException(status_code=404, detail=f"no open turn {request.task_id!r}") + turn.reply.set_result(request) + + @extension(DRIVER_V1.exec) + async def exec_command(self, request: DriverExecRequest) -> DriverExecResponse: + self._check(request.token) + if not self._shell: + raise HTTPException(status_code=409, detail="this stand-in works in no sandbox, so it runs no commands") + command_id = uuid.uuid4().hex + self._commands[command_id] = asyncio.create_task(_run(request.command)) + return DriverExecResponse(exec_id=command_id) + + @extension(DRIVER_V1.output) + async def output(self, request: DriverOutputRequest) -> DriverOutputResponse: + self._check(request.token) + if (command := self._commands.get(request.exec_id)) is None: + raise HTTPException(status_code=404, detail=f"no command {request.exec_id!r}") + await asyncio.wait((command,), timeout=self._long_poll_seconds) + if not command.done(): + return DriverOutputResponse(done=False) + self._commands.pop(request.exec_id, None) + return DriverOutputResponse(done=True, output=command.result()) + + @extension(SKILL_CONFIG_V1.add.inline) + async def add_inline_skill(self, request: InlineSkillRequest) -> dict[str, str]: + folder = self._skill_folder(request.name) + folder.mkdir(parents=True) + (folder / "SKILL.md").write_text(request.skill_md) + return {"name": request.name} + + @extension(SKILL_CONFIG_V1.add.bundle) + async def add_bundle_skill(self, request: BundleSkillRequest) -> dict[str, str]: + folder = self._skill_folder(request.name) + for item in request.skill_bundle.files: + await download(item.object, folder / item.path) + return {"name": request.name} + + def _check(self, token: str) -> None: + if self._token is None or not hmac.compare_digest(self._token, token): + raise HTTPException(status_code=401, detail="the driver's token is missing or wrong") + + def _skill_folder(self, name: str) -> Path: + if not self._shell: + raise HTTPException(status_code=409, detail="this stand-in works in no sandbox, so it keeps no skills") + return SKILLS_DIR / name + + def _next_change(self) -> asyncio.Future[None]: + if self._changed is None: + self._changed = asyncio.get_running_loop().create_future() + return self._changed + + def _bump(self) -> None: + self._version += 1 + if self._changed is not None: + self._changed.set_result(None) + self._changed = None + + +async def _run(command: str) -> str: + """``command``'s stdout then its stderr, as verifiers' bash tool returns them. It runs in a process group of its + own, stopped with whatever it started if its output isn't done within ``COMMAND_TIMEOUT_SECONDS``, or if the call + is cancelled, even when ``bash`` has exited and a child it left holds the output open.""" + process = await asyncio.create_subprocess_exec( + "bash", "-c", command, stdin=asyncio.subprocess.DEVNULL, stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, start_new_session=True, + ) + try: + stdout, stderr = await asyncio.wait_for(process.communicate(), COMMAND_TIMEOUT_SECONDS) + except BaseException as e: + with contextlib.suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGKILL) + await process.wait() + if isinstance(e, asyncio.TimeoutError): + return f"error: the command timed out after {COMMAND_TIMEOUT_SECONDS}s" + raise + return (stdout + stderr).decode(errors="replace") + + +if __name__ == "__main__": + StandInAgent().serve() diff --git a/packages/agentenv-protocol/tests/test_a2a_stand_in.py b/packages/agentenv-protocol/tests/test_a2a_stand_in.py new file mode 100644 index 00000000..4fac062a --- /dev/null +++ b/packages/agentenv-protocol/tests/test_a2a_stand_in.py @@ -0,0 +1,205 @@ +from __future__ import annotations + +import subprocess +import time +from typing import Any + +import pytest +from starlette.testclient import TestClient + +from agentenv_protocol.a2a_agent import stand_in +from agentenv_protocol.a2a_agent.stand_in import StandInAgent + +TOKEN = "t" * 32 + + +def _send(client: TestClient, text: str, *, context_id: str = "context-1") -> str: + response = client.post("/a2a", json={ + "jsonrpc": "2.0", "id": "1", "method": "message/send", + "params": { + "message": {"kind": "message", "messageId": f"message-{text}", "role": "user", "contextId": context_id, + "parts": [{"kind": "text", "text": text}]}, + "configuration": {"blocking": False}, + }, + }) + return response.json()["result"]["id"] + + +def _task(client: TestClient, task_id: str) -> dict[str, Any]: + return client.post("/a2a", json={"jsonrpc": "2.0", "id": "1", "method": "tasks/get", + "params": {"id": task_id}}).json()["result"] + + +def _settled(client: TestClient, task_id: str) -> dict[str, Any]: + deadline = time.monotonic() + 10 + while (task := _task(client, task_id))["status"]["state"] not in ("completed", "failed", "canceled"): + assert time.monotonic() < deadline, task + time.sleep(0.02) + return task + + +def _driver(client: TestClient, operation: str, **body: Any): + return client.post(f"/ext/driver/{operation}", json={"token": TOKEN, **body}) + + +def _turns(client: TestClient, after: int = 0) -> dict[str, Any]: + response = _driver(client, "next", after=after) + assert response.status_code == 200, response.text + return response.json() + + +def _ran(client: TestClient, command: str) -> str: + exec_id = _driver(client, "exec", command=command).json()["exec_id"] + while not (result := _driver(client, "output", exec_id=exec_id).json())["done"]: + pass + return result["output"] + + +def _stopped(pid: int) -> None: + """Until ``pid`` is gone, or a zombie: a killed child whose parent has gone stays one until what adopted it reaps it.""" + deadline = time.monotonic() + 10 + while (state := _state(pid)) and not state.startswith("Z"): + assert time.monotonic() < deadline, state + time.sleep(0.02) + + +def _state(pid: int) -> str: + return subprocess.run(["ps", "-o", "stat=", "-p", str(pid)], capture_output=True, text=True).stdout.strip() + + +@pytest.fixture +def client(monkeypatch, tmp_path): + monkeypatch.setattr(stand_in, "SKILLS_DIR", tmp_path / "skills") + monkeypatch.chdir(tmp_path) + with TestClient(StandInAgent(long_poll_seconds=0.2).create_app()) as client: + assert client.post("/ext/driver/claim", json={"token": TOKEN}).status_code == 200 + yield client + + +def test_the_card_advertises_the_driver_and_takes_a_system_prompt(): + card = StandInAgent().create_app().state.agentenv_a2a.card.model_dump(mode="json") + extensions = {extension["uri"]: extension for extension in card["capabilities"]["extensions"]} + + assert sorted(extensions["urn:agentenv:driver/v1"]["params"]["methods"]) == [ + "claim", "exec", "next", "output", "reply"] + assert "system_prompt" in extensions["urn:agentenv:agent-config/v1"]["params"]["methods"]["set"]["request"][ + "supported"] + + +def test_the_first_claim_sets_the_token_that_every_driver_call_needs(): + with TestClient(StandInAgent(long_poll_seconds=0.2).create_app()) as client: + assert _driver(client, "next").status_code == 401 + assert client.post("/ext/driver/claim", json={"token": "short"}).status_code == 400 + assert client.post("/ext/driver/claim", json={"token": TOKEN}).status_code == 200 + assert client.post("/ext/driver/claim", json={"token": TOKEN}).status_code == 200 + assert client.post("/ext/driver/claim", json={"token": "u" * 32}).status_code == 409 + assert client.post("/ext/driver/next", json={"token": "u" * 32}).status_code == 401 + assert _driver(client, "next").status_code == 200 + + +def test_a_message_is_an_open_turn_until_the_driver_replies(client): + client.post("/ext/agent-config", json={"system_prompt": "be brief"}) + task_id = _send(client, "fix the build") + + seen = _turns(client) + assert seen["turns"] == [{"task_id": task_id, "context_id": "context-1", "text": "fix the build", + "system_prompt": "be brief", "skills": []}] + span = {"name": "bash", "attributes": {"gen_ai.operation.name": "execute_tool"}} + assert _driver(client, "reply", task_id=task_id, text="fixed", trajectory=[span], tool_call_count=1).status_code \ + == 200 + + task = _settled(client, task_id) + assert task["status"]["state"] == "completed" + parts = task["status"]["message"]["parts"] + assert parts[0] == {"kind": "text", "text": "fixed"} + assert parts[1]["data"]["usage"]["tool_call_count"] == 1 + assert client.post("/ext/trajectory", json={"task_id": task_id}).json()["trajectory"] == [span] + assert _turns(client, after=seen["version"])["turns"] == [] + + +def test_an_error_reply_fails_the_turn(client): + task_id = _send(client, "do it") + + _driver(client, "reply", task_id=task_id, error="the driver gave up") + + task = _settled(client, task_id) + assert task["status"]["state"] == "failed" + assert "the driver gave up" in task["status"]["message"]["parts"][0]["text"] + + +def test_a_reply_names_exactly_one_open_turn(client): + task_id = _send(client, "do it") + + assert _driver(client, "reply", task_id=task_id).status_code == 400 + assert _driver(client, "reply", task_id="nope", text="done").status_code == 404 + _driver(client, "reply", task_id=task_id, text="done") + _settled(client, task_id) + assert _driver(client, "reply", task_id=task_id, text="again").status_code == 404 + + +def test_a_turn_its_sender_cancels_is_withdrawn(client): + task_id = _send(client, "do it") + assert [turn["task_id"] for turn in _turns(client)["turns"]] == [task_id] + + client.post("/a2a", json={"jsonrpc": "2.0", "id": "1", "method": "tasks/cancel", "params": {"id": task_id}}) + + assert _settled(client, task_id)["status"]["state"] == "canceled" + assert _turns(client)["turns"] == [] + + +def test_next_waits_for_a_change_after_the_version_it_was_given(client): + version = _turns(client)["version"] + + started = time.monotonic() + unchanged = _turns(client, after=version) + + assert unchanged["version"] == version + assert time.monotonic() - started >= 0.2 + + +def test_exec_returns_stdout_then_stderr_from_the_folder_it_works_in(client, tmp_path): + (tmp_path / "inputs").mkdir() + + assert _ran(client, "ls; echo out; echo err >&2; exit 3") == "inputs\nout\nerr\n" + + +def test_a_command_that_outlives_its_timeout_is_stopped_with_what_it_started(client, monkeypatch, tmp_path): + monkeypatch.setattr(stand_in, "COMMAND_TIMEOUT_SECONDS", 0.5) + + output = _ran(client, "sleep 60 & echo $! > child; wait") + + assert output == "error: the command timed out after 0.5s" + _stopped(int((tmp_path / "child").read_text())) + + +def test_a_child_left_holding_the_output_is_stopped_when_the_command_times_out(client, monkeypatch, tmp_path): + monkeypatch.setattr(stand_in, "COMMAND_TIMEOUT_SECONDS", 0.5) + + output = _ran(client, "sleep 60 & echo $! > child") + + assert output == "error: the command timed out after 0.5s" + _stopped(int((tmp_path / "child").read_text())) + + +def test_output_names_a_command_it_started(client): + assert _driver(client, "output", exec_id="nope").status_code == 404 + + +def test_a_skill_is_written_where_its_turns_say_it_is(client, tmp_path): + response = client.post("/ext/skill-config", json={"name": "deploy", "description": "How to deploy.", + "skill_md": "# Deploy\n"}) + assert response.status_code == 200, response.text + _send(client, "go") + + skill = _turns(client)["turns"][0]["skills"] + assert skill == [{"name": "deploy", "description": "How to deploy.", "path": str(tmp_path / "skills" / "deploy")}] + assert (tmp_path / "skills" / "deploy" / "SKILL.md").read_text() == "# Deploy\n" + + +def test_a_stand_in_without_a_shell_keeps_no_skills_and_runs_no_commands(): + with TestClient(StandInAgent(shell=False, long_poll_seconds=0.2).create_app()) as client: + client.post("/ext/driver/claim", json={"token": TOKEN}) + + assert _driver(client, "exec", command="true").status_code == 409 + assert client.post("/ext/skill-config", json={"name": "x", "description": "d", "skill_md": "#"}).status_code \ + == 409 From 24c6f76938f64d4392f35dd90f95f0dd3ea719b2 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 21 Oct 2026 09:32:54 -0700 Subject: [PATCH 9/9] refactor(agents): the stand-in is an agent of the agents bundle, written by the episode as a run would The stand-in moves from a2a_agent/stand_in/, which staged the installed protocol package into its own build context, to agents/stand-in/, an agent folder like claude-code's and pi's: a Dockerfile, an a2a_server.py declaring its install/v1, an install.sh for a VM's host, the protocol from PyPI in requirements.txt, which a release pins, and an agent.toml setting takes_credentials = false, which [metadata] now takes. Its id is @local/agentenv-framework/agents/stand-in. An episode writes it with write_installed, which plans one installed bundle's entity with what it references (plan_entities, the checks plan_bundle makes, with nothing run) and materializes it, so the ledger reuses it until its folder changes. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 3 +- src/agent_env/a2a_agent/a2a_agent.py | 25 +++++--- src/agent_env/a2a_agent/stand_in/Dockerfile | 13 ---- src/agent_env/a2a_agent/stand_in/__init__.py | 63 ------------------- .../a2a_agent/stand_in/a2a_server.py | 37 ----------- src/agent_env/agents/README.md | 7 +++ .../agents/agents/stand-in/Dockerfile | 17 +++++ .../agents/agents/stand-in/a2a_server.py | 31 +++++++++ .../agents/agents/stand-in/agent.toml | 3 + .../agents/agents/stand-in/install.sh | 27 ++++++++ .../agents/agents/stand-in/requirements.txt | 3 + src/agent_env/bundle/installed.py | 22 ++++++- src/agent_env/bundle/plan.py | 26 +++++++- src/agent_env/task/__init__.py | 2 +- src/agent_env/task/episode/rules.py | 4 +- src/agent_env/task/episode/steps.py | 7 ++- tst/unit/a2a_agent/stand_in_test.py | 52 --------------- tst/unit/agents/stand_in/__init__.py | 0 tst/unit/agents/stand_in/agent.py | 12 ++++ .../agents/stand_in/test_stand_in_agent.py | 56 +++++++++++++++++ tst/unit/bundle/installed_entities_test.py | 48 +++++++++++++- tst/unit/bundle/materialize_test.py | 14 +++-- tst/unit/bundle/to_toml_round_trip_test.py | 13 ++-- tst/unit/task_step/install_agent_test.py | 2 +- 24 files changed, 294 insertions(+), 193 deletions(-) delete mode 100644 src/agent_env/a2a_agent/stand_in/Dockerfile delete mode 100644 src/agent_env/a2a_agent/stand_in/__init__.py delete mode 100644 src/agent_env/a2a_agent/stand_in/a2a_server.py create mode 100644 src/agent_env/agents/agents/stand-in/Dockerfile create mode 100644 src/agent_env/agents/agents/stand-in/a2a_server.py create mode 100644 src/agent_env/agents/agents/stand-in/agent.toml create mode 100644 src/agent_env/agents/agents/stand-in/install.sh create mode 100644 src/agent_env/agents/agents/stand-in/requirements.txt delete mode 100644 tst/unit/a2a_agent/stand_in_test.py create mode 100644 tst/unit/agents/stand_in/__init__.py create mode 100644 tst/unit/agents/stand_in/agent.py create mode 100644 tst/unit/agents/stand_in/test_stand_in_agent.py diff --git a/AGENTS.md b/AGENTS.md index ba2cf431..6f3f57d4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -204,7 +204,8 @@ orchestrators call it the same way. What callers rely on: `context.heartbeat()`, a no-op without one. The key is in `_REDACTED_KEYS`, so it is never persisted. - An episode (`EpisodeSpec`, `task/episode/`) is one `Task.run()` of a copy of the task, with namespace routing on, in which each of its `trainer_agents` is played through a stand-in agent - (`agentenv_protocol.a2a_agent.stand_in`, written as `@local/agentenv-stand-in` from its build context). + (the `agents` bundle's `stand-in`, `@local/agentenv-framework/agents/stand-in`, which the episode writes + from its folder as a bundle run would). The agent's `deploy_agent` or `install_agent`, wrapped as `episode_agent`, deploys the stand-in in the agent's place when the task uses the agent's sandbox and serves it in this process when it doesn't; the agent's `prompt_agent` steps message it as they would the agent, and the caller plays each diff --git a/src/agent_env/a2a_agent/a2a_agent.py b/src/agent_env/a2a_agent/a2a_agent.py index b65b3410..b5a748cc 100644 --- a/src/agent_env/a2a_agent/a2a_agent.py +++ b/src/agent_env/a2a_agent/a2a_agent.py @@ -120,6 +120,7 @@ class A2AAgent: # What an agent.toml's [metadata] may set: the types each key takes, and how a problem names them. toml_metadata: ClassVar[dict[str, tuple[tuple[type, ...], str]]] = { "default_model": ((str,), "a string"), "min_disk_size_gb": ((int, float), "a number"), + TAKES_CREDENTIALS: ((bool,), "true or false"), } EXT_MCP_CONFIG: ClassVar[str] = "urn:agentenv:mcp-config/v1" @@ -370,10 +371,10 @@ def from_toml(cls, data: dict[str, Any], ctx: AuthoringContext) -> A2AAgent: def to_toml(self, out: ExportContext) -> dict[str, Any]: """The agent.toml of a folder this agent is built from, which ``from_toml`` reads back as it.""" - metadata = {key: value for key, value in self.metadata.items() if key in self.toml_metadata - and isinstance(value, self.toml_metadata[key][0]) and not isinstance(value, bool)} + metadata = {key: value for key, value in self.metadata.items() + if key in self.toml_metadata and self._fits_toml_metadata(key, value)} if dropped := sorted(set(self.metadata) - set(metadata)): - out.left_out("metadata", f"an agent.toml's [metadata] takes only {' and '.join(self.toml_metadata)}, not " + out.left_out("metadata", f"an agent.toml's [metadata] takes only {self._toml_metadata_keys()}, not " f"{', '.join(dropped)}") toml: dict[str, Any] = {"image": out.image(self.docker_image_artifact)} if self.default_env_vars: @@ -390,16 +391,26 @@ def accept_toml(cls, data: dict[str, Any], ctx: AuthoringContext) -> dict[str, A for name, value in fields.get("metadata", {}).items(): if name not in cls.toml_metadata: values.append(f"unknown [metadata] key {name!r}; an agent's [metadata] takes " - f"{' and '.join(cls.toml_metadata)}") + f"{cls._toml_metadata_keys()}") continue - kinds, described = cls.toml_metadata[name] - if isinstance(value, bool) or not isinstance(value, kinds): - values.append(f"metadata.{name} must be {described}, not {value!r}") + if not cls._fits_toml_metadata(name, value): + values.append(f"metadata.{name} must be {cls.toml_metadata[name][1]}, not {value!r}") problems += [ctx.config_problem(problem) for problem in values] if problems: ctx.refuse(problems) return fields + @classmethod + def _toml_metadata_keys(cls) -> str: + *rest, last = cls.toml_metadata + return f"{', '.join(rest)} and {last}" + + @classmethod + def _fits_toml_metadata(cls, name: str, value: Any) -> bool: + """A bool is an int to ``isinstance``, so it fits only a key that takes one.""" + kinds = cls.toml_metadata[name][0] + return isinstance(value, kinds) and (bool in kinds or not isinstance(value, bool)) + @classmethod def get(cls, id: str, version: Optional[int] = None) -> A2AAgent: from .store import get_a2a_agent_store diff --git a/src/agent_env/a2a_agent/stand_in/Dockerfile b/src/agent_env/a2a_agent/stand_in/Dockerfile deleted file mode 100644 index 67edcb2a..00000000 --- a/src/agent_env/a2a_agent/stand_in/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM mirror.gcr.io/library/python:3.12-slim@sha256:423ed6ab25b1921a477529254bfeeabf5855151dc2c3141699a1bfc852199fbf -RUN apt-get update \ - && apt-get install -y --no-install-recommends bash ca-certificates coreutils curl findutils git procps \ - && rm -rf /var/lib/apt/lists/* -COPY requirements.txt /opt/agentenv-stand-in/ -RUN pip install --no-cache-dir -r /opt/agentenv-stand-in/requirements.txt && useradd --create-home agent -COPY agentenv_protocol/ /opt/agentenv-stand-in/agentenv_protocol/ -ENV PYTHONPATH=/opt/agentenv-stand-in -WORKDIR /app -COPY a2a_server.py . -RUN chown agent:agent /app -USER agent -CMD ["python", "/app/a2a_server.py"] diff --git a/src/agent_env/a2a_agent/stand_in/__init__.py b/src/agent_env/a2a_agent/stand_in/__init__.py deleted file mode 100644 index 40f014b9..00000000 --- a/src/agent_env/a2a_agent/stand_in/__init__.py +++ /dev/null @@ -1,63 +0,0 @@ -"""The stand-in agent (``agentenv_protocol.a2a_agent.stand_in``) as the Episode API deploys it, in the place of an agent -a loop plays: ``@local/agentenv-stand-in``, an image registered by its build context alone and built where each deploy -runs. Its context is this folder, the installed protocol package, and that package's requirements for serving an -agent, pinned to the versions installed here.""" - -from __future__ import annotations - -import shutil -import tempfile -import threading -from importlib.metadata import requires, version -from pathlib import Path - -import agentenv_protocol -from packaging.requirements import Requirement - -from agent_env.a2a_agent.a2a_agent import TAKES_CREDENTIALS, A2AAgent -from agent_env.artifact.artifacts.docker_image import DockerImageArtifact -from agent_env.store.base import NotFoundError - -STAND_IN_AGENT_ID = "@local/agentenv-stand-in" -_IMAGE_ID = f"{STAND_IN_AGENT_ID}/image" -_PROTOCOL_DISTRIBUTION = "agentenv-framework-protocol" -_HERE = Path(__file__).parent -_writing = threading.Lock() - - -def stand_in_agent() -> A2AAgent: - """The stand-in, written to the store first when it isn't there or its build context has changed.""" - with _writing, tempfile.TemporaryDirectory() as staged: - context = _stage(Path(staged)) - dockerfile = str(context / "Dockerfile") - digest, _ = DockerImageArtifact.context_identity(_IMAGE_ID, str(context), dockerfile, platform=None) - try: - agent = A2AAgent.get(STAND_IN_AGENT_ID) - if agent.docker_image_artifact.source_digest == digest: - return agent - except NotFoundError: - pass - image = DockerImageArtifact.put_context( - _IMAGE_ID, description="The stand-in agent: a driver outside its sandbox answers its turns.", - context_path=str(context), dockerfile_path=dockerfile, platform=None, - ) - return A2AAgent.put(id=STAND_IN_AGENT_ID, docker_image_artifact=image, metadata={TAKES_CREDENTIALS: False}) - - -def _stage(context: Path) -> Path: - for name in ("Dockerfile", "a2a_server.py"): - shutil.copy2(_HERE / name, context / name) - shutil.copytree(Path(agentenv_protocol.__file__).parent, context / "agentenv_protocol", - ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) - (context / "requirements.txt").write_text("".join(f"{line}\n" for line in _requirements())) - return context - - -def _requirements() -> list[str]: - pinned = [] - for spec in requires(_PROTOCOL_DISTRIBUTION) or []: - requirement = Requirement(spec) - if requirement.marker is None or requirement.marker.evaluate({"extra": "agent"}): - extras = f"[{','.join(sorted(requirement.extras))}]" if requirement.extras else "" - pinned.append(f"{requirement.name}{extras}=={version(requirement.name)}") - return sorted(pinned) diff --git a/src/agent_env/a2a_agent/stand_in/a2a_server.py b/src/agent_env/a2a_agent/stand_in/a2a_server.py deleted file mode 100644 index dadb0b38..00000000 --- a/src/agent_env/a2a_agent/stand_in/a2a_server.py +++ /dev/null @@ -1,37 +0,0 @@ -"""The stand-in agent as AgentEnv deploys it: served from ``/app`` in its image, or installed on a VM host as -``install_agent`` installs it, in the place of the agent it replaces (its ``{workdir}``, as its ``{user}``). -``AGENT_CARD`` is what ``install_agent`` reads; importing this module serves nothing. ``install_agent`` formats each -command with ``str.format``, so a literal brace is doubled.""" - -from agentenv_protocol.a2a_agent import INSTALL_V1, a2a_agent, enable -from agentenv_protocol.a2a_agent.stand_in import EXTENSIONS, IDENTITY, StandInAgent, StandInConfig - -HOME = "/opt/agentenv-stand-in-{agent_name}" -INSTALL_COMMANDS_HOST = [ - "bash -c 'set -e; python3 -c \"import ensurepip, venv\" 2>/dev/null || {{ export DEBIAN_FRONTEND=noninteractive; " - "apt-get update -qq; apt-get install -y -qq --no-install-recommends python3 python3-venv; }}'", - f"rm -rf {HOME} && mkdir -p {HOME} && tar -xzf {{agent_ctx_tar}} -C {HOME}", - f"python3 -m venv {HOME}/venv && {HOME}/venv/bin/pip install -q -r {HOME}/requirements.txt", - "id -u {user} >/dev/null 2>&1 || useradd -m {user}; mkdir -p {workdir} && chown -R {user}: {workdir}", - f"nohup runuser -u {{user}} -- env -C {{workdir}} A2A_PORT={{a2a_port}} PYTHONPATH={HOME} {HOME}/venv/bin/python " - f"{HOME}/a2a_server.py > /tmp/agentenv-stand-in-{{agent_name}}.log 2>&1 < /dev/null &", -] - - -@a2a_agent( - identity=IDENTITY, - config=StandInConfig, - extensions=( - *EXTENSIONS, - enable(INSTALL_V1, install_commands_host=INSTALL_COMMANDS_HOST, - required_params_host=["agent_ctx_tar", "a2a_port", "agent_name", "workdir", "user"], a2a_port=8000), - ), -) -class InstalledStandIn(StandInAgent): - pass - - -AGENT_CARD = InstalledStandIn().create_app().state.agentenv_a2a.card - -if __name__ == "__main__": - InstalledStandIn().serve() diff --git a/src/agent_env/agents/README.md b/src/agent_env/agents/README.md index dec562ce..401345ac 100644 --- a/src/agent_env/agents/README.md +++ b/src/agent_env/agents/README.md @@ -26,5 +26,12 @@ the `[model]` endpoint as an OpenAI-compatible one and takes its model from the `[model] default` (or the `agent` role) set. It declares `install/v1` too, for Debian or Ubuntu targets. Its folder's `README.md` covers the extensions it implements. +`agents/stand-in/` is the Episode API's, not an agent to pick for a task. It's the protocol's `StandInAgent`, which +has no model: an episode deploys it in the place of the agent its caller plays, answers its turns through +`urn:agentenv:driver/v1`, and runs the caller's `bash` in it. Its image works in `/app` as user `agent`. On a `vm` +sandbox's host, `install.sh` installs it where the agent it replaces says it works, with the host's own Python in a +venv. Its `agent.toml` sets `takes_credentials = false`, so it's given no model key or store credentials. It has no +smoke task: with nobody answering its turns, a prompt only waits out its deadline. + `agent-env run agents` runs every agent's smoke task, building each image; `--task claude-code-smoke` runs Claude Code's alone, `--task pi-smoke` pi's. diff --git a/src/agent_env/agents/agents/stand-in/Dockerfile b/src/agent_env/agents/agents/stand-in/Dockerfile new file mode 100644 index 00000000..4e088a62 --- /dev/null +++ b/src/agent_env/agents/agents/stand-in/Dockerfile @@ -0,0 +1,17 @@ +FROM mirror.gcr.io/library/python:3.12-slim@sha256:423ed6ab25b1921a477529254bfeeabf5855151dc2c3141699a1bfc852199fbf +# The tools the steps that act on an agent's sandbox run: collect_artifacts, verify_sandbox, run_code, load_artifact. +RUN apt-get update \ + && apt-get install -y --no-install-recommends bash ca-certificates coreutils curl findutils git procps \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /opt/agent +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt +# install.sh is install/v1's: in the image, so it stays in its build context. +COPY a2a_server.py install.sh ./ +RUN useradd --create-home agent && mkdir -p /app && chown agent:agent /app +ENV A2A_PORT=8000 +USER agent +# The commands the driver runs work where the stand-in does: in /app, as agent. +WORKDIR /app +EXPOSE 8000 +CMD ["python", "/opt/agent/a2a_server.py"] diff --git a/src/agent_env/agents/agents/stand-in/a2a_server.py b/src/agent_env/agents/agents/stand-in/a2a_server.py new file mode 100644 index 00000000..3b7c2970 --- /dev/null +++ b/src/agent_env/agents/agents/stand-in/a2a_server.py @@ -0,0 +1,31 @@ +"""Serves the stand-in agent. ``AGENT_CARD`` is what agent-env's install/v1 reads. ``install_agent`` formats each +command with ``str.format``.""" + +from agentenv_protocol.a2a_agent import INSTALL_V1, a2a_agent, enable +from agentenv_protocol.a2a_agent.stand_in import EXTENSIONS, IDENTITY, StandInAgent, StandInConfig + +_UNPACK = "mkdir -p {work_dir}/agent && tar -xzf {agent_ctx_tar} -C {work_dir}/agent" +# In the place of the agent it replaces: its {workdir}, as its {user}. +_INSTALL_COMMANDS_HOST = [_UNPACK, "sh {work_dir}/agent/install.sh {agent_name} {a2a_port} {workdir} {user}"] + + +@a2a_agent( + identity=IDENTITY, + config=StandInConfig, + extensions=( + *EXTENSIONS, + enable(INSTALL_V1, install_commands_host=_INSTALL_COMMANDS_HOST, + required_params_host=["work_dir", "agent_ctx_tar", "a2a_port", "agent_name", "workdir", "user"], + a2a_port=8000), + ), +) +class InstalledStandIn(StandInAgent): + pass + + +agent = InstalledStandIn() +app = agent.create_app() +AGENT_CARD = app.state.agentenv_a2a.card + +if __name__ == "__main__": + agent.serve() diff --git a/src/agent_env/agents/agents/stand-in/agent.toml b/src/agent_env/agents/agents/stand-in/agent.toml new file mode 100644 index 00000000..fce9bb75 --- /dev/null +++ b/src/agent_env/agents/agents/stand-in/agent.toml @@ -0,0 +1,3 @@ +[metadata] +# The commands a driver runs in its sandbox can read its environment, so it's given no model credentials. +takes_credentials = false diff --git a/src/agent_env/agents/agents/stand-in/install.sh b/src/agent_env/agents/agents/stand-in/install.sh new file mode 100644 index 00000000..7e152dde --- /dev/null +++ b/src/agent_env/agents/agents/stand-in/install.sh @@ -0,0 +1,27 @@ +#!/bin/sh +# Installs the stand-in onto this Linux host from the folder this script is in, and starts it on PORT, as USER in +# WORKDIR, the place of the agent it replaces: +# +# install.sh NAME PORT WORKDIR USER +# +# It runs in a venv of the host's own Python, which apt installs when the host has none. +set -eu +name=$1 port=$2 workdir=$3 user=$4 +here=$(cd "$(dirname "$0")" && pwd) +root=/opt/agentenv-stand-in-$name + +if ! python3 -c "import ensurepip, venv" 2>/dev/null; then + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get install -y -qq --no-install-recommends python3 python3-venv +fi +rm -rf "$root" +mkdir -p "$root" +cp "$here/a2a_server.py" "$here/requirements.txt" "$root/" +python3 -m venv "$root/venv" +"$root/venv/bin/pip" install -q -r "$root/requirements.txt" +id -u "$user" >/dev/null 2>&1 || useradd -m "$user" +mkdir -p "$workdir" +chown -R "$user": "$workdir" +nohup runuser -u "$user" -- env -C "$workdir" A2A_PORT="$port" "$root/venv/bin/python" "$root/a2a_server.py" \ + >"/tmp/agentenv-stand-in-$name.log" 2>&1 =0.1.356 diff --git a/src/agent_env/bundle/installed.py b/src/agent_env/bundle/installed.py index 72268a68..a27a15bd 100644 --- a/src/agent_env/bundle/installed.py +++ b/src/agent_env/bundle/installed.py @@ -24,8 +24,9 @@ from agent_env.store.ids import LOCAL_PREFIX from ._fs import os_reason -from .parse import Bundle, BundleError, parse_bundle -from .plan import check_bundle +from .materialize import materialize +from .parse import Bundle, BundleError, BundleKind, parse_bundle +from .plan import check_bundle, plan_entities from .resolve import resolve_bundle BUNDLES = "agent_env.bundles" @@ -106,6 +107,23 @@ def checked(bundle: InstalledBundle, *, load: bool = True) -> Bundle: return parsed +def write_installed(entity_id: str) -> int: + """Write ``entity_id``, an env, agent or artifact an installed bundle defines, with what it references, as a run of + a bundle naming it does, and return its version: the one already written when nothing it's made from changed. + Needs namespace routing. Raises BundleError.""" + bundles = installed_bundles() + bundle = next((bundle for bundle in bundles if entity_id.startswith(f"{bundle.id_root}/")), None) + if bundle is None: + raise BundleError([f"{entity_id!r} isn't an entity of an installed bundle"]) + if bundle.problem: + raise BundleError([f"{bundle.qualified}: {bundle.problem}"]) + plan = plan_entities(resolve_bundle(parse_bundle(bundle.root, id_root=bundle.id_root)), [entity_id], + installed=lambda: bundles) + [write] = [write for write in plan.writes + if write.id == entity_id and write.kind not in (BundleKind.TASK, BundleKind.EVAL)] + return materialize(plan).version_of(write.kind.store, entity_id) + + def _installed(name: str, plugin: Plugin, ep: EntryPoint, *, repeated: bool) -> InstalledBundle: package, version = identity(ep) diff --git a/src/agent_env/bundle/plan.py b/src/agent_env/bundle/plan.py index 045d3363..2bae5ba6 100644 --- a/src/agent_env/bundle/plan.py +++ b/src/agent_env/bundle/plan.py @@ -117,6 +117,13 @@ def plan_bundle(resolved: ResolvedBundle, *, tasks: Sequence[str] = (), evals: S return _Planner(resolved, installed).plan(tasks, evals) +def plan_entities(resolved: ResolvedBundle, ids: Sequence[str], *, installed: Installed | None = None) -> Plan: + """Plan writing the envs, agents and artifacts ``ids`` names in this bundle, with what they reference, and + running nothing: what a run writes for one a task of another bundle names. Raises BundleError listing every + problem found.""" + return _Planner(resolved, installed).plan_entities(ids) + + def check_bundle(resolved: ResolvedBundle, *, installed: Installed | None = None) -> None: """The checks ``plan_bundle`` makes without reading a store, over every task and eval in the bundle. Raises BundleError listing every problem found.""" @@ -284,7 +291,16 @@ def plan(self, tasks: Sequence[str], evals: Sequence[str]) -> Plan: self._check_writers() self._check_loops() selected_tasks, selected_evals = self._selection(tasks, evals) - closure = self._closure(selected_tasks + selected_evals) + return self._planned(selected_tasks + selected_evals, selected_tasks, selected_evals) + + def plan_entities(self, ids: Sequence[str]) -> Plan: + self._check_writers() + self._check_loops() + return self._planned(self._select_entities(ids), [], []) + + def _planned(self, roots: list[ResolvedEntry], selected_tasks: list[ResolvedEntry], + selected_evals: list[ResolvedEntry]) -> Plan: + closure = self._closure(roots) store_refs, store_latest = self._check_store_refs(closure) self._build_tasks(closure) self._check_files(closure) @@ -415,6 +431,14 @@ def _select(self, kind: BundleKind, wanted: Sequence[str]) -> list[ResolvedEntry self.problems.append(f"{flag}: this bundle has no {kind.store} with that name or id{listed}") return list(selected.values()) + def _select_entities(self, ids: Sequence[str]) -> list[ResolvedEntry]: + entities = {_nfc(entry.entry.id): entry for entry in self.entries + if entry.entry.kind not in (BundleKind.TASK, BundleKind.EVAL)} + for entity_id in ids: + if _nfc(entity_id) not in entities: + self.problems.append(f"this bundle has no env, agent or artifact {entity_id!r}") + return [entities[key] for key in dict.fromkeys(map(_nfc, ids)) if key in entities] + def _closure(self, roots: list[ResolvedEntry]) -> set[_Key]: closure: set[_Key] = set() frontier = [_key(root.entry) for root in roots] diff --git a/src/agent_env/task/__init__.py b/src/agent_env/task/__init__.py index fbaffc5f..a16aa803 100644 --- a/src/agent_env/task/__init__.py +++ b/src/agent_env/task/__init__.py @@ -1,13 +1,13 @@ """Task module for AgentEnv.""" from .task import Task -from .episode import Episode, EpisodeError, EpisodeSpec, Skill, Turn from .store import ( TaskStore, TaskQuery, get_task_store, set_task_store, reset_task_store, TaskInstance, TaskInstanceStore, get_task_instance_store, set_task_instance_store, reset_task_instance_store, TaskStepResult, TaskStepStatus, SupersededAttemptError, record_step_complete, record_task_cancelled, record_task_failure, ) +from .episode import Episode, EpisodeError, EpisodeSpec, Skill, Turn __all__ = [ "Task", diff --git a/src/agent_env/task/episode/rules.py b/src/agent_env/task/episode/rules.py index 176ce491..e2a4819c 100644 --- a/src/agent_env/task/episode/rules.py +++ b/src/agent_env/task/episode/rules.py @@ -1,7 +1,7 @@ """Which steps a task may have when some of its agents are played from outside AgentEnv, and why the others are refused. -An outside agent is played through a stand-in agent deployed in its place (``agentenv_protocol.a2a_agent.stand_in``), -so steps that act on the agent, or in its sandbox, act on the stand-in. Every registered step type is either checked +An outside agent is played through a stand-in agent deployed in its place (the ``agents`` bundle's ``stand-in``), so +steps that act on the agent, or in its sandbox, act on the stand-in. Every registered step type is either checked here (``CHECKS``) or listed as untouched by an outside agent (``UNAFFECTED``). A test holds the registry to the two, so a new step type is classified before an episode runs it. Plugin step types aren't inspected. """ diff --git a/src/agent_env/task/episode/steps.py b/src/agent_env/task/episode/steps.py index f6df0f33..09cb9b1e 100644 --- a/src/agent_env/task/episode/steps.py +++ b/src/agent_env/task/episode/steps.py @@ -5,13 +5,16 @@ from agent_env.a2a_agent import protocol from agent_env.a2a_agent.a2a_agent import A2AAgent -from agent_env.a2a_agent.stand_in import STAND_IN_AGENT_ID, stand_in_agent +from agent_env.bundle.installed import write_installed from agent_env.task.episode.run import _episode_of from agent_env.task_step.context import TaskStepContext from agent_env.task_step.registry import get_task_step_registry from agent_env.task_step.task_step import TaskStep from agent_env.task_step.task_steps.deploy_agent import bound_deployment, live_mcp_url +# The agents bundle's stand-in, deployed in the place of an agent an episode plays. +STAND_IN_AGENT_ID = "@local/agentenv-framework/agents/stand-in" + class _EpisodeAgentStep(TaskStep): """An outside agent's ``deploy_agent`` or ``install_agent``. In a task that uses the agent's sandbox it runs that @@ -52,7 +55,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: config = A2AAgent.negotiate_agent_config(deployed.a2a_card, {"system_prompt": system_prompt}) await protocol.post_agent_config(deployed.a2a_url + config.endpoint, config.fields) return context - await asyncio.to_thread(stand_in_agent) + await asyncio.to_thread(write_installed, STAND_IN_AGENT_ID) override = {"a2a_agent_id": STAND_IN_AGENT_ID} if self.step.type == "deploy_agent" else { "stand_in": STAND_IN_AGENT_ID} step_params = context.metadata.setdefault("user_overrides", {}).setdefault("step_params", {}) diff --git a/tst/unit/a2a_agent/stand_in_test.py b/tst/unit/a2a_agent/stand_in_test.py deleted file mode 100644 index 41d7c3ca..00000000 --- a/tst/unit/a2a_agent/stand_in_test.py +++ /dev/null @@ -1,52 +0,0 @@ -"""The stand-in agent the Episode API deploys in place of an agent played from outside: written to the store as an -image's build context, and written again only when that context changes.""" - -from importlib.metadata import version - -from agent_env.a2a_agent import stand_in -from agent_env.a2a_agent.a2a_agent import TAKES_CREDENTIALS, A2AAgent -from agent_env.a2a_agent.stand_in import STAND_IN_AGENT_ID, stand_in_agent - - -def test_the_stand_in_is_a_build_context_that_takes_no_credentials(local_stores, cli_routing): - agent = stand_in_agent() - - assert agent.id == STAND_IN_AGENT_ID - assert agent.docker_image_artifact.context_only - assert (agent.default_env_vars, agent.metadata) == ({}, {TAKES_CREDENTIALS: False}) - - -class _NoModelConfig: - def __getattr__(self, name): - raise AssertionError(f"an agent that takes no credentials asked config for {name}") - - -def test_an_agent_that_takes_no_credentials_gets_no_model_endpoint_key_or_headers_whoever_passes_them(): - agent = A2AAgent(id="a", version=1, docker_image_artifact=None, metadata={TAKES_CREDENTIALS: False}) - passed = {"LITELLM_API_KEY": "sk-from-the-step", "LITELLM_BASE_URL": "http://model", "LITELLM_EXTRA_HEADERS": "{}", - "GITHUB_TOKEN": "the task's own"} - - assert agent._with_model_credentials(passed, _NoModelConfig()) == {"GITHUB_TOKEN": "the task's own"} - - -def test_an_unchanged_context_reuses_the_stand_in_and_a_changed_one_writes_it_again( - local_stores, cli_routing, monkeypatch): - first = stand_in_agent() - assert stand_in_agent().version == first.version - - monkeypatch.setattr(stand_in, "_requirements", lambda: ["httpx==0.0.1"]) - - assert stand_in_agent().version == first.version + 1 - - -def test_its_context_holds_the_server_the_protocol_package_and_the_pinned_requirements_to_serve(tmp_path): - context = stand_in._stage(tmp_path) - - assert sorted(path.name for path in context.iterdir()) == [ - "Dockerfile", "a2a_server.py", "agentenv_protocol", "requirements.txt"] - assert (context / "agentenv_protocol" / "a2a_agent" / "stand_in.py").is_file() - assert not list(context.rglob("__pycache__")) - pinned = dict(line.split("==") for line in (context / "requirements.txt").read_text().splitlines()) - assert {"a2a-sdk[http-server]", "httpx", "pydantic", "starlette", "uvicorn"} <= pinned.keys() - assert not pinned.keys() & {"fastmcp", "pytest", "pytest-asyncio"} - assert all(pinned[name] == version(name.split("[")[0]) for name in pinned) diff --git a/tst/unit/agents/stand_in/__init__.py b/tst/unit/agents/stand_in/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tst/unit/agents/stand_in/agent.py b/tst/unit/agents/stand_in/agent.py new file mode 100644 index 00000000..7bf5d791 --- /dev/null +++ b/tst/unit/agents/stand_in/agent.py @@ -0,0 +1,12 @@ +"""The stand-in's a2a_server, imported from its folder by its path: Claude Code's, imported by the same name, may be +loaded already.""" + +import importlib.util +from pathlib import Path + +import agent_env + +AGENT_DIR = Path(agent_env.__file__).parent / "agents" / "agents" / "stand-in" +_spec = importlib.util.spec_from_file_location("stand_in_a2a_server", AGENT_DIR / "a2a_server.py") +a2a_server = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(a2a_server) diff --git a/tst/unit/agents/stand_in/test_stand_in_agent.py b/tst/unit/agents/stand_in/test_stand_in_agent.py new file mode 100644 index 00000000..d01cd7c3 --- /dev/null +++ b/tst/unit/agents/stand_in/test_stand_in_agent.py @@ -0,0 +1,56 @@ +"""The agents bundle's stand-in: its card, how it installs on a VM's host, and how the episode writes it.""" + +from __future__ import annotations + +import re +import subprocess + +from agent_env.a2a_agent.a2a_agent import A2AAgent +from agent_env.bundle.installed import write_installed +from agent_env.task.episode.steps import STAND_IN_AGENT_ID +from agent_env.task_step.task_steps.install_agent import _build_param_resolvers + +from tst.unit.agents.stand_in.agent import AGENT_DIR, a2a_server + + +def _extensions() -> dict: + card = a2a_server.AGENT_CARD.model_dump(mode="json", exclude_none=True, by_alias=True) + return {ext["uri"]: ext for ext in card["capabilities"]["extensions"]} + + +def test_the_card_offers_the_driver_and_a_host_install_only(): + extensions = _extensions() + assert {"urn:agentenv:driver/v1", "urn:agentenv:mcp-config/v1", "urn:agentenv:trajectory/v1", + "urn:agentenv:install/v1"} <= set(extensions) + params = extensions["urn:agentenv:install/v1"]["params"] + assert params["install_commands_host"] and not params.get("install_commands") + + +def test_the_host_install_asks_only_for_params_install_agent_supplies_and_runs_the_folders_script(): + params = _extensions()["urn:agentenv:install/v1"]["params"] + supplied = set(_build_param_resolvers(container_name=None, work_dir="/w", agent_ctx_tar="/w/a.tgz", a2a_port=8000, + config=None, agent_name="a")) + placeholders = {name for command in params["install_commands_host"] for name in re.findall(r"\{(\w+)\}", command)} + assert placeholders == set(params["required_params_host"]) and placeholders <= supplied + assert params["install_commands_host"][-1] == "sh {work_dir}/agent/install.sh {agent_name} {a2a_port} {workdir} {user}" + + +def test_the_install_script_parses_and_the_image_keeps_it_in_its_build_context(): + subprocess.run(["sh", "-n", str(AGENT_DIR / "install.sh")], check=True) + assert "COPY a2a_server.py install.sh ./" in (AGENT_DIR / "Dockerfile").read_text() + + +def test_the_image_works_in_app_as_agent(): + lines = (AGENT_DIR / "Dockerfile").read_text().splitlines() + last = {key: next(line.split(maxsplit=1)[1] for line in reversed(lines) if line.startswith(f"{key} ")) + for key in ("USER", "WORKDIR")} + assert last == {"USER": "agent", "WORKDIR": "/app"} + + +def test_the_episode_writes_it_from_the_bundle_taking_no_credentials(local_stores, cli_routing): + version = write_installed(STAND_IN_AGENT_ID) + agent = A2AAgent.get(STAND_IN_AGENT_ID) + + assert (agent.version, agent.metadata) == (version, {"takes_credentials": False}) + assert agent.docker_image_artifact.context_only + assert write_installed(STAND_IN_AGENT_ID) == version diff --git a/tst/unit/bundle/installed_entities_test.py b/tst/unit/bundle/installed_entities_test.py index 26af5b58..317228eb 100644 --- a/tst/unit/bundle/installed_entities_test.py +++ b/tst/unit/bundle/installed_entities_test.py @@ -12,10 +12,12 @@ from agent_env.bundle import BundleError, BundleKind, dry_run_bundle, parse_bundle from agent_env.bundle import preflight as preflight_module from agent_env.bundle import run as run_module -from agent_env.bundle.installed import InstalledBundle +from agent_env.a2a_agent.a2a_agent import A2AAgent +from agent_env.bundle import installed as installed_module +from agent_env.bundle.installed import InstalledBundle, write_installed from agent_env.bundle.ledger import locked_ids from agent_env.bundle.materialize import materialize -from agent_env.bundle.plan import check_bundle, plan_bundle +from agent_env.bundle.plan import check_bundle, plan_bundle, plan_entities from agent_env.bundle.resolve import resolve_bundle from tst.unit.bundle._support import RefusingStore, layout, local_builds @@ -184,6 +186,48 @@ def test_a_pulled_entity_naming_an_image_folder_of_its_bundle_pulls_the_image(bu (BundleKind.TASK, f"{ROOT}/t")] + +def test_an_entity_planned_alone_is_written_with_what_it_references_and_nothing_runs(tmp_path): + kit = _kit(tmp_path, {"artifacts/base/Dockerfile": "FROM scratch\n", "agents/coder/agent.toml": 'image = "base"\n'}) + plan = plan_entities(resolve_bundle(parse_bundle(kit.root, id_root=KIT)), [f"{KIT}/coder"], installed=lambda: (kit,)) + + assert _writes(plan) == [(BundleKind.ARTIFACT, f"{KIT}/base"), (BundleKind.AGENT, f"{KIT}/coder")] + assert (plan.tasks, plan.evals) == ((), ()) + + +def test_planning_an_entity_the_bundle_doesnt_define_is_refused(tmp_path): + kit = _kit(tmp_path, {"tasks/t.json": json.dumps([{"id": "agent", "type": "deploy_agent", "a2a_agent_id": SOLVER}])}) + + with pytest.raises(BundleError) as caught: + plan_entities(resolve_bundle(parse_bundle(kit.root, id_root=KIT)), [f"{KIT}/coder", f"{KIT}/t"]) + assert caught.value.problems == (f"this bundle has no env, agent or artifact '{KIT}/coder'", + f"this bundle has no env, agent or artifact '{KIT}/t'") + + +def test_an_installed_bundles_agent_is_written_by_its_id_and_anew_only_once_it_changes( + tmp_path, monkeypatch, local_stores, cli_routing): + kit = _kit(tmp_path) + monkeypatch.setattr(installed_module, "installed_bundles", lambda: (kit,)) + + version = write_installed(SOLVER) + assert A2AAgent.get(SOLVER).docker_image_artifact.id == f"{SOLVER}__agent_image" + assert write_installed(SOLVER) == version + (kit.root / "agents/solver/Dockerfile").write_text("FROM scratch\nENV CHANGED=1\n") + assert write_installed(SOLVER) == version + 1 + + +@pytest.mark.parametrize(("installed", "problem"), [ + ((), "'@local/demo-dist/kit/solver' isn't an entity of an installed bundle"), + (("broken",), "demo-dist/kit: its folder is gone"), +]) +def test_an_entity_no_installed_bundle_can_write_is_refused(tmp_path, monkeypatch, installed, problem): + bundles = tuple(_kit(tmp_path, problem="its folder is gone") for _ in installed) + monkeypatch.setattr(installed_module, "installed_bundles", lambda: bundles) + + with pytest.raises(BundleError) as caught: + write_installed(SOLVER) + assert caught.value.problems == (problem,) + def test_a_pulled_entity_naming_the_running_bundles_own_entity_uses_it(tmp_path, local_stores, cli_routing): base = _installed(tmp_path / "site" / "base", { "artifacts/base/Dockerfile": "FROM scratch\n", diff --git a/tst/unit/bundle/materialize_test.py b/tst/unit/bundle/materialize_test.py index 102c0414..ebed67c4 100644 --- a/tst/unit/bundle/materialize_test.py +++ b/tst/unit/bundle/materialize_test.py @@ -347,6 +347,7 @@ def refuse(ref, auth): [metadata] default_model = "claude-sonnet-4-6" min_disk_size_gb = 20 +takes_credentials = false """ @@ -359,7 +360,7 @@ def test_an_agent_is_written_from_its_agent_toml_over_a_store_image(bundle_dir): agent = A2AAgent.get(f"{ROOT}/solver") assert (agent.docker_image_artifact.id, agent.docker_image_artifact.version) == ("claude-image", 1) assert (agent.default_env_vars, agent.metadata) == ( - {"LOG_LEVEL": "debug"}, {"default_model": "claude-sonnet-4-6", "min_disk_size_gb": 20}) + {"LOG_LEVEL": "debug"}, {"default_model": "claude-sonnet-4-6", "min_disk_size_gb": 20, "takes_credentials": False}) assert Task.get(f"{ROOT}/t").steps[0].a2a_agent_id == f"{ROOT}/solver" assert _summary(_run(bundle_dir))[f"{ROOT}/solver"] == (1, True, ()) @@ -372,11 +373,16 @@ def test_an_agent_is_written_from_its_agent_toml_over_a_store_image(bundle_dir): ('image = "claude-image"\ndefault_env_vars = { PORT = 8080 }\n', "agent.toml: default_env_vars.PORT must be a string, not 8080"), ('image = "claude-image"\n[metadata]\nowner = "me"\n', - "agent.toml: unknown [metadata] key 'owner'; an agent's [metadata] takes default_model and min_disk_size_gb"), + "agent.toml: unknown [metadata] key 'owner'; an agent's [metadata] takes default_model, min_disk_size_gb and " + "takes_credentials"), ('image = "claude-image"\n[metadata]\nmin_disk_size_gb = "20"\n', "agent.toml: metadata.min_disk_size_gb must be a number, not '20'"), ('image = "claude-image"\n[metadata]\nmin_disk_size_gb = true\n', "agent.toml: metadata.min_disk_size_gb must be a number, not True"), + ('image = "claude-image"\n[metadata]\ntakes_credentials = "no"\n', + "agent.toml: metadata.takes_credentials must be true or false, not 'no'"), + ('image = "claude-image"\n[metadata]\ntakes_credentials = 0\n', + "agent.toml: metadata.takes_credentials must be true or false, not 0"), ('image = "no-such-image"\n', "image: there is no artifact 'no-such-image' in the store"), ('image = "greeting"\n', "image: 'greeting' is this bundle's file, but this field takes docker_image"), ]) @@ -397,8 +403,8 @@ def test_every_problem_in_an_agent_toml_is_refused_together(bundle_dir): assert sorted(_problems(lambda: _run(bundle_dir))) == [ "agents/solver: agent.toml: default_env_vars.PORT must be a string, not 8080", - "agents/solver: agent.toml: unknown [metadata] key 'color'; an agent's [metadata] takes default_model and " - "min_disk_size_gb", + "agents/solver: agent.toml: unknown [metadata] key 'color'; an agent's [metadata] takes default_model, " + "min_disk_size_gb and takes_credentials", "agents/solver: agent.toml: unknown key 'owner'; an a2a_agent agent takes image, default_env_vars, " "metadata, type and id", ] diff --git a/tst/unit/bundle/to_toml_round_trip_test.py b/tst/unit/bundle/to_toml_round_trip_test.py index 6a5fd41a..bd47bb90 100644 --- a/tst/unit/bundle/to_toml_round_trip_test.py +++ b/tst/unit/bundle/to_toml_round_trip_test.py @@ -112,17 +112,20 @@ def test_a_website_env_reads_back_with_both_images_built_from_its_one_folder(hom def test_an_agent_reads_back_with_its_env_vars_and_the_metadata_its_toml_carries(home): A2AAgent.put(id="solver", docker_image_artifact=context_image(home, "solver-image"), default_env_vars={"MODE": "fast", "OPENAI_API_KEY": "sk-not-real"}, - metadata={"default_model": "claude", "min_disk_size_gb": 20, "agent_card": {"name": "x"}}) + metadata={"default_model": "claude", "min_disk_size_gb": 20, "takes_credentials": False, + "agent_card": {"name": "x"}}) _env(home, "crm") steps = [{"id": "env", "type": "deploy_env", "env_id": "crm"}, {"id": "agent", "type": "deploy_agent", "env_ids": ["crm"], "a2a_agent_id": "solver"}] with _read_back(home, steps, "agent", "solver", user=True) as (read, imported): assert read.default_env_vars == {"MODE": "fast"} - assert read.metadata == {"default_model": "claude", "min_disk_size_gb": 20} - assert {(drop.where, drop.why.split(",")[0]) for drop in imported.dropped} >= { - ("agents/solver: metadata", "an agent.toml's [metadata] takes only default_model and min_disk_size_gb"), - ("agents/solver/agent.toml: default_env_vars.OPENAI_API_KEY", "dropped: its name looks like a secret")} + assert read.metadata == {"default_model": "claude", "min_disk_size_gb": 20, "takes_credentials": False} + whys = {drop.where: drop.why for drop in imported.dropped} + assert whys["agents/solver: metadata"] == ( + "an agent.toml's [metadata] takes only default_model, min_disk_size_gb and takes_credentials, not agent_card") + assert whys["agents/solver/agent.toml: default_env_vars.OPENAI_API_KEY"].startswith( + "dropped: its name looks like a secret") @pytest.mark.parametrize("description", ["rows.csv", "the crm's rows"]) diff --git a/tst/unit/task_step/install_agent_test.py b/tst/unit/task_step/install_agent_test.py index c2525076..6cb06402 100644 --- a/tst/unit/task_step/install_agent_test.py +++ b/tst/unit/task_step/install_agent_test.py @@ -312,7 +312,7 @@ def test_the_trust_lines_resolve_as_the_trust_env_param(): resolvers = _build_param_resolvers(container_name="c", work_dir="/w", agent_ctx_tar="/w/a.tgz", a2a_port=8000, config=None, agent_name="a", trust_lines="SSL_CERT_FILE=/x\n") assert resolvers["trust_env"]() == "SSL_CERT_FILE=/x\n" -STAND_IN = "@local/agentenv-stand-in" +STAND_IN = "@local/agentenv-framework/agents/stand-in" CODEX_INSTALL = {"params": {"workdir_host": "/opt/agent-{agent_name}/workspace", "user_host": "agent-{agent_name}", "install_commands_host": ["install codex"], "a2a_port": 8000}} STAND_IN_INSTALL = {"params": {"install_commands_host": ["serve as {user} in {workdir} on {a2a_port}"],