diff --git a/AGENTS.md b/AGENTS.md index 284ff365..bca3cd73 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -81,7 +81,8 @@ or one of them only where it is used. what steps staged in `/tmp` under that id. Those processes are found through `/proc`, so on macOS they keep running; files the install wrote elsewhere (`/opt`, packages) stay. - `[agents] default_a2a_agent_id`: the agent a `deploy_agent` step without an id deploys; built-in - `a2a-default`, overridable by `configure(default_a2a_agent_id=...)`. + `a2a-default`, overridable by `configure(default_a2a_agent_id=...)`. Any step's agent field left unset + falls back to it (`entity.refs.default_agent_refs`), so a bundle run plans the default for it. - Secrets never live in the file: use `secret:KEY` or `env:NAME` references, resolved through `[stores.secret]`. - agent-env has no stage concept: "dev" versus "prod" is which file `AGENT_ENV_CONFIG` names. A @@ -134,7 +135,10 @@ Out-of-tree code plugs in either through `config.toml`, as a `module:Class` poin - Bundles install through `agent_env.bundles` entry points (` = ""`, the folder at `//`), with ids rooted at `@local//`. A shared name is not left out: a bundle runs by its bare name unless another distribution installs that name too, and then by - `/`, except that agent-env's own bundles keep their bare names. + `/`, except that agent-env's own bundles keep their bare names. Another bundle uses an + installed bundle's env, agent or artifact by its id, or as the configured default agent: the planner + pulls it into the run as one of the bundle's own (`bundle/plan.py`), and its ledger row is keyed by the + installed bundle, so every bundle shares one write of it. Such an id takes no version pin. - Environment providers have no table: an installed package registers one as an `agent_env.env_providers` entry point named its `type`, which records carry as `env_provider_type`. - `[plugins.]` is reserved for an installed plugin's own settings, read with diff --git a/src/agent_env/a2a_agent/a2a_agent.py b/src/agent_env/a2a_agent/a2a_agent.py index cd0cd772..f22016cc 100644 --- a/src/agent_env/a2a_agent/a2a_agent.py +++ b/src/agent_env/a2a_agent/a2a_agent.py @@ -64,6 +64,15 @@ _DIND_PORT = 2375 +def unwritten_local_agent(agent_id: str, field: str) -> str: + """Why a step can't deploy ``agent_id``, an ``@local`` agent the store doesn't hold, which its ``field`` names or, + left unset, the configured default does: an ``agent-env run`` of a bundle writes one, and a step reads only the + store.""" + return (f"there is no agent {agent_id!r} in the store: an @local agent is written by `agent-env run` of the bundle " + f"that defines it or uses it, so run the task through that bundle, or point {field} or [agents] " + "default_a2a_agent_id at an agent the store holds") + + @dataclass class DeployedA2AAgent: agent_id: str diff --git a/src/agent_env/artifact/artifacts/docker_image.py b/src/agent_env/artifact/artifacts/docker_image.py index abf99242..2c3e4443 100644 --- a/src/agent_env/artifact/artifacts/docker_image.py +++ b/src/agent_env/artifact/artifacts/docker_image.py @@ -64,6 +64,9 @@ def _git_clone_commands(owner: str, repo: str, ref: str | None, token: str | Non f"trap 'rm -f /tmp/git-askpass.sh' EXIT; GIT_ASKPASS=/tmp/git-askpass.sh git clone --depth 1 {ref_flag} {clone_url} /tmp/repo", ] +# How much of a build context's source digest its image's tag holds. +_SOURCES_TAG_CHARS = 16 + class DockerImageArtifact(Artifact): """A Docker image artifact: a tar.gz of the image in the object store; without one, a registry reference a sandbox pulls; or, with neither, only its build context.""" @@ -235,38 +238,42 @@ def put_ref(cls, id: str, *, description: str, image_name: str, @classmethod def put_context(cls, id: str, *, description: str, context_path: str, dockerfile_path: str, - platform: str = DEFAULT_BUILD_PLATFORM) -> "DockerImageArtifact": - """Register an image by its build context alone, to be built where it runs: the context is uploaded and - nothing is built, pushed or saved, so no Docker is needed here. Its ``image_name``, - ``local/-:v``, names no registry: it's the tag the build gives the image. - ``source_digest`` is equal for equal sources, so one build can serve both.""" + platform: str | None = DEFAULT_BUILD_PLATFORM) -> "DockerImageArtifact": + """Register an image by its build context alone, to be built where it runs, for ``platform``, or for each + sandbox's own when None: the context is uploaded and nothing is built, pushed or saved, so no Docker is needed + here. Its ``image_name``, ``local/-:``, names no registry: it's the tag the build gives the + image, the same for the same sources, so a build of one version serves another. ``source_digest`` is equal for + equal sources.""" from agent_env.artifact.store import get_artifact_store from agent_env.config import get_config - platform = normalize_platform(platform) - root = Path(context_path) - dockerfile = _resolve_dockerfile(root, dockerfile_path) - if not dockerfile.is_file(): - raise ValueError(f"no Dockerfile at {dockerfile}") - context = BuildContext.of(root, dockerfile) - if context.dockerfile is None: - raise ValueError(f"{dockerfile} is outside the build context {root}; the image is built from the context " - "alone, so the Dockerfile must be in it") + platform = normalize_platform(platform) if platform else None + context = _context_of(Path(context_path), dockerfile_path) store = get_artifact_store() version = store.next_version(id) prefix = store.attempt_prefix("docker_image", id) build_context_object_url = _upload_context(get_config().get_object_store_to_write(prefix, id), prefix, context) + source_digest = context.source_digest(platform) return store.put_document(cls( id=id, version=version, description=description, - image_name=f"{local_image_repository(id)}:v{version}", + image_name=_context_image_name(id, source_digest), build_context_object_url=build_context_object_url, dockerfile_path=context.dockerfile, platform=platform, - source_digest=context.source_digest(platform), + source_digest=source_digest, )) + @staticmethod + def context_identity(id: str, context_path: str, dockerfile_path: str, + platform: str | None = DEFAULT_BUILD_PLATFORM) -> tuple[str, str]: + """The ``source_digest`` and ``image_name`` ``put_context`` gives ``id`` for this build context, found without + writing anything.""" + platform = normalize_platform(platform) if platform else None + source_digest = _context_of(Path(context_path), dockerfile_path).source_digest(platform) + return source_digest, _context_image_name(id, source_digest) + def load(self) -> bytes: from agent_env.artifact.store import get_artifact_store if not self.tar_gz_object_url: @@ -599,6 +606,22 @@ def _resolve_dockerfile(context_dir: Path, dockerfile_path: str | None) -> Path return dockerfile if dockerfile.is_absolute() or dockerfile.exists() else context_dir / dockerfile +def _context_of(root: Path, dockerfile_path: str) -> BuildContext: + dockerfile = _resolve_dockerfile(root, dockerfile_path) + if not dockerfile.is_file(): + raise ValueError(f"no Dockerfile at {dockerfile}") + context = BuildContext.of(root, dockerfile) + if context.dockerfile is None: + raise ValueError(f"{dockerfile} is outside the build context {root}; the image is built from the context " + "alone, so the Dockerfile must be in it") + return context + + +def _context_image_name(id: str, source_digest: str) -> str: + """The tag a build gives a context-only image: the same for the same sources.""" + return f"{local_image_repository(id)}:{source_digest.removeprefix('sha256:')[:_SOURCES_TAG_CHARS]}" + + def _upload_context(objects, prefix: str, context: BuildContext) -> str: """Write ``context`` under ``prefix`` as a tar.gz and return its object URL.""" with tempfile.TemporaryDirectory(prefix="agent-env-context-") as tmp: diff --git a/src/agent_env/bundle/importer.py b/src/agent_env/bundle/importer.py index 7cf67a21..39bdc586 100644 --- a/src/agent_env/bundle/importer.py +++ b/src/agent_env/bundle/importer.py @@ -19,11 +19,12 @@ from agent_env.bundle import _toml from agent_env.bundle._fs import fold, os_reason, show from agent_env.bundle._secrets import scrub +from agent_env.bundle.installed import InstalledBundle, installed_bundles from agent_env.bundle.parse import BundleError, parse_bundle, root_problems, segment_problem from agent_env.bundle.plan import STORE_GETTERS, check_bundle from agent_env.bundle.resolve import resolve_bundle from agent_env.config import get_config -from agent_env.entity.refs import EntityKind, RefRole, RefSite, ref_sites +from agent_env.entity.refs import EntityKind, EntityRef, RefRole, RefSite, default_agent_refs, ref_sites from agent_env.env import registry as env_registry from agent_env.env.registry import get_env_registry from agent_env.eval import Eval @@ -34,10 +35,7 @@ from agent_env.task import Task from agent_env.task_step import registry as task_step_registry from agent_env.task_step.registry import get_task_step_registry -from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep from agent_env.task_step.task_steps.deploy_env import DeployEnvTaskStep -from agent_env.task_step.task_steps.install_agent import InstallAgentTaskStep -from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep # How long a name taken from an id may be, in bytes; ids are capped far above it, and a longer name helps nobody. _MAX_NAME_BYTES = 200 @@ -183,6 +181,7 @@ def __init__(self, dest: Path, include_scopes: Collection[FieldScope] = ()): self.read: dict[tuple[EntityKind, str, int | None], int | str] = {} self.types: dict[tuple[EntityKind, str, int | None], str] = {} # each entity read, by its read key self.origins: dict[tuple[str, str], str] | None = None + self.installed: tuple[InstalledBundle, ...] | None = None _check_destination(self.dest) def add_task(self, task: Task) -> str: @@ -214,7 +213,7 @@ def add_task(self, task: Task) -> str: if _key(site) not in outputs: # the run writes an output again, so it takes no pin self._pin(site, f"{where}: {site.path}", deployed) self._need_type(site, f"{where}: {site.path}") - self._default_agents(step, where) + self._default_agents(step, cls.entity_refs or (), where) self.files[path] = json.dumps(steps, indent=2, ensure_ascii=False, allow_nan=False) + "\n" self.tasks.append(ImportedTask(path, task.id, task.version, len(steps))) return name @@ -242,7 +241,7 @@ def write(self, description: str) -> BundleImport: (staging / path).write_text(text, encoding="utf-8") except OSError as e: raise BundleError([f"{show(self.dest.parent)}: can't write the import there: {os_reason(e)}"]) from None - check_bundle(resolve_bundle(parse_bundle(staging))) + check_bundle(resolve_bundle(parse_bundle(staging)), installed=installed_bundles) self._move(staging) finally: shutil.rmtree(staging, ignore_errors=True) @@ -267,6 +266,9 @@ def _move(self, staging: Path) -> None: def _pin(self, site: RefSite, where: str, deployed: set[str]) -> None: kind, entity_id = site.ref.kind, site.value + if (bundle := self._installed_for(kind, entity_id)) is not None: + self._from_installed(site, where, bundle) + return if site.version is not None or site.version_key is not None: read = self._read(kind, entity_id, site.version) if isinstance(read, str): @@ -282,6 +284,23 @@ def _pin(self, site: RefSite, where: str, deployed: set[str]) -> None: self.floating.append(Floating(where, kind, entity_id, "the field takes no version and no step of the task " "deploys it, so a run reads the latest")) + def _installed_for(self, kind: EntityKind, entity_id: str) -> InstalledBundle | None: + """The installed bundle whose ids ``entity_id``, an env, agent or artifact, is under: a run writes it from + there.""" + if kind is EntityKind.TASK or not entity_id.startswith(LOCAL_PREFIX): + return None + if self.installed is None: + self.installed = installed_bundles() + return next((bundle for bundle in self.installed if entity_id.startswith(f"{bundle.id_root}/")), None) + + def _from_installed(self, site: RefSite, where: str, bundle: InstalledBundle) -> None: + """An installed bundle's entity has no fixed version, so it isn't pinned, and a pin the store held is dropped.""" + why = f"it comes with the installed bundle {bundle.qualified}, which a run writes it from" + if site.version is not None and site.version_key is not None: + site.owner[site.version_key] = None + why += f"; the stored pin to v{site.version} is dropped, as such an entity has no fixed version" + self.floating.append(Floating(where, site.ref.kind, site.value, why)) + def _read(self, kind: EntityKind, entity_id: str, version: int | None = None) -> int | str: """The version of ``entity_id`` the store holds, at ``version`` or its latest, or why it can't be read.""" if (kind, entity_id, version) not in self.read: @@ -325,17 +344,13 @@ def _need(self, where: str, what: str, group: str, name: str, cls: type | None) by = distribution if registered and _names(value, cls) else "configuration" self.needs.append(Need(where, what, name, by)) - def _default_agents(self, step: dict, where: str) -> None: - """A step that names no agent deploys the configured default, which no field pins.""" - if step["type"] in (DeployAgentTaskStep.type, InstallAgentTaskStep.type) and step.get("a2a_agent_id") is None: - field, what = "a2a_agent_id", "names no agent" - elif (step["type"] == RubricsVerifierTaskStep.type and step.get("use_agent_judge") - and step.get("agent_name") is None and step.get("judge_a2a_agent_id") is None): - field, what = "judge_a2a_agent_id", "names no judge agent" - else: - return - self.floating.append(Floating(f"{where}: {field}", EntityKind.AGENT, get_config().get_default_a2a_agent_id(), - f"the step {what}, so a run deploys the configured default")) + def _default_agents(self, step: dict, refs: tuple[EntityRef, ...], where: str) -> None: + """A step that leaves an agent field unset falls back to the configured default, which no field pins.""" + for ref in default_agent_refs(refs, step): + self.floating.append(Floating(f"{where}: {ref.path}", EntityKind.AGENT, + get_config().get_default_a2a_agent_id(), + f"the step leaves {ref.path} unset, so it falls back to the configured " + "default")) def _name(self, kind: str, entity_id: str) -> str: """A name in the ``kind`` folder: the id's last segment made valid, numbered when taken, caselessly.""" diff --git a/src/agent_env/bundle/installed.py b/src/agent_env/bundle/installed.py index 1b14cf26..72268a68 100644 --- a/src/agent_env/bundle/installed.py +++ b/src/agent_env/bundle/installed.py @@ -102,7 +102,7 @@ def checked(bundle: InstalledBundle, *, load: bool = True) -> Bundle: store. Raises BundleError.""" parsed = parse_bundle(bundle.root, id_root=bundle.id_root) if load: - check_bundle(resolve_bundle(parsed)) + check_bundle(resolve_bundle(parsed), installed=installed_bundles) return parsed diff --git a/src/agent_env/bundle/ledger.py b/src/agent_env/bundle/ledger.py index 3a58dc17..68cddacc 100644 --- a/src/agent_env/bundle/ledger.py +++ b/src/agent_env/bundle/ledger.py @@ -1,7 +1,8 @@ """The bundle ledger: what each version a bundle wrote was made from, so a re-run writes only what changed. A row per written version records a digest of the write's inputs and one digest per input, so a new -version can say why it was written ("files changed: server.py"). The ledger lives in the ``@local`` +version can say why it was written ("files changed: server.py"). A row is keyed by the bundle the write comes from, so +an entity a run pulls in from an installed bundle has one row however many bundles use it. The ledger lives in the ``@local`` namespace's own store, beside the entities it describes, and is never read through the router, so it never reaches a configured store. Reading it never creates the store. """ @@ -11,7 +12,7 @@ import hashlib import json import stat -from collections.abc import Callable, Iterator, Mapping +from collections.abc import Callable, Iterable, Iterator, Mapping from contextlib import contextmanager from dataclasses import dataclass from datetime import date, datetime, time, timezone @@ -29,20 +30,16 @@ from agent_env.store.document_store import LocalSqliteDocumentStore from agent_env.store.local_state import holding_locks from agent_env.task.store import TASKS_COLLECTION -from agent_env.utils.docker_build import DEFAULT_BUILD_PLATFORM from .authoring import build_context_files -from .parse import Bundle, BundleKind +from .parse import BundleKind from .plan import Plan, Write, artifact_writer, env_writer, folder_files, unpinned_store_refs from .resolve import BuiltImage, BundleImage, ReferencedImage LEDGER_COLLECTION = "bundle_ledger" # Bumped by hand when what a digest covers changes, or a writer's output changes enough that every -# version it wrote should be written again. -SCHEME = 1 - -_CONTEXT = "build context" -_FORMS = {None: "an image built on this machine", _CONTEXT: "a build context, built where it runs"} +# version it wrote should be written again: 2 writes every image the bundle builds as its build context alone. +SCHEME = 2 _COLLECTIONS = {"env": ENVS_COLLECTION, "agent": A2A_AGENTS_COLLECTION, "artifact": ARTIFACTS_COLLECTION, "task": TASKS_COLLECTION, "eval": EVALS_COLLECTION} @@ -52,8 +49,7 @@ @dataclass(frozen=True) class Digest: value: str - inputs: dict[str, Any] # type, config, files {key: sha256}, needs and store_refs {" ": version}, and - # for an image written as its build context, its form and the platform it's built for + inputs: dict[str, Any] # type, config, files {key: sha256}, needs and store_refs {" ": version} @dataclass(frozen=True) @@ -81,15 +77,13 @@ def next_version(self) -> int: class Ledger: """One plan's rows in the ``@local`` namespace's store.""" - def __init__(self, store: LocalSqliteDocumentStore, plan: Plan, contexts: frozenset[str] = frozenset()): + def __init__(self, store: LocalSqliteDocumentStore, plan: Plan): self._store = store self._plan = plan - self._bundle = plan.bundle.bundle.id_root - self._contexts = contexts # the built images written as build contexts @classmethod - def for_plan(cls, plan: Plan, contexts: frozenset[str] = frozenset()) -> Ledger: - return cls(get_config().local_namespace_document_store(), plan, contexts) + def for_plan(cls, plan: Plan) -> Ledger: + return cls(get_config().local_namespace_document_store(), plan) def check(self, write: Write, needs: Mapping[tuple[str, str], int]) -> Check: """Compare ``write`` with the latest version the ledger records for it. ``needs`` holds, by (store, id), @@ -107,7 +101,7 @@ def check(self, write: Write, needs: Mapping[tuple[str, str], int]) -> Check: row = self._latest(write.kind.store, write.id) recorded = row["version"] if row else None reasons = [] - if row is not None and row["bundle"] != self._bundle: + if row is not None and row["bundle"] != self._bundle(write): reasons.append(f"last written by the bundle {row['bundle']}") if stored is None: reasons.append("new" if row is None else f"its v{recorded} is not in the store") @@ -128,11 +122,8 @@ def digest(self, write: Write, needs: Mapping[tuple[str, str], int]) -> Digest | if write.kind in (BundleKind.ARTIFACT, BundleKind.SKILL) and not isinstance(write.source, ReferencedImage): built = isinstance(write.source, BuiltImage) listing = build_context_files if built else folder_files - for key, path in listing(self._plan.bundle.bundle, write.source.entry).items(): + for key, path in listing(self._plan.bundle_of(write.source.entry), write.source.entry).items(): inputs["files"][key] = _file_sha256(path) + (_mode(path) if built else "") - if built and write.id in self._contexts: - # Only then, so an image built on this machine is hashed as before and earlier rows stay reused. - inputs.update(form=_CONTEXT, platform=DEFAULT_BUILD_PLATFORM) if write.kind not in (BundleKind.TASK, BundleKind.EVAL): # A document records the versions of what it references (an env its images, a universe its # environments), so one written anew means it must be written again. A task or eval names its @@ -148,7 +139,7 @@ def record(self, check: Check, write: Callable[[], int]) -> int: """Write ``check``'s entity with ``write``, which returns the version it wrote, and record it. A pending row marks the attempt until it's done, with its version once ``write`` returns. One an interrupted run of this bundle left is kept by ``check`` when its version can be, and dropped here.""" - key = {"store": check.write.kind.store, "id": check.write.id, "bundle": self._bundle} + key = {"store": check.write.kind.store, "id": check.write.id, "bundle": self._bundle(check.write)} self._store.ensure_index(LEDGER_COLLECTION, ["store", "id", "status"]) while self._store.delete(LEDGER_COLLECTION, Filter.of(**key, status="pending")): pass @@ -169,7 +160,7 @@ def record(self, check: Check, write: Callable[[], int]) -> int: def adopt(self, check: Check) -> None: """Record the version an interrupted run wrote, which ``check`` keeps, as that run would have.""" - key = {"store": check.write.kind.store, "id": check.write.id, "bundle": self._bundle} + key = {"store": check.write.kind.store, "id": check.write.id, "bundle": self._bundle(check.write)} done = {**key, "status": "done", "scheme": SCHEME, "digest": check.digest.value, "inputs": check.digest.inputs, "version": check.version, "at": _now()} self._store.replace(LEDGER_COLLECTION, Filter.of(**key, status="pending"), done, upsert=True) @@ -177,12 +168,16 @@ def adopt(self, check: Check) -> None: def _orphan(self, write: Write, stored: int | None, digest: Digest) -> int | None: """The version an interrupted run of this bundle wrote and didn't record, when it's still the store's latest and was made from what ``write`` is now made from.""" - pending = self._find(LEDGER_COLLECTION, Filter.of(store=write.kind.store, id=write.id, bundle=self._bundle, - status="pending")) + pending = self._find(LEDGER_COLLECTION, Filter.of(store=write.kind.store, id=write.id, + bundle=self._bundle(write), status="pending")) if pending is None or pending.get("version") is None or pending["version"] != stored: return None return stored if (pending["scheme"], pending["digest"]) == (SCHEME, digest.value) else None + def _bundle(self, write: Write) -> str: + """The id root of the bundle ``write`` comes from, which keys its rows.""" + return self._plan.bundle_of(write.source.entry).id_root + def _latest(self, store: str, id: str) -> dict | None: return self._find(LEDGER_COLLECTION, Filter.of(store=store, id=id, status="done")) @@ -198,14 +193,19 @@ def _find(self, collection: str, filter: Filter) -> dict | None: @contextmanager -def materializing(bundle: Bundle, on_wait: Callable[[], None] | None = None) -> Iterator[None]: - """Hold a lock on every id the bundle's entries write while its entities are written. Another run that - writes any of those ids, of this bundle or another, waits here, calling ``on_wait`` first. Task runs don't - hold them.""" - with holding_locks((entry.id for entry in bundle.entries), on_wait): +def materializing(ids: Iterable[str], on_wait: Callable[[], None] | None = None) -> Iterator[None]: + """Hold a lock on each of ``ids`` while a run writes them: a plan's ``locked_ids``. Another run that writes any of + them, of this bundle or another, waits here, calling ``on_wait`` first. Task runs don't hold them.""" + with holding_locks(ids, on_wait): yield +def locked_ids(plan: Plan) -> list[str]: + """The ids a run of ``plan`` locks: every one its bundle's entries write, and each entity it pulls in from another + bundle, which another bundle's run can write too.""" + return [*(entry.id for entry in plan.bundle.bundle.entries), *(entry.id for entry in plan.origins)] + + def _tracked(write: Write) -> bool: """Whether the ledger can list everything ``write`` is made from. Not yet for an env type with a ``from_toml`` of its own, which may read its folder in ways it can't see. An artifact, a skill included, is made from its toml, @@ -253,10 +253,6 @@ def _changes(row: Mapping[str, Any], digest: Digest) -> list[str]: reasons.append(f"type changed: {before['type']} → {after['type']}") if before.get("config") != after["config"]: reasons.append("config changed") - if before.get("form") != after.get("form"): - reasons.append(f"now written as {_FORMS[after.get('form')]}, not {_FORMS[before.get('form')]}") - elif before.get("platform") != after.get("platform"): - reasons.append(f"platform changed: {before.get('platform')} → {after.get('platform')}") for label, names in zip(("files added", "files removed", "files changed"), _diff(before["files"], after["files"])): if names: shown = ", ".join(names[:_LISTED]) + (f" and {len(names) - _LISTED} more" if len(names) > _LISTED else "") diff --git a/src/agent_env/bundle/materialize.py b/src/agent_env/bundle/materialize.py index 8a11ab37..9029eed2 100644 --- a/src/agent_env/bundle/materialize.py +++ b/src/agent_env/bundle/materialize.py @@ -3,8 +3,8 @@ Materializing first refuses every write this release has no writer for, so nothing is written for a bundle that can't be written whole. It needs the CLI's namespace routing, which sends ``@local`` writes to the ``@local`` namespace's store. Holding the bundle's lock, it writes the entities in the plan's order, an image -built from an entry's Dockerfile just before the entry: with ``docker build`` on this machine, or, for one -whose deploys build it where they run, as its build context alone. Then it builds and preflights every task +built from an entry's Dockerfile just before the entry, as its build context alone, which each deploy builds where +it runs (a sandbox on this machine with this machine's Docker). Then it builds and preflights every task before writing any of them, and writes the evals last, since they name the tasks. Each write goes through the ledger, so one whose inputs haven't changed reuses the version the bundle last wrote. A reused task keeps whatever its steps took from config when it was first written, such as a rubrics verifier's default judge @@ -30,17 +30,17 @@ from agent_env.entity.refs import EntityRef, RefRole, ref_sites from agent_env.env.registry import get_env_registry from agent_env.eval import Eval, EvalTask +from agent_env.providers.sandbox_providers.host_images import build, build_staged from agent_env.store.base import NotFoundError -from agent_env.store.ids import image_repository from agent_env.store.routing import namespace_routing_enabled from agent_env.task import Task from agent_env.task_step.registry import get_task_step_registry from agent_env.task_step.task_step import TaskStep -from agent_env.utils.docker_build import DockerBuildError, build_image +from agent_env.utils.docker_build import DockerBuildError -from ._fs import relative, with_article +from ._fs import with_article from .authoring import AuthoringContext, build_context_files -from .ledger import Ledger, materializing +from .ledger import Ledger, locked_ids, materializing from .parse import BundleError, BundleKind from .plan import Plan, Write, artifact_writer, env_writer, unpinned_store_refs from .resolve import BuiltImage, BundleImage, ReferencedImage, build_step @@ -62,7 +62,6 @@ class Materialization: plan: Plan writes: tuple[Materialized, ...] # in the plan's order - contexts: frozenset[str] = frozenset() # the ids of the built images written as build contexts # A dry run's steps with a preflight that read what it would write first, so the store can't check them yet. not_preflighted: tuple[tuple[Write, TaskStep], ...] = () @@ -80,13 +79,14 @@ def materialize( on_wait: Callable[[], None] | None = None, on_build: Callable[[Write], None] | None = None, on_write: Callable[[Materialized], None] | None = None, - contexts: frozenset[str] = frozenset(), + build_here: frozenset[str] = frozenset(), ) -> Materialization: """Write ``plan``'s entities, tasks and evals. The first write that fails stops it, and every earlier one stays: they're in the ledger, so the next run reuses them. ``on_wait`` is called when another run - holds a lock this one needs, ``on_build`` before an image is built, and ``on_write`` after each write, - reused or not. An image the bundle builds whose id is in ``contexts`` is written as its build context - alone, for the providers that deploy it to build (``preflight_run`` decides which). + holds a lock this one needs, ``on_build`` before an image is built here, and ``on_write`` after each write, + reused or not. An image the bundle builds whose id is in ``build_here``, as a deploy on this machine runs it + (``preflight_run``), is also built with this machine's Docker, before it's written or, reused, when Docker doesn't + hold it, so a build that fails writes nothing; its version is its build context either way. ``dry_run`` checks and preflights what a run would, and writes nothing: no entity, ledger row or lock. Each write gets the version it would reuse or the store's next, which another run can take first. A step @@ -95,7 +95,7 @@ def materialize( if not namespace_routing_enabled(): raise RuntimeError("materializing a bundle needs namespace routing, which the agent-env CLI turns on; " "call it inside agent_env.store.routing.namespace_routing()") - ledger = Ledger.for_plan(plan, contexts) + ledger = Ledger.for_plan(plan) entities = [write for write in plan.writes if write.kind not in (BundleKind.TASK, BundleKind.EVAL)] tasks = [write for write in plan.writes if write.kind is BundleKind.TASK] evals = [write for write in plan.writes if write.kind is BundleKind.EVAL] @@ -116,10 +116,12 @@ def through_ledger(write: Write, write_fn: Callable[[], int]) -> None: if on_write is not None: on_write(done[_key(write)]) - with nullcontext() if dry_run else materializing(plan.bundle.bundle, on_wait): - _refuse_builds_without_docker(plan, ledger, contexts) # once another run writing these ids is done + with nullcontext() if dry_run else materializing(locked_ids(plan), on_wait): for write in entities: - through_ledger(write, lambda: _write_entity(plan, write, on_build, contexts)) + here = write.id in build_here and not dry_run + through_ledger(write, lambda: _write_entity(plan, write, here, on_build)) + if here and done[_key(write)].reused: + _build_reused(plan, write, done[_key(write)].version, on_build) unwritten = {_key(item.write) for item in done.values() if not item.reused} if dry_run else set() built, problems, unchecked = {}, [], [] for write in tasks: @@ -134,20 +136,15 @@ def through_ledger(write: Write, write_fn: Callable[[], int]) -> None: through_ledger(write, lambda: Task.put(id=write.id, steps=built[write.id].steps).version) for write in evals: through_ledger(write, lambda: _WRITERS[write.kind](plan, write)) - return Materialization(plan, tuple(done[_key(write)] for write in plan.writes), contexts, tuple(unchecked)) + return Materialization(plan, tuple(done[_key(write)] for write in plan.writes), tuple(unchecked)) -def _write_entity(plan: Plan, write: Write, on_build: Callable[[Write], None] | None, - contexts: frozenset[str]) -> int: +def _write_entity(plan: Plan, write: Write, here: bool, on_build: Callable[[Write], None] | None) -> int: if isinstance(write.source, ReferencedImage): return _write_referenced_image(plan, write) - if not isinstance(write.source, BuiltImage): - return _WRITERS[write.kind](plan, write) - if write.id in contexts: - return _write_build_context(plan, write) - if on_build is not None: - on_build(write) - return _write_built_image(plan, write) + if isinstance(write.source, BuiltImage): + return _write_build_context(plan, write, here, on_build) + return _WRITERS[write.kind](plan, write) @contextmanager @@ -156,26 +153,58 @@ def _staged(plan: Plan, image: BuiltImage) -> Iterator[Path]: what it hashed.""" with tempfile.TemporaryDirectory(prefix="agent-env-build-") as staged: context = Path(staged) - for key, path in build_context_files(plan.bundle.bundle, image.entry).items(): + for key, path in build_context_files(plan.bundle_of(image.entry), image.entry).items(): (context / key).parent.mkdir(parents=True, exist_ok=True) shutil.copy2(path, context / key) yield context -def _write_build_context(plan: Plan, write: Write) -> int: +def _write_build_context(plan: Plan, write: Write, here: bool, on_build: Callable[[Write], None] | None) -> int: """Write the image an entry's Dockerfile describes as a docker_image artifact holding its build context alone, - which each provider that deploys it builds.""" + which each deploy builds where it runs, for its sandbox's platform; ``here``, built first with this machine's + Docker, for the deploys here.""" image = write.source with _staged(plan, image) as context: + if here: + sources, name = DockerImageArtifact.context_identity(write.id, str(context), str(context / image.dockerfile), + platform=None) + try: + build_staged(context, image.dockerfile, name, sources, _before(on_build, write)) + except DockerBuildError as e: + raise BundleError([f"{_path(plan, write)}: {_tail(str(e))}"]) from None try: return DockerImageArtifact.put_context( write.id, description=_image_description(plan, write), context_path=str(context), - dockerfile_path=str(context / image.dockerfile), + dockerfile_path=str(context / image.dockerfile), platform=None, ).version except (ValueError, RuntimeError) as e: # a context it can't write, or the object store raise BundleError([f"{_path(plan, write)}: {e}"]) from None +def _build_reused(plan: Plan, write: Write, version: int, on_build: Callable[[Write], None] | None) -> None: + """Build the image the bundle wrote earlier with this machine's Docker, when Docker doesn't hold it.""" + image = DockerImageArtifact.get(write.id, version) + if not image.context_only: + return + try: + build(image, _before(on_build, write)) + except DockerBuildError as e: + raise BundleError([f"{_path(plan, write)}: {_tail(str(e))}"]) from None + + +def _before(on_build: Callable[[Write], None] | None, write: Write) -> Callable[[], None] | None: + return None if on_build is None else lambda: on_build(write) + + +_BUILD_OUTPUT_TAIL_LINES = 40 + + +def _tail(message: str) -> str: + """A failed build's message, its first line and the end of docker's output.""" + first, _, output = message.partition("\n") + return "\n".join([first, *output.splitlines()[-_BUILD_OUTPUT_TAIL_LINES:]]) + + def _write_referenced_image(plan: Plan, write: Write) -> int: """Register the registry image an entry's image key or image folder names, its tag pinned to the digest the registry serves now. Written again with the same ref, as for a new description, it keeps the digest it has.""" @@ -196,25 +225,6 @@ def _latest_image_name(image_id: str) -> str | None: return latest.image_name if isinstance(latest, DockerImageArtifact) else None -def _write_built_image(plan: Plan, write: Write) -> int: - """Build the image an entry's Dockerfile describes and write it as a docker_image artifact: pushed to the - image store, saved as a tarball, and its build context kept for installing it into a running container.""" - image = write.source - tag = f"{image_repository(write.id)}:bundle" - with _staged(plan, image) as context: - try: - build_image(context / image.dockerfile, context, tag, platform=None) - except DockerBuildError as e: - raise BundleError([f"{_path(plan, write)}: {_tail(str(e))}"]) from None - try: - return DockerImageArtifact.put( - id=write.id, description=_image_description(plan, write), image_name=tag, - build_context_path=str(context), dockerfile_path=str(context / image.dockerfile), - ).version - except RuntimeError as e: # the image store, docker push or docker save - raise BundleError([f"{_path(plan, write)}: {e}"]) from None - - def _image_description(plan: Plan, write: Write) -> str: """An image folder's description, else where the image comes from.""" image = write.source @@ -225,31 +235,22 @@ def _image_description(plan: Plan, write: Write) -> str: return f"referenced by {_path(plan, write)}" -_BUILD_OUTPUT_TAIL_LINES = 40 - - -def _tail(message: str) -> str: - """A failed build's message, its first line and the end of docker's output.""" - first, _, output = message.partition("\n") - return "\n".join([first, *output.splitlines()[-_BUILD_OUTPUT_TAIL_LINES:]]) - - def _write_artifact(plan: Plan, write: Write) -> int: entry = write.source.entry cls = get_artifact_registry()[canonical_type(entry.type)] - return cls.from_toml(_pinned(plan, write, cls.toml_refs), AuthoringContext(plan.bundle.bundle, entry)).version + return cls.from_toml(_pinned(plan, write, cls.toml_refs), AuthoringContext(plan.bundle_of(entry), entry)).version def _write_agent(plan: Plan, write: Write) -> int: entry = write.source.entry return A2AAgent.from_toml(_pinned(plan, write, A2AAgent.toml_refs), - AuthoringContext(plan.bundle.bundle, entry)).version + AuthoringContext(plan.bundle_of(entry), entry)).version def _write_env(plan: Plan, write: Write) -> int: entry = write.source.entry cls = get_env_registry()[entry.type] - return cls.from_toml(_pinned(plan, write, cls.toml_refs), AuthoringContext(plan.bundle.bundle, entry)).version + return cls.from_toml(_pinned(plan, write, cls.toml_refs), AuthoringContext(plan.bundle_of(entry), entry)).version def _pinned(plan: Plan, write: Write, refs: tuple[EntityRef, ...]) -> Any: @@ -290,19 +291,6 @@ def _refuse_unwritable(plan: Plan) -> None: raise BundleError(problems) -def _refuse_builds_without_docker(plan: Plan, ledger: Ledger, contexts: frozenset[str]) -> None: - """An image the ledger will reuse, or one written as its build context, needs no docker, so only the ones it - would build are refused.""" - if shutil.which("docker") is not None: - return - problems = [f"{_path(plan, write)}: building its image from {write.source.dockerfile} needs docker, and it isn't " - "on PATH" for write in plan.writes - if isinstance(write.source, BuiltImage) and write.id not in contexts - and not ledger.check(write, {}).unchanged] - if problems: - raise BundleError(problems) - - def _unwritable(write: Write) -> str | None: """What ``write`` would write, when this release has no writer for it.""" if isinstance(write.source, BundleImage) or write.kind is BundleKind.TASK: @@ -361,4 +349,4 @@ def _key(write: Write) -> tuple[str, str]: def _path(plan: Plan, write: Write) -> str: - return relative(plan.bundle.bundle.root, write.source.entry.path) + return plan.path(write.source.entry) diff --git a/src/agent_env/bundle/plan.py b/src/agent_env/bundle/plan.py index 3adaf384..045d3363 100644 --- a/src/agent_env/bundle/plan.py +++ b/src/agent_env/bundle/plan.py @@ -5,23 +5,28 @@ before a write is reported together: an id two writers claim, a reference loop, a store id that doesn't exist or has the wrong type, and a task whose steps don't build. It reads stores but never writes to one. + +A reference by id to an env, agent or artifact an installed bundle defines, and a step's fallback to the configured +default agent when an installed bundle defines it, pulls that entity into the plan with what it references in its +bundle, so the run checks and writes it as it does its own. Such an id has no fixed version, and one the installed +bundle doesn't define is refused rather than read from the store. """ from __future__ import annotations import heapq import unicodedata -from collections.abc import Mapping, Sequence -from dataclasses import dataclass +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field, replace from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any from agent_env.a2a_agent import A2AAgent from agent_env.artifact import Artifact, CliArtifact, EnvironmentArtifact, SeedsEnvironments from agent_env.artifact.artifact import FolderChild from agent_env.artifact.registry import canonical_type, get_artifact_registry from agent_env.config import get_config -from agent_env.entity.refs import EntityKind +from agent_env.entity.refs import EntityKind, default_agent_refs from agent_env.env.env import Env from agent_env.env.envs._deployment import provider_refusal from agent_env.env.envs.mcp_server import MCPServerEnv @@ -46,10 +51,17 @@ from ._fs import fold, relative, with_article from .authoring import AuthoringContext, entry_files -from .parse import Bundle, BundleEntry, BundleError, BundleKind -from .resolve import BuiltImage, BundleImage, Reference, ResolvedBundle, ResolvedEntry, build_step +from .parse import NAMED_BY, Bundle, BundleEntry, BundleError, BundleKind, parse_bundle +from .resolve import BuiltImage, BundleImage, Reference, ResolvedBundle, ResolvedEntry, build_step, resolve_bundle + +if TYPE_CHECKING: + from .installed import InstalledBundle _Key = tuple[str, str] # (store, id) +# What lists the installed bundles, installed_bundles, which imports this module. +Installed = Callable[[], "tuple[InstalledBundle, ...]"] +# The kinds a reference pulls in from an installed bundle. +_PULLED = (EntityKind.ENV, EntityKind.AGENT, EntityKind.ARTIFACT) STORE_GETTERS = {EntityKind.ENV: Env.get, EntityKind.AGENT: A2AAgent.get, EntityKind.ARTIFACT: Artifact.get, EntityKind.TASK: Task.get} @@ -84,51 +96,190 @@ class Plan: evals: tuple[ResolvedEntry, ...] store_refs: tuple[Reference, ...] # the store ids the writes reference, checked to exist store_latest: dict[tuple[EntityKind, str], int] # the version each unpinned store ref named when planned + origins: Mapping[BundleEntry, Bundle] = field(default_factory=dict) # each entry from another bundle, with it + + def bundle_of(self, entry: BundleEntry) -> Bundle: + """The bundle ``entry`` comes from: the installed one for an entry pulled in from one, else the run's.""" + return self.origins.get(entry, self.bundle.bundle) + def path(self, entry: BundleEntry) -> str: + """``entry``'s path, as messages name it: in the run's bundle, or after the name of the bundle it comes from + (``agentenv-framework/agents:agents/claude-code``).""" + return _shown_path(self.bundle_of(entry), entry, pulled=entry in self.origins) -def plan_bundle(resolved: ResolvedBundle, *, tasks: Sequence[str] = (), evals: Sequence[str] = ()) -> Plan: + +def plan_bundle(resolved: ResolvedBundle, *, tasks: Sequence[str] = (), evals: Sequence[str] = (), + installed: Installed | None = None) -> Plan: """Plan a run of ``tasks`` and ``evals``, each a name or id in this bundle. With neither, the - run is every eval or, in a bundle without evals, every task. Raises BundleError listing every - problem found.""" - return _Planner(resolved).plan(tasks, evals) + run is every eval or, in a bundle without evals, every task. ``installed`` lists the installed bundles whose + entities a reference pulls in (``installed_bundles``); without it, an id under one is a store id. Raises + BundleError listing every problem found.""" + return _Planner(resolved, installed).plan(tasks, evals) -def check_bundle(resolved: ResolvedBundle) -> None: +def check_bundle(resolved: ResolvedBundle, *, installed: Installed | None = None) -> None: """The checks ``plan_bundle`` makes without reading a store, over every task and eval in the bundle. Raises BundleError listing every problem found.""" - _Planner(resolved).check() + _Planner(resolved, installed).check() + + +@dataclass(frozen=True) +class _Pullable: + """An installed bundle, resolved: its entities by (kind, id), as a reference names one, each with its resolved + entry or, for an image folder, its image.""" + + resolved: ResolvedBundle + by_id: dict[tuple[EntityKind, str], BundleEntry] + by_entry: dict[BundleEntry, ResolvedEntry | BundleImage] class _Planner: - def __init__(self, resolved: ResolvedBundle): + def __init__(self, resolved: ResolvedBundle, installed: Installed | None = None): self.resolved = resolved self.problems: list[str] = [] self.nodes: dict[_Key, ResolvedEntry | BundleImage] = {} self.rank: dict[_Key, int] = {} self.edges: dict[_Key, list[_Key]] = {} + self.named = {fold(entry.entry.name): entry.entry for entry in resolved.entries} + self.identified = {entry.entry.id: entry.entry for entry in resolved.entries} + self.outputs = {(output.kind.value, output.id): (entry.entry, output) + for entry in resolved.entries for output in entry.outputs} + self.accepted: dict[_Key, dict] = {} # each toml a type checks, as it accepted it + self.answers: dict[tuple[_Key, str], tuple | BundleError | None] = {} # each artifact type's folder hooks + self.steps: dict[_Key, list[TaskStep]] = {} # each task's steps that build, as a write will build them + self.origins: dict[BundleEntry, Bundle] = {} # each entry from another bundle, with it + self.installed = installed + self.others: tuple[InstalledBundle, ...] | None = None # the installed bundles but this one, once listed + self.pullable: dict[str, _Pullable | str] = {} # each installed bundle read, by id root, or why it can't be + self.unpulled: dict[tuple[EntityKind, str, int | None], str] = {} # why an installed bundle's id isn't pulled + self.pulling: set[_Key] = set() + self.own = _by_id(resolved.bundle) # what a pulled entity's id names in this bundle images: dict[BundleEntry, list[BundleImage]] = {} for image in resolved.images: images.setdefault(image.entry, []).append(image) entries = {entry.entry: entry for entry in resolved.entries} + self.entries: list[ResolvedEntry] = [] # the bundle's own, each reference to another bundle's entity local for bundle_entry in resolved.bundle.entries: for image in images.get(bundle_entry, ()): self._add(_key(image), image, []) if (entry := entries.get(bundle_entry)) is None: # an image folder, which is its image continue - needs = dict.fromkeys(_key(ref.local) for ref in entry.references if ref.local is not None) - self._add(_key(entry.entry), entry, list(needs)) - self.named = {fold(entry.entry.name): entry.entry for entry in resolved.entries} - self.identified = {entry.entry.id: entry.entry for entry in resolved.entries} - self.outputs = {(output.kind.value, output.id): (entry.entry, output) - for entry in resolved.entries for output in entry.outputs} - self.accepted: dict[_Key, dict] = {} # each toml a type checks, as it accepted it - self.answers: dict[tuple[_Key, str], tuple | BundleError | None] = {} # each artifact type's folder hooks - self.steps: dict[_Key, list[TaskStep]] = {} # each task's steps that build, as a write will build them + entry = self._localized(entry) + self.entries.append(entry) + self._add(_key(entry.entry), entry, _needs(entry)) def _add(self, key: _Key, source: ResolvedEntry | BundleImage, needs: list[_Key]) -> None: """Record a write in bundle order, a built image just before the entry that builds it.""" self.nodes[key], self.rank[key], self.edges[key] = source, len(self.rank), needs + # Entities from installed bundles + + def _localized(self, entry: ResolvedEntry) -> ResolvedEntry: + """``entry`` with each reference to an installed bundle's entity pointing at it, pulled into the plan, as a + reference to the bundle's own entity points at that; for a task, with a reference for each step that deploys + the configured default agent when this bundle or an installed one defines it.""" + references = [ref if ref.local is not None else self._pulled(ref) for ref in entry.references] + if entry.entry.kind is BundleKind.TASK: + references += self._default_agents(entry) + if references == list(entry.references): + return entry + return replace(entry, references=tuple(references)) + + def _default_agents(self, task: ResolvedEntry) -> list[Reference]: + """A reference to the configured default agent for each step of ``task`` that leaves an agent field unset, + when this bundle or an installed one defines that agent. One in the store is preflight's to check.""" + registry = get_task_step_registry() + found = [] + for step in task.config: + cls = registry.get(step.get("type")) if isinstance(step.get("type"), str) else None + for field_ref in default_agent_refs(cls.entity_refs or () if cls is not None else (), step): + version = step.get(field_ref.version_field) if field_ref.version_field else None + ref = Reference(EntityKind.AGENT, get_config().get_default_a2a_agent_id(), version, None, + f"step {step.get('id')!r}: {field_ref.path} (the default agent)", None) + pulled = self._pulled(ref) + if pulled.local is not None or (ref.kind, ref.id, ref.version) in self.unpulled: + found.append(pulled) + return found + + def _pulled(self, ref: Reference) -> Reference: + """``ref``, a store reference, pointing at what its id names: this bundle's entity, for a reference a pulled + entity or a default makes, or an installed bundle's, pulled into the plan. One naming neither stays a store + reference, as does one that can't be pulled, whose problem the store ref check reports.""" + if ref.kind not in _PULLED: + return ref + if (own := self.own.get((ref.kind, ref.id))) is not None: + target, problem = own, _misnamed(own, ref, "this bundle's", "is defined in this bundle") + elif ref.id.startswith(LOCAL_PREFIX) and (installed := self._installed_for(ref.id)) is not None: + target, problem = self._target(installed, ref) + else: + return ref + if problem is not None: + self.unpulled[ref.kind, ref.id, ref.version] = problem + return ref + return replace(ref, local=target) + + def _installed_for(self, entity_id: str) -> InstalledBundle | None: + """The installed bundle, other than this one, whose ids ``entity_id`` is under.""" + if self.installed is None: + return None + if self.others is None: + self.others = tuple(bundle for bundle in self.installed() if bundle.id_root != self.resolved.bundle.id_root) + return next((bundle for bundle in self.others if entity_id.startswith(f"{bundle.id_root}/")), None) + + def _target(self, installed: InstalledBundle, ref: Reference) -> tuple[BundleEntry | None, str | None]: + """What ``ref`` names in ``installed``, pulled into the plan, or why it names nothing there.""" + name = installed.qualified + pullable = self._pullable(installed) + if isinstance(pullable, str): + return None, f"{ref.id!r} comes with the installed bundle {name}, which can't be read: {pullable}" + target = pullable.by_id.get((ref.kind, ref.id)) + if target is None: + other = next((found for (kind, id), found in pullable.by_id.items() if id == ref.id), None) + if other is not None: + return None, (f"{ref.id!r} is the installed bundle {name}'s {other.kind.store}, but this field takes " + f"{with_article(ref.kind.value)}") + return None, f"there is no {ref.kind.value} {ref.id!r} in the installed bundle {name}" + if problem := _misnamed(target, ref, f"the installed bundle {name}'s", f"comes with the installed bundle {name}"): + return None, problem + self._pull(pullable, target) + return target, None + + def _pullable(self, installed: InstalledBundle) -> _Pullable | str: + """``installed`` parsed and resolved, once, or the problem that stops it.""" + if installed.id_root not in self.pullable: + if installed.problem or installed.root is None: + self.pullable[installed.id_root] = installed.problem or "it has no folder" + else: + try: + resolved = resolve_bundle(parse_bundle(installed.root, id_root=installed.id_root)) + except BundleError as e: + self.pullable[installed.id_root] = e.summary + else: + sources: dict[BundleEntry, ResolvedEntry | BundleImage] = { + image.entry: image for image in resolved.images if image.entry.id == image.id} + sources.update({entry.entry: entry for entry in resolved.entries}) + self.pullable[installed.id_root] = _Pullable(resolved, _by_id(resolved.bundle), sources) + return self.pullable[installed.id_root] + + def _pull(self, pullable: _Pullable, target: BundleEntry | BundleImage) -> None: + """Add ``target``, an installed bundle's entity or image, to the plan as the bundle's own are added, with what + it references in its bundle and, through its store references, in other installed bundles.""" + key = _key(target) + if key in self.nodes or key in self.pulling: + return + source = target if isinstance(target, BundleImage) else pullable.by_entry[target] + self.origins[source.entry] = pullable.resolved.bundle + if isinstance(source, BundleImage): # an image key's, or an image folder, which is its image + self._add(key, source, []) + return + self.pulling.add(key) + for ref in source.references: + if ref.local is not None: + self._pull(pullable, ref.local) + source = self._localized(source) + self.pulling.discard(key) + self._add(key, source, _needs(source)) + def plan(self, tasks: Sequence[str], evals: Sequence[str]) -> Plan: self._check_writers() self._check_loops() @@ -145,12 +296,12 @@ def plan(self, tasks: Sequence[str], evals: Sequence[str]) -> Plan: if self.problems: raise BundleError(self.problems) return Plan(self.resolved, self._ordered(closure), tuple(selected_tasks), tuple(selected_evals), store_refs, - store_latest) + store_latest, dict(self.origins)) def check(self) -> None: self._check_writers() self._check_loops() - if not any(entry.entry.kind in (BundleKind.TASK, BundleKind.EVAL) for entry in self.resolved.entries): + if not any(entry.entry.kind in (BundleKind.TASK, BundleKind.EVAL) for entry in self.entries): self.problems.append(_NOTHING_TO_RUN) everything = set(self.nodes) self._check_store_refs(everything, stores=False) @@ -176,7 +327,7 @@ def _check_writers(self) -> None: claims.append((key, self._path(source.entry), self._path(source.entry))) for id, folder in self._written_beside(source): claims.append(((BundleKind.ARTIFACT.store, id), folder, f"the artifact written from {folder}")) - for entry in self.resolved.entries: + for entry in self.entries: for output in entry.outputs: claims.append(((output.kind.value, output.id), f"{self._path(entry.entry)}: step {output.step_id!r}", f"step {output.step_id!r} of {self._path(entry.entry)}")) @@ -221,7 +372,7 @@ def _answer(self, entry: BundleEntry, hook: str) -> tuple | BundleError | None: elif not issubclass(cls, _FOLDER_HOOKS[hook]): self.answers[key] = () else: - self.answers[key] = getattr(cls, hook)(AuthoringContext(self.resolved.bundle, entry)) + self.answers[key] = getattr(cls, hook)(self._ctx(entry)) except BundleError as e: self.answers[key] = e return self.answers[key] @@ -237,8 +388,8 @@ def _check_loops(self) -> None: def _selection(self, tasks: Sequence[str], evals: Sequence[str]) -> tuple[list[ResolvedEntry], list[ResolvedEntry]]: if tasks or evals: return self._select(BundleKind.TASK, tasks), self._select(BundleKind.EVAL, evals) - every_task = [entry for entry in self.resolved.entries if entry.entry.kind is BundleKind.TASK] - every_eval = [entry for entry in self.resolved.entries if entry.entry.kind is BundleKind.EVAL] + every_task = [entry for entry in self.entries if entry.entry.kind is BundleKind.TASK] + every_eval = [entry for entry in self.entries if entry.entry.kind is BundleKind.EVAL] if every_eval: return [], every_eval if not every_task: @@ -246,7 +397,7 @@ def _selection(self, tasks: Sequence[str], evals: Sequence[str]) -> tuple[list[R return every_task, [] def _select(self, kind: BundleKind, wanted: Sequence[str]) -> list[ResolvedEntry]: - candidates = [entry for entry in self.resolved.entries if entry.entry.kind is kind] + candidates = [entry for entry in self.entries if entry.entry.kind is kind] exact = {_nfc(text): entry for entry in candidates for text in (entry.entry.name, entry.entry.id)} near = {fold(text): entry for entry in candidates for text in (entry.entry.name, entry.entry.id)} selected: dict[_Key, ResolvedEntry] = {} @@ -290,7 +441,7 @@ def _check_store_refs( for ref in source.references: if ref.local is not None: continue - problem = self._bundle_ref_problem(ref) + problem = self.unpulled.get((ref.kind, ref.id, ref.version)) or self._bundle_ref_problem(ref) if problem is None and stores: problem = self._store_ref_problem(ref, read) if problem: @@ -359,9 +510,9 @@ def _check_files(self, closure: set[_Key]) -> None: accept = A2AAgent.accept_toml if agent else self._env_accept(source.entry) try: if accept is not None: - self.accepted[key] = accept(source.config, AuthoringContext(self.resolved.bundle, source.entry)) + self.accepted[key] = accept(source.config, self._ctx(source.entry)) except BundleError as e: - self.problems.extend(e.problems) + self.problems.extend(self._named(source.entry, e.problems)) continue if source.entry.kind not in (BundleKind.ARTIFACT, BundleKind.SKILL): continue @@ -369,10 +520,10 @@ def _check_files(self, closure: set[_Key]) -> None: if not artifact_writer(cls): continue try: - folder_files(self.resolved.bundle, source.entry) + folder_files(self._bundle_of(source.entry), source.entry) except BundleError as e: - self.problems.extend(e.problems) - ctx = AuthoringContext(self.resolved.bundle, source.entry) + self.problems.extend(self._named(source.entry, e.problems)) + ctx = self._ctx(source.entry) accept = getattr(cls, "accept_toml", None) try: if accept is None: @@ -380,11 +531,12 @@ def _check_files(self, closure: set[_Key]) -> None: else: self.accepted[key] = accept(source.config, ctx) except BundleError as e: - self.problems.extend(e.problems) + self.problems.extend(self._named(source.entry, e.problems)) for hook in _FOLDER_HOOKS: if isinstance(refused := self._answer(source.entry, hook), BundleError): # What only the hook refuses, less what the type's accept_toml already reported. - self.problems.extend([problem for problem in refused.problems if problem not in self.problems]) + named = self._named(source.entry, refused.problems) + self.problems.extend([problem for problem in named if problem not in self.problems]) def _check_cli_loads(self, closure: set[_Key]) -> None: """Each load of one of the bundle's CLIs names what installing it needs, an agent and an env whose gateway it @@ -641,8 +793,21 @@ def _ordered(self, closure: set[_Key]) -> tuple[Write, ...]: heapq.heappush(ready, (self.rank[dependent], dependent)) return tuple(writes) + def _bundle_of(self, entry: BundleEntry) -> Bundle: + return self.origins.get(entry, self.resolved.bundle) + + def _ctx(self, entry: BundleEntry) -> AuthoringContext: + return AuthoringContext(self._bundle_of(entry), entry) + def _path(self, entry: BundleEntry) -> str: - return relative(self.resolved.bundle.root, entry.path) + return _shown_path(self._bundle_of(entry), entry, pulled=entry in self.origins) + + def _named(self, entry: BundleEntry, problems: Sequence[str]) -> list[str]: + """``problems``, which name paths in ``entry``'s bundle, as the plan names them: after the name of the bundle + an entry pulled in comes from.""" + if entry not in self.origins: + return list(problems) + return [f"{_bundle_name(self.origins[entry])}:{problem}" for problem in problems] def unpinned_store_refs(plan: Plan, write: Write) -> dict[tuple[EntityKind, str], int]: @@ -673,6 +838,35 @@ def folder_files(bundle: Bundle, entry: BundleEntry) -> Mapping[str, Path]: return entry_files(bundle, entry, empty_ok=True) +def _needs(entry: ResolvedEntry) -> list[_Key]: + """The writes ``entry`` references, each once.""" + return list(dict.fromkeys(_key(ref.local) for ref in entry.references if ref.local is not None)) + + +def _by_id(bundle: Bundle) -> dict[tuple[EntityKind, str], BundleEntry]: + """``bundle``'s envs, agents and artifacts by (kind, id), image folders included, as a reference names one.""" + return {(kind, entry.id): entry for entry in bundle.entries for kind in _PULLED if entry.kind in NAMED_BY[kind]} + + +def _misnamed(target: BundleEntry, ref: Reference, whose: str, comes: str) -> str | None: + """Why ``ref`` can't name ``target``, an entity a run writes: a pin, or a type its field doesn't take.""" + if ref.version is not None: + return f"{ref.id!r} {comes}, so it has no fixed version; drop the pin" + if ref.artifact_type and canonical_type(target.type) != canonical_type(ref.artifact_type): + return f"{ref.id!r} is {whose} {target.type}, but this field takes {ref.artifact_type}" + return None + + +def _shown_path(bundle: Bundle, entry: BundleEntry, *, pulled: bool) -> str: + where = relative(bundle.root, entry.path) + return f"{_bundle_name(bundle)}:{where}" if pulled else where + + +def _bundle_name(bundle: Bundle) -> str: + """An installed bundle's name as messages give it, ``/``: its id root, less the namespace.""" + return bundle.id_root.removeprefix(LOCAL_PREFIX) + + def _within(where: str, path: str) -> str: """Where ``path`` in the folder at ``where`` is, ``""`` being the folder itself.""" return f"{where}/{path}" if path else where diff --git a/src/agent_env/bundle/preflight.py b/src/agent_env/bundle/preflight.py index 13754bac..a4384585 100644 --- a/src/agent_env/bundle/preflight.py +++ b/src/agent_env/bundle/preflight.py @@ -8,13 +8,13 @@ would pull from this machine's registry; a VM asked of a provider that can't create one; or containers on the local provider without a Docker daemon. An image's tar.gz or build context that only this machine's object store holds isn't refused: agent-env pushes it to the sandbox. A deploy_agent step that -names no agent, and a judge that names none, deploy the configured default agent, which must be in the store. An env the -bundle writes is checked from its planned env.toml. +names no agent, and a judge that names none, deploy the configured default agent: one this bundle or an installed one +defines is planned as the bundle's own, and any other must be in the store. An env the bundle writes, or pulls in from an +installed bundle, is checked from its planned env.toml. -An image the bundle builds is written in the form its deploys can run: built on this machine when they deploy on the -local provider, and as its build context alone when they deploy on providers that build it, in a VM or, as Modal does, -themselves. One version is one of the two, so a run deploying it both ways is refused, as is one deploying it on a -provider that runs an image by name. +An image the bundle builds is written as its build context alone, whichever way its deploys run, and each deploy builds +it where it runs: in a VM, or as Modal does, itself, or for a sandbox on this machine with this machine's Docker, which +the run does before its tasks start. A deploy on a provider elsewhere that runs an image by name is refused. A gateway deploy runs on infra envs the store may not hold yet: the gateway, the service-db its local Postgres state runs from, and the website browser it adds for websites. In local stores the run builds those once its writes are done @@ -73,7 +73,6 @@ from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep from agent_env.utils.docker_build import DEFAULT_BUILD_PLATFORM, docker_unreachable -from ._fs import relative from .parse import BundleError, BundleKind from .plan import Plan from .resolve import BuiltImage, Reference, ReferencedImage, ResolvedEntry, build_step @@ -87,7 +86,7 @@ class Preflight: infra_kinds: frozenset[str] # the infra its deploys on the local provider run on infra: tuple[InfraBuild, ...] # those the store doesn't hold yet, or holds built from other inputs - contexts: frozenset[str] = frozenset() # the ids of the images the bundle builds that it writes as build contexts + built_here: frozenset[str] = frozenset() # the ids of the images the bundle builds that a deploy here runs def preflight_run(plan: Plan, tasks: Iterable[ResolvedEntry], sandbox: str | None) -> Preflight: @@ -140,7 +139,7 @@ def task(self, task: ResolvedEntry) -> None: steps = [build_step(config) for config in task.config] sandboxes = {step.sandbox_name: step for step in steps if isinstance(step, DeploySandboxTaskStep)} for step in steps: - where = f"{relative(self.plan.bundle.bundle.root, task.entry.path)}: step {step.id!r}" + where = f"{self.plan.path(task.entry)}: step {step.id!r}" try: if isinstance(step, DeployEnvTaskStep): self._env(where, step) @@ -161,7 +160,7 @@ def finish(self) -> Preflight: infra = infra_to_build(self.infra) except InfraError as e: self.run_problems.extend(e.problems) - contexts = self._contexts() + built_here = self._built_here() self._default_agent() reason = docker_unreachable() if self.docker_users or infra else None if reason and self.docker_users: @@ -178,7 +177,7 @@ def finish(self) -> Preflight: problems.append(f"{first}{more}: {problem}") if problems: raise BundleError(problems) - return Preflight(frozenset(self.infra), tuple(infra), contexts) + return Preflight(frozenset(self.infra), tuple(infra), built_here) # Deploys, each resolving its provider as its step does when it runs @@ -356,7 +355,7 @@ def _check(self, where: str, provider: SandboxProvider, accepts: Callable[[Sandb """The providers the deploy ``where`` may run on that take it, refusing the deploy, naming why each refuses (``_link_refusals``), when none does: a deploy skips a provider that refuses it, so one that takes it is enough. One on this machine that takes it runs it on this machine's Docker; one it skips needs none. An image the - bundle builds is noted with them, for ``_contexts`` to choose its form.""" + bundle builds is noted with them, for ``_built_here``.""" links = list(provider.links) refusals = [(link, self._link_refusals(link, accepts, images, vms_load=vms_load, elsewhere=elsewhere)) for link in links] @@ -426,37 +425,25 @@ def _remote_infra(self, link: SandboxProvider, kinds: set[str]) -> list[str]: image), vms_load=True))] return found - def _contexts(self) -> frozenset[str]: - """The images the bundle builds that it writes as build contexts: those every deploy runs on providers that - build them. One its deploys run only on the local provider, or that none runs, is built on this machine.""" - contexts = set() + def _built_here(self) -> frozenset[str]: + """The images the bundle builds that a deploy runs on this machine, which the run builds here before its tasks + start, with the Docker those deploys need. Any other deploy builds the image where it runs, a plugin step's + included, but not on a provider elsewhere that runs an image by name.""" + here = set() for image_id, uses in self.uses.items(): built = self.built[image_id] - source = f"{relative(self.plan.bundle.bundle.root, built.entry.path)}/{built.dockerfile}" - local, vms = [], [] + source = f"{self.plan.path(built.entry)}/{built.dockerfile}" for where, what, links, accepts in uses: for link in links: if link.ON_THIS_MACHINE: - local.append((where, what, links, link)) + here.add(image_id) elif accepts(link) is Accepts.NAME: self._problem(where, f"deploys {what} on the {_shown(link)} sandbox provider, which runs it by " f"name, and the bundle builds it from {source}, which a sandbox runs only " "once it's built on this machine or by the provider it runs on; run it " "with --sandbox local, or on a provider that builds it, such as --sandbox " "modal") - else: - vms.append((where, what, links, link)) - if local and vms: - (where, what, links, link), (elsewhere, _, _, vm) = local[0], vms[0] - on = (f"{_named(links)} sandbox provider" if elsewhere == where else - f"{_shown(link)} sandbox provider, and {elsewhere} on the {_shown(vm)} one") - self._problem(where, f"deploys {what} on the {on}: the bundle builds it from {source} on this machine " - "for the local provider, and as a build context for one that builds it, and a " - "run writes it one way; run its deploys on one kind, such as --sandbox local or " - "--sandbox modal_vm") - elif vms: - contexts.add(image_id) - return frozenset(contexts) + return frozenset(here) def _default_agent(self) -> None: if not self.default_agent_users: diff --git a/src/agent_env/bundle/run.py b/src/agent_env/bundle/run.py index 37dcc943..f28b08ce 100644 --- a/src/agent_env/bundle/run.py +++ b/src/agent_env/bundle/run.py @@ -32,9 +32,8 @@ from agent_env.task.task import recorded_scores from agent_env.task.teardown import TeardownReport, teardown_run from agent_env.task_step.context import TaskStepContext -from agent_env.utils.docker_build import DEFAULT_BUILD_PLATFORM -from ._fs import relative +from .installed import installed_bundles from .materialize import Materialization, Materialized, materialize from .parse import BundleEntry, BundleError, BundleKind, parse_bundle from .plan import Plan, Write, plan_bundle @@ -119,7 +118,7 @@ def failed(self) -> bool: def path(self, entry: BundleEntry) -> str: """``entry``'s path in the bundle, as the summary names it (``tasks/hello.json``).""" - return relative(self.materialization.plan.bundle.bundle.root, entry.path) + return self.materialization.plan.path(entry) def teardown(self) -> BundleRun: """Tear down the runs whose sandboxes were kept up, as ``run_bundle(keep=True)`` leaves them, and @@ -154,21 +153,23 @@ class DryRun: runs: tuple[BundleEntry, ...] # the tasks that would run, each once, in the plan's order skipped: tuple[BundleEntry, ...] # the tasks no eval names, which running every eval leaves out infra: tuple[InfraBuild, ...] = () # the infra envs the run would build first + built_here: frozenset[str] = frozenset() # the images the run would build on this machine, for its deploys here def path(self, entry: BundleEntry) -> str: """``entry``'s path in the bundle (``tasks/hello.json``).""" - return relative(self.materialization.plan.bundle.bundle.root, entry.path) + return self.materialization.plan.path(entry) def images(self) -> tuple[str, ...]: """A line per image the bundle writes, an image key's or an image folder's: which, the version it would leave, and how it would be written.""" - plan, contexts = self.materialization.plan, self.materialization.contexts + plan = self.materialization.plan return tuple( f"{_label(plan, done.write)} v{done.version}: " + ("pulled from its registry, its tag pinned to a digest when it's written" if isinstance(done.write.source, ReferencedImage) - else f"a build context, which each deploy builds where it runs, for {DEFAULT_BUILD_PLATFORM}" - if done.write.id in contexts else "built on this machine") + else "a build context, built with docker on this machine" + if done.write.id in self.built_here + else "a build context, which each deploy builds where it runs, for its sandbox's platform") for done in self.materialization.writes if isinstance(done.write.source, BundleImage) ) @@ -229,7 +230,7 @@ def run_bundle( on_wait=lambda: say("waiting for another agent-env run to finish writing this bundle's ids"), on_build=lambda write: say(f"{_label(plan, write)}: building with docker, which can take minutes"), on_write=lambda done: say(_written(plan, done)), - contexts=preflight.contexts, + build_here=preflight.built_here, ) entries = _to_run(plan) to_run = [(entry, Task.get(entry.id, materialization.version_of("task", entry.id))) for entry in entries] @@ -272,10 +273,10 @@ def dry_run_bundle( say = _progress(on_progress) with namespace_routing(): plan, preflight = _planned(root, tasks, evals, id_root, sandbox) - materialization = materialize(plan, dry_run=True, on_write=lambda done: say(_written(plan, done)), - contexts=preflight.contexts) + materialization = materialize(plan, dry_run=True, on_write=lambda done: say(_written(plan, done))) runs = _to_run(plan) - return DryRun(materialization, runs, _skipped(plan, runs, every=not tasks and not evals), preflight.infra) + return DryRun(materialization, runs, _skipped(plan, runs, every=not tasks and not evals), preflight.infra, + preflight.built_here) def _mark_cancelled(runs: tuple[TaskRun, ...], reason: str) -> None: @@ -301,7 +302,8 @@ def _bundle_run(plan: Plan, materialization: Materialization, runs: tuple[TaskRu def _planned(root: Path | str, tasks: Sequence[str], evals: Sequence[str], id_root: str | None, sandbox: str | None) -> tuple[Plan, Preflight]: - plan = plan_bundle(resolve_bundle(parse_bundle(Path(root), id_root=id_root)), tasks=tasks, evals=evals) + plan = plan_bundle(resolve_bundle(parse_bundle(Path(root), id_root=id_root)), tasks=tasks, evals=evals, + installed=installed_bundles) refuse_store_tasks(plan) to_run = set(_to_run(plan)) return plan, preflight_run(plan, [write.source for write in plan.writes @@ -327,7 +329,7 @@ def _skipped(plan: Plan, runs: Sequence[BundleEntry], every: bool) -> tuple[Bund def refuse_store_tasks(plan: Plan) -> None: """Refuse a selected eval that names a store task: a bundle's evals run only the bundle's own tasks.""" problems = [ - f"{_path(plan, entry.entry)}: {ref.where.partition('.')[0]}: {ref.id!r} is a store task, and a bundle's " + f"{plan.path(entry.entry)}: {ref.where.partition('.')[0]}: {ref.id!r} is a store task, and a bundle's " f"evals run only the bundle's own tasks for now; run it on its own with {_task_run_command(ref)}" for entry in plan.evals for ref in entry.references if ref.local is None ] @@ -362,7 +364,7 @@ async def settle(context: TaskStepContext, cancelled: bool) -> TeardownReport | return TeardownReport.skipped(context) async def run_one(entry: BundleEntry, task: Task) -> TaskRun: - tag = f"[{_path(plan, entry)}]" + tag = f"[{plan.path(entry)}]" metadata = {"run_group_id": run_group_id, "task_id": task.id} if overrides: metadata["user_overrides"] = dict(overrides) @@ -474,11 +476,7 @@ def _pinned(done: Materialized) -> str: def _label(plan: Plan, write: Write) -> str: """How a write is named: its entry's folder, and for an image built from it, which image.""" - path = _path(plan, write.source.entry) + path = plan.path(write.source.entry) if isinstance(write.source, BuiltImage): return f"{path} ({write.source.dockerfile} image)" return f"{path} ({write.source.ref} image)" if isinstance(write.source, ReferencedImage) else path - - -def _path(plan: Plan, entry: BundleEntry) -> str: - return relative(plan.bundle.bundle.root, entry.path) diff --git a/src/agent_env/entity/refs.py b/src/agent_env/entity/refs.py index 8f9b1f07..3e4568fc 100644 --- a/src/agent_env/entity/refs.py +++ b/src/agent_env/entity/refs.py @@ -19,6 +19,9 @@ class LoadArtifactTaskStep(TaskStep): Fields that name something else, such as another step's id, are left out. A step type's declarations are not inherited, so a subclass declares its own; ``toml_refs`` are, as ``from_toml`` is. + +A step's agent field left unset falls back to the configured default agent (``[agents] default_a2a_agent_id``), so a +tool reading a step knows which agent it may deploy without being told (``default_agent_refs``). """ from __future__ import annotations @@ -124,6 +127,13 @@ def rewrite(self, value: Any, version: int | None = None) -> None: self.owner[self.version_key] = version +def default_agent_refs(refs: Iterable[EntityRef], step: dict) -> list[EntityRef]: + """The agent fields ``step`` leaves unset, each falling back to the configured default agent: every top-level one it + doesn't set.""" + return [ref for ref in refs if ref.kind is EntityKind.AGENT and ref.role is RefRole.INPUT and ref.path == ref.field + and step.get(ref.field) is None] + + def parse_toml_ref(kind: EntityKind, value: Any) -> tuple[str, int | None]: """The id and version a toml ref names: an id, or ``{ = "", version = }`` with the version optional. Raises ValueError naming the expected form.""" diff --git a/src/agent_env/providers/sandbox_providers/host_images.py b/src/agent_env/providers/sandbox_providers/host_images.py new file mode 100644 index 00000000..0ce2f494 --- /dev/null +++ b/src/agent_env/providers/sandbox_providers/host_images.py @@ -0,0 +1,130 @@ +"""Images built from their build context on this machine, for the sandboxes that run here. + +An image that is only its build context is built where it runs. Here that is this machine's Docker: built natively, +under the image's name, labeled with the sources it was built from, and built again only when those change, so a later +deploy, a parallel one or another run finds it. A VM on this machine starts with no build cache, so it loads a saved +copy of that build, kept once per sources. Neither is a store version: the image is its build context. +""" + +from __future__ import annotations + +import asyncio +import gzip +import hashlib +import io +import logging +import platform as host +import subprocess +import tarfile +import tempfile +from collections.abc import Callable +from pathlib import Path + +from agent_env.artifact.artifacts.docker_image import DockerImageArtifact +from agent_env.config import get_config +from agent_env.store.local_state import holding_locks +from agent_env.utils.docker_build import build_image, normalize_platform + +logger = logging.getLogger(__name__) + +_SOURCES_LABEL = "dev.agentenv.sources" +_SAVED_PREFIX = "host-images" + + +def host_platform() -> str: + """The platform this machine's Docker builds natively.""" + return normalize_platform(f"linux/{host.machine()}") + + +def held(image: DockerImageArtifact) -> bool: + """Whether this machine's Docker holds ``image``, built from its sources.""" + return holds(image.image_name, sources(image)) + + +def holds(image_name: str, image_sources: str) -> bool: + """Whether this machine's Docker holds ``image_name`` built from ``image_sources``.""" + return _built_from(image_name) == image_sources + + +def built_here(image_name: str) -> bool: + """Whether this machine's Docker holds ``image_name`` built here from a build context, which no registry has.""" + return _built_from(image_name) is not None + + +def sources(image: DockerImageArtifact) -> str: + """What an image built from ``image``'s build context is labeled with.""" + return image.source_digest or image.build_context_object_url + + +def build(image: DockerImageArtifact, before: Callable[[], None] | None = None) -> None: + """Have this machine's Docker hold ``image``, built natively from its stored build context unless it holds one built + from the same sources, calling ``before`` first. Builds of one image wait for each other. Raises DockerBuildError.""" + with holding_locks([f"image-build:{sources(image)}"]): + if held(image): + return + if before is not None: + before() + with tempfile.TemporaryDirectory(prefix="agent-env-build-") as work: + context = Path(work) + data = get_config().get_object_store_at(image.build_context_object_url).get(image.build_context_object_url) + with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as archive: + archive.extractall(context, filter="data") + logger.info(f"Building {image.image_name} from its build context on this machine...") + _build(context, image.dockerfile_path, image.image_name, sources(image)) + + +def build_staged(context: Path, dockerfile: str, image_name: str, image_sources: str, + before: Callable[[], None] | None = None) -> None: + """``build`` from the build context at ``context``, before it's stored, as ``image_name`` from ``image_sources``, + calling ``before`` first. Raises DockerBuildError.""" + with holding_locks([f"image-build:{image_sources}"]): + if holds(image_name, image_sources): + return + if before is not None: + before() + _build(context, dockerfile, image_name, image_sources) + + +def _build(context: Path, dockerfile: str, image_name: str, image_sources: str) -> None: + build_image(context / dockerfile, context, image_name, platform=None, labels={_SOURCES_LABEL: image_sources}) + + +async def build_here(image: DockerImageArtifact) -> None: + """``build`` off the event loop.""" + await asyncio.to_thread(build, image) + + +async def saved_here(image: DockerImageArtifact) -> str: + """The object url of ``image`` built here and saved as a tar.gz, which a VM on this machine loads; saved once per + sources.""" + return await asyncio.to_thread(_saved_here, image) + + +def _built_from(image_name: str) -> str | None: + """The sources this machine's image ``image_name`` was built from, or None when there is none.""" + try: + result = subprocess.run(["docker", "image", "inspect", "--format", + f'{{{{ index .Config.Labels "{_SOURCES_LABEL}" }}}}', image_name], + capture_output=True, text=True, errors="replace") + except FileNotFoundError: # no docker on PATH, so no image here; a build says so + return None + return (result.stdout.strip() or None) if result.returncode == 0 else None + + +def _saved_here(image: DockerImageArtifact) -> str: + build(image) + objects = get_config().get_object_store() + # Per name too: images of equal sources under other names share a build, but a save loads under its own name. + name = hashlib.sha256(image.image_name.encode()).hexdigest()[:16] + key = f"{_SAVED_PREFIX}/{sources(image).removeprefix('sha256:')}-{host_platform().replace('/', '-')}-{name}.tar.gz" + with holding_locks([f"image-save:{key}"]): + if objects.exists(key): + return objects.object_url(key) + with tempfile.NamedTemporaryFile(suffix=".tar.gz") as saved: + save = subprocess.Popen(["docker", "save", image.image_name], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + with gzip.open(saved.name, "wb", compresslevel=1) as out: + while chunk := save.stdout.read(1 << 20): + out.write(chunk) + if save.wait() != 0: + raise RuntimeError(f"docker save {image.image_name} failed: {save.stderr.read().decode(errors='replace')}") + return objects.put_file(key, saved.name, "application/gzip") diff --git a/src/agent_env/providers/sandbox_providers/local_sandbox.py b/src/agent_env/providers/sandbox_providers/local_sandbox.py index 6523ed6d..de46188f 100644 --- a/src/agent_env/providers/sandbox_providers/local_sandbox.py +++ b/src/agent_env/providers/sandbox_providers/local_sandbox.py @@ -38,6 +38,7 @@ from agent_env import config from agent_env.attribution import Attribution from agent_env.config.paths import state_root +from agent_env.providers.sandbox_providers.host_images import build_here, built_here from agent_env.providers.sandbox_providers.sandbox import SANDBOX_LABEL, NetworkPolicy, VmSandbox from agent_env.providers.sandbox_providers.sandbox_provider import ( SANDBOX_MODE_CONTAINER, @@ -205,6 +206,21 @@ def __init__(self, exposed_ports: list[int] | None = None, work_dir: Path | None tempfile.mkdtemp(prefix=f"agent-env-{self.sandbox_id}-", dir=_local_sandbox_root()) ) + async def build_images(self, artifacts: list) -> None: + """Build each on this machine's Docker, which runs this sandbox's containers: natively, once per sources. One + given a platform of its own, as a remote VM records, builds as a VM does, for that platform.""" + if self.platform is not None: + await super().build_images(artifacts) + return + for artifact in artifacts: + await build_here(artifact) + + async def pull_images(self, image_names: list[str]) -> None: + """Pull each image but one this machine built from a build context, which no registry holds.""" + built = await asyncio.gather(*(asyncio.to_thread(built_here, name) for name in image_names)) + if pulled := [name for name, here in zip(image_names, built) if not here]: + await super().pull_images(pulled) + def host_port(self, port: int) -> int: """The allocated host port for a published container port (identity if unmapped).""" return self._port_map.get(port, port) @@ -545,8 +561,10 @@ class LocalSandboxProvider(SandboxProvider): EXTRA_CONTAINER_RUN_ARGS = " ".join(f"--add-host {entry}" for entry in _EXTRA_HOSTS) ON_THIS_MACHINE = True - # An agent's sandbox is a container from its image's name, though a VM it creates (create_vm) loads images. - SANDBOX_ACCEPTS = Accepts.NAME + # Its containers start from an image's name, or one it builds first from the image's build context (prepare_image); + # a VM it creates (create_vm) loads images. + SANDBOX_ACCEPTS = Accepts.NAME_OR_CONTEXT + CONTAINER_ACCEPTS = Accepts.NAME_OR_CONTEXT def url_from_sandbox(self, url: str) -> str: return self.get_external_url(url) @@ -638,6 +656,11 @@ async def _start_container(self, sandbox: VmSandbox, *, image_name: str, port: i await sandbox.exec_script(f"docker create {args} > /dev/null") await start_trusting(sandbox, sandbox.container_name, trust_dir) + async def prepare_image(self, image: Any, *, attribution: Optional[Attribution] = None) -> None: + """Build a context-only image on this machine, whose containers the local provider starts by name.""" + if image.context_only: + await build_here(image) + async def create_sandbox( self, *, diff --git a/src/agent_env/providers/sandbox_providers/sandbox.py b/src/agent_env/providers/sandbox_providers/sandbox.py index 81714474..422dd336 100644 --- a/src/agent_env/providers/sandbox_providers/sandbox.py +++ b/src/agent_env/providers/sandbox_providers/sandbox.py @@ -350,13 +350,14 @@ async def build_images(self, artifacts: list) -> None: async def _build_image(self, images: list) -> None: """Download the context ``images`` share, extract it, and ``docker build`` it, tagged with each image's name, - for the platform it records, or this VM's when its tar.gz is for another. The work folder is this build's own, - since a host can run builds of one context at once: two of the local provider's sandboxes, or two deploys onto - one VM.""" + for the platform it records, or this VM's when it records none or its tar.gz is for another. The work folder is + this build's own, since a host can run builds of one context at once: two of the local provider's sandboxes, or + two deploys onto one VM.""" first = images[0] names = list(dict.fromkeys(image.image_name for image in images)) work = f"/tmp/agent-env-build-{uuid.uuid4().hex}" - platform = self.platform if first.rebuilt_for(self.platform) else first.platform or DEFAULT_BUILD_PLATFORM + own = first.platform is None or first.rebuilt_for(self.platform) + platform = (self.platform if own else first.platform) or DEFAULT_BUILD_PLATFORM tags = " ".join(f"-t {shlex.quote(name)}" for name in names) logger.info(f"Building {', '.join(names)} from its build context, for {platform}...") await self.exec_script(f"rm -rf {work} && mkdir -p {work}/context") diff --git a/src/agent_env/providers/sandbox_providers/smol_vm/sandbox.py b/src/agent_env/providers/sandbox_providers/smol_vm/sandbox.py index 92afdba2..561aba7e 100644 --- a/src/agent_env/providers/sandbox_providers/smol_vm/sandbox.py +++ b/src/agent_env/providers/sandbox_providers/smol_vm/sandbox.py @@ -12,6 +12,7 @@ from agent_env.config import get_config from agent_env.providers.sandbox_providers.sandbox import CURL_RETRY_FLAGS, VmSandbox +from agent_env.providers.sandbox_providers.host_images import host_platform, saved_here from agent_env.providers.sandbox_providers.local_sandbox import LocalSandboxProvider from agent_env.store.object_store.local.store import LocalFilesystemObjectStore from agent_env.store.object_store.local.tls import local_ca @@ -45,6 +46,15 @@ async def detect_platform(self) -> None: raise RuntimeError(f"Cannot determine SmolVM guest architecture: {stderr[-500:]}") self.platform = normalize_platform(f"linux/{stdout.strip()}") + async def build_images(self, artifacts: list) -> None: + """Load each image this machine builds natively, from the copy it saves, since a VM here starts with no build + cache; one for a VM of another platform is built in the VM.""" + if self.platform != host_platform(): + await super().build_images(artifacts) + return + saved = [artifact.model_copy(update={"tar_gz_object_url": await saved_here(artifact)}) for artifact in artifacts] + await self._load_tarballs(saved) + async def terminate(self) -> None: await asyncio.to_thread(self._machine.delete) diff --git a/src/agent_env/task_step/task_steps/deploy_agent.py b/src/agent_env/task_step/task_steps/deploy_agent.py index 19113525..87696db1 100644 --- a/src/agent_env/task_step/task_steps/deploy_agent.py +++ b/src/agent_env/task_step/task_steps/deploy_agent.py @@ -29,7 +29,7 @@ from agent_env.env.env import DeployedEnv, DeployedSandboxEnv, Env from agent_env.env.gateway import AGENT_ENV_ROLE_HEADER from agent_env.providers.sandbox_providers.local_sandbox import host_url_for -from agent_env.a2a_agent.a2a_agent import DEFAULT_A2A_PORT +from agent_env.a2a_agent.a2a_agent import DEFAULT_A2A_PORT, unwritten_local_agent from agent_env.providers.sandbox_providers.sandbox_provider import ( SANDBOX_MODE_VM, reachable_url, @@ -42,6 +42,8 @@ from agent_env.task_step.task_step import TaskStep, TaskStepDependency from agent_env.task_step.task_steps.mcp_cli_builder.codegen import ROLE_ENV_VAR from agent_env.attribution import deploy_attribution +from agent_env.store.base import NotFoundError +from agent_env.store.ids import is_local_id logger = logging.getLogger(__name__) @@ -354,7 +356,12 @@ async def _execute(self, context: TaskStepContext, deployed_agents: list) -> Tas config = get_config() user_overrides = context.metadata.get("user_overrides", {}) a2a_agent_id, a2a_agent_version = self._resolve_agent_ref(context) - agent = A2AAgent.get(a2a_agent_id, a2a_agent_version) + try: + agent = A2AAgent.get(a2a_agent_id, a2a_agent_version) + except NotFoundError: + if not is_local_id(a2a_agent_id): + raise + raise NotFoundError(unwritten_local_agent(a2a_agent_id, "a2a_agent_id")) from None requested_disk = user_overrides.get("agent_disk_size_gb") or self.disk_size_gb min_disk = agent.metadata.get("min_disk_size_gb") agent_disk_size_gb = max(requested_disk or 0, min_disk or 0) or None diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index f1d8bd00..43427dca 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -15,6 +15,7 @@ fetch_trajectory, trajectory_mode, ) +from agent_env.a2a_agent.a2a_agent import unwritten_local_agent from agent_env.a2a_agent.staging import transfer_store from agent_env.config import get_config from agent_env.config.model import ModelParam @@ -35,6 +36,8 @@ ) from agent_env.entity.refs import EntityRef from agent_env.entity.scopes import FieldScope +from agent_env.store.base import NotFoundError +from agent_env.store.ids import is_local_id from agent_env.task_step.task_step import TaskStep, TaskStepDependency from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score from agent_env.task_step.task_steps.sandbox_utils.sandbox_utils import ( @@ -634,7 +637,12 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: ) env_vars = {"LITELLM_API_KEY": api_key, "LITELLM_BASE_URL": litellm_base_url} resolved_judge_id = self.judge_a2a_agent_id or config.get_default_a2a_agent_id() - a2a_agent = A2AAgent.get(resolved_judge_id) + try: + a2a_agent = A2AAgent.get(resolved_judge_id) + except NotFoundError: + if not is_local_id(resolved_judge_id): + raise + raise NotFoundError(unwritten_local_agent(resolved_judge_id, "judge_a2a_agent_id")) from None deploy_kwargs = {"env_vars": env_vars} resolved_sandbox_type = overrides.get("agent_sandbox") or self.judge_sandbox_type if resolved_sandbox_type: diff --git a/src/agent_env/utils/docker_build.py b/src/agent_env/utils/docker_build.py index aedeae8a..fe20916e 100644 --- a/src/agent_env/utils/docker_build.py +++ b/src/agent_env/utils/docker_build.py @@ -32,12 +32,14 @@ class DockerBuildError(RuntimeError): def build_image(dockerfile: Path, context: Path, tag: str, *, platform: str | None, - build_args: Mapping[str, str] | None = None) -> None: + build_args: Mapping[str, str] | None = None, labels: Mapping[str, str] | None = None) -> None: """Build ``context`` with ``dockerfile`` into the local image ``tag``, for ``platform``, or for this host's platform when it's None or empty. Raises DockerBuildError, with the build's output, when it fails.""" argv = ["docker", "build", *(["--platform", platform] if platform else [])] for name, value in (build_args or {}).items(): argv += ["--build-arg", f"{name}={value}"] + for name, value in (labels or {}).items(): + argv += ["--label", f"{name}={value}"] argv += ["-f", str(dockerfile), "-t", tag, str(context)] try: result = subprocess.run(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, errors="replace") diff --git a/tst/integration/cli/run_bundle_installed_agent_test.py b/tst/integration/cli/run_bundle_installed_agent_test.py new file mode 100644 index 00000000..8ae0f7d1 --- /dev/null +++ b/tst/integration/cli/run_bundle_installed_agent_test.py @@ -0,0 +1,110 @@ +"""``agent-env run`` on bundles using an agent an installed package's bundle ships, with a real docker build and the +agent deployed on the local sandbox: one bundle names it by id, another deploys it as the configured default, and the +second reuses the first's write of it. Needs a Docker daemon and the local registry.""" + +import importlib +import json +import logging +import shutil +import sys + +import pytest +from click.testing import CliRunner + +from agent_env.artifact.store import reset_artifact_store +from agent_env.bundle.installed import BUNDLES +from agent_env.cli import cli +from agent_env.config import configure, reset_config +from agent_env.store.object_store.local.grant_server import grant_server +from tst.util.a2a_test_agent import AGENT_DIR, PROTOCOL_DIR + +pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] + +AGENT = "@local/demo-kit/kit/echo" +BASE = (AGENT_DIR / "Dockerfile").read_text().splitlines()[0] # the echo agent's pinned base image +DOCKERFILE = f'''{BASE} +WORKDIR /app +COPY agentenv-protocol/ ./agentenv-protocol/ +RUN pip install --no-cache-dir './agentenv-protocol[agent]' +COPY agent.py ./ +CMD ["python", "agent.py"] +''' +NO_MODEL = {"LITELLM_API_KEY": "unused", "LITELLM_BASE_URL": "http://unused.invalid"} + + +def _task(**deploy): + return json.dumps([ + {"id": "deploy", "type": "deploy_agent", "env_ids": [], "sandbox_type": "local", "env_vars": NO_MODEL, + **deploy}, + {"id": "ask", "type": "prompt_agent", "prompt": "hello kit", "prompt_id": "ask", "depends_on": ["deploy"]}, + {"id": "check", "type": "agent_prompt_response_verifier", "prompt_id": "ask", "verifier_id": "reply", + "criteria": [{"type": "response_contains", "needles": ["Echo: hello kit"]}], "depends_on": ["ask"]}, + ]) + + +@pytest.fixture +def installed(tmp_path, monkeypatch): + """A distribution, demo-kit, on sys.path, registering the bundle ``kit`` with one agent folder, ``echo``.""" + site = tmp_path / "site" + folder = site / "demo_kit" / "kit" / "agents" / "echo" + folder.mkdir(parents=True) + (site / "demo_kit" / "__init__.py").write_text("") + (folder / "Dockerfile").write_text(DOCKERFILE) + shutil.copy(AGENT_DIR / "agent.py", folder / "agent.py") + shutil.copytree(PROTOCOL_DIR, folder / "agentenv-protocol", + ignore=shutil.ignore_patterns("__pycache__", "*.pyc", "tests", "examples", "*.egg-info")) + dist_info = site / "demo_kit-1.0.dist-info" + dist_info.mkdir() + (dist_info / "METADATA").write_text("Metadata-Version: 2.1\nName: demo-kit\nVersion: 1.0\n") + (dist_info / "entry_points.txt").write_text(f"[{BUNDLES}]\nkit = demo_kit\n") + (dist_info / "RECORD").write_text("") + monkeypatch.syspath_prepend(str(site)) + importlib.invalidate_caches() + yield folder + for name in [name for name in sys.modules if name.startswith("demo_kit")]: + del sys.modules[name] + + +@pytest.fixture +def state(monkeypatch, tmp_path): + """Local stores under this test's folder. Whatever ran, no sandbox work folder may be left behind.""" + sandboxes = tmp_path / "sandboxes" + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("XDG_STATE_HOME", str(tmp_path / "state")) + monkeypatch.setenv("AGENT_ENV_DOCUMENT_STORE", "local") + monkeypatch.setenv("AGENT_ENV_OBJECT_STORE", "local") + monkeypatch.setenv("AGENT_ENV_LOCAL_SANDBOX_DIR", str(sandboxes)) + configure(default_a2a_agent_id=AGENT) + reset_artifact_store() + logging.disable(logging.CRITICAL) # pytest's live logging would take CliRunner's stdout + try: + yield tmp_path + assert not sandboxes.exists() or not any(sandboxes.iterdir()), "a run left a sandbox work folder" + finally: + logging.disable(logging.NOTSET) + grant_server(None, None).close() + reset_artifact_store() + reset_config() + + +def _bundle(root, task): + (root / "tasks").mkdir(parents=True) + (root / "tasks/t.json").write_text(task) + return root + + +def test_bundles_naming_an_installed_agent_or_defaulting_to_it_build_it_once_and_run_it(state, installed): + by_id = CliRunner().invoke(cli, ["run", str(_bundle(state / "by-id", _task(a2a_agent_id=AGENT)))]) + + assert by_id.exit_code == 0, by_id.output + assert "demo-kit/kit:agents/echo (Dockerfile image): building with docker" in by_id.output, by_id.output + assert "demo-kit/kit:agents/echo: v1 (new)" in by_id.output, by_id.output + assert "tasks/t.json v1: passed" in by_id.output, by_id.output + + by_default = CliRunner().invoke(cli, ["run", str(_bundle(state / "by-default", _task()))]) + + assert by_default.exit_code == 0, by_default.output + assert "building with docker" not in by_default.output, by_default.output + assert "demo-kit/kit:agents/echo (Dockerfile image): v1, unchanged" in by_default.output, by_default.output + assert "demo-kit/kit:agents/echo: v1, unchanged" in by_default.output, by_default.output + assert "tasks/t.json v1: passed" in by_default.output, by_default.output diff --git a/tst/integration/env/envs/build_context_local_test.py b/tst/integration/env/envs/build_context_local_test.py index abd4cb35..ad2604e4 100644 --- a/tst/integration/env/envs/build_context_local_test.py +++ b/tst/integration/env/envs/build_context_local_test.py @@ -53,6 +53,20 @@ async def test_an_image_that_is_only_a_build_context_is_built_on_the_vm(local_st assert f"Building {image.image_name} from its build context, for {vm_platform}..." in caplog.messages +@pytest.mark.asyncio +async def test_a_build_context_recording_no_platform_is_built_for_the_vms(local_stack, vm_platform, tmp_path, caplog): + """As a bundle writes its images: a VM of either platform, such as an arm64 SmolVM, builds one for its own.""" + _seed_bootstrap_envs() + uid = uuid.uuid4().hex[:8] + image = DockerImageArtifact.put_context(f"ctx-items-{uid}", description="build context e2e", + context_path=str(_items_context(tmp_path / "items")), + dockerfile_path="Dockerfile", platform=None) + + await _deploys_and_answers(image, uid) + + assert f"Building {image.image_name} from its build context, for {vm_platform}..." in caplog.messages + + @pytest.mark.asyncio async def test_an_image_whose_tar_gz_is_for_another_platform_is_built_from_the_context_beside_it( local_stack, vm_platform, tmp_path, caplog): diff --git a/tst/unit/a2a_agent/context_only_deploy_test.py b/tst/unit/a2a_agent/context_only_deploy_test.py index 34fca12b..53490a27 100644 --- a/tst/unit/a2a_agent/context_only_deploy_test.py +++ b/tst/unit/a2a_agent/context_only_deploy_test.py @@ -1,13 +1,13 @@ -"""An agent whose image is only a build context deploys only where it's built, in a VM or by Modal: a provider that runs an -agent in a container by its image's name refuses it before creating anything, and Modal is told of it first. An image a -provider can't get otherwise, such as one a sandbox elsewhere would pull from this machine's registry, is refused before -anything is created too.""" +"""An agent whose image is only a build context deploys only where it's built, in a VM, by Modal or with this machine's +Docker: a provider that runs an agent in a container by its image's name refuses it before creating anything, and the +local provider and Modal are told of it first. An image a provider can't get otherwise, such as one a sandbox elsewhere +would pull from this machine's registry, is refused before anything is created too.""" import pytest from agent_env.a2a_agent.a2a_agent import A2AAgent from agent_env.artifact.artifacts.docker_image import DockerImageArtifact -from agent_env.providers.sandbox_providers import sandbox_provider +from agent_env.providers.sandbox_providers import local_sandbox, sandbox_provider from agent_env.providers.sandbox_providers.chained_sandbox_provider import ChainedSandboxProvider from agent_env.providers.sandbox_providers.local_sandbox import LocalSandboxProvider from agent_env.providers.sandbox_providers.modal_sandbox import ModalSandboxProvider @@ -18,11 +18,23 @@ class _Created(Exception): """Raised by a fake provider once asked for a sandbox, which is as far as a test needs to go.""" -class _LocalProvider(LocalSandboxProvider): +class _ByNameProvider(SandboxProvider): + """A plugin's provider that runs an agent's image by name, in a container of its own.""" + async def create_sandbox(self, **_): raise AssertionError("an image that is only a build context was run by name") +class _LocalProvider(LocalSandboxProvider): + def __init__(self, steps): + super().__init__() + self.steps = steps + + async def create_sandbox(self, **kwargs): + self.steps.append(("create", kwargs["image_name"])) + raise _Created + + class _VmProvider(SandboxProvider): SANDBOX_ACCEPTS = Accepts.LOADABLE @@ -60,15 +72,15 @@ def litellm(monkeypatch): monkeypatch.setenv("LITELLM_BASE_URL", "http://litellm.example/v1") -BY_NAME = ("with _LocalProvider, which runs an agent's image by name: 'solver-image' v2 is only a build context, which has " - "to be built, not pulled") +BY_NAME = ("with _ByNameProvider, which runs an agent's image by name: 'solver-image' v2 is only a build context, which " + "has to be built, not pulled") @pytest.mark.asyncio @pytest.mark.parametrize("provider, refusal", [ - (_LocalProvider(), BY_NAME), - (ChainedSandboxProvider([_LocalProvider(), _LocalProvider()]), f"{BY_NAME}; nor {BY_NAME}"), -], ids=["local", "chain-of-locals"]) + (_ByNameProvider(), BY_NAME), + (ChainedSandboxProvider([_ByNameProvider(), _ByNameProvider()]), f"{BY_NAME}; nor {BY_NAME}"), +], ids=["by-name", "chain-of-by-name"]) async def test_a_provider_that_runs_the_agent_by_name_refuses_it(monkeypatch, local_stores, litellm, provider, refusal): monkeypatch.setattr(sandbox_provider, "build_sandbox_provider", lambda name: provider) @@ -80,12 +92,28 @@ async def test_a_provider_that_runs_the_agent_by_name_refuses_it(monkeypatch, lo async def test_a_chain_skips_a_provider_that_runs_the_agent_by_name_for_one_that_builds_it(monkeypatch, local_stores, litellm, caplog): monkeypatch.setattr(sandbox_provider, "build_sandbox_provider", - lambda name: ChainedSandboxProvider([_LocalProvider(), _VmProvider()])) + lambda name: ChainedSandboxProvider([_ByNameProvider(), _VmProvider()])) with pytest.raises(RuntimeError, match=r"All 1 providers failed: _VmProvider: _Created\(\)"): await A2AAgent(id="solver", version=1, docker_image_artifact=IMAGE).deploy(sandbox_type="any") - assert ("sandbox_provider_skipped provider=_LocalProvider: which runs an agent's image by name: 'solver-image' v2 is " - "only a build context") in caplog.text + assert ("sandbox_provider_skipped provider=_ByNameProvider: which runs an agent's image by name: 'solver-image' v2 " + "is only a build context") in caplog.text + + +@pytest.mark.asyncio +async def test_the_local_provider_builds_the_image_with_this_machines_docker_before_it_runs_it(monkeypatch, local_stores, + litellm): + steps = [] + + async def build_here(image): + steps.append(("build", image.image_name)) + + monkeypatch.setattr(local_sandbox, "build_here", build_here) + monkeypatch.setattr(sandbox_provider, "build_sandbox_provider", lambda name: _LocalProvider(steps)) + + with pytest.raises(_Created): + await A2AAgent(id="solver", version=1, docker_image_artifact=IMAGE).deploy(sandbox_type="any") + assert steps == [("build", IMAGE.image_name), ("create", IMAGE.image_name)] @pytest.mark.asyncio diff --git a/tst/unit/artifact/test_docker_image_build_context.py b/tst/unit/artifact/test_docker_image_build_context.py index d6b8dc4d..98458069 100644 --- a/tst/unit/artifact/test_docker_image_build_context.py +++ b/tst/unit/artifact/test_docker_image_build_context.py @@ -41,9 +41,14 @@ def test_a_context_is_registered_with_no_image_and_a_name_that_names_no_registry stored = DockerImageArtifact.get("solver-image", image.version) assert (stored.tar_gz_object_url, stored.dockerfile_path, stored.platform) == (None, "Dockerfile", "linux/amd64") - assert stored.image_name.startswith("local/solver-image-") and stored.image_name.endswith(":v1") + sources = BuildContext.of(context, context / "Dockerfile").source_digest("linux/amd64") + assert stored.image_name.startswith("local/solver-image-") + assert stored.image_name.endswith(f":{sources.removeprefix('sha256:')[:16]}") assert not names_registry(stored.image_name) - assert stored.source_digest == BuildContext.of(context, context / "Dockerfile").source_digest("linux/amd64") + assert stored.source_digest == sources + again = DockerImageArtifact.put_context("solver-image", description="d", context_path=str(context), + dockerfile_path=str(context / "Dockerfile")) + assert (again.version, again.image_name) == (image.version + 1, stored.image_name) assert _names(stored) == [".dockerignore", "Dockerfile", "src", "src/app.py"] diff --git a/tst/unit/bundle/_support.py b/tst/unit/bundle/_support.py index 109060af..5596581a 100644 --- a/tst/unit/bundle/_support.py +++ b/tst/unit/bundle/_support.py @@ -5,7 +5,7 @@ from agent_env.bundle import parse_bundle from agent_env.bundle.plan import plan_bundle -from agent_env.bundle.resolve import resolve_bundle +from agent_env.bundle.resolve import BuiltImage, resolve_bundle from agent_env.config import get_config from agent_env.store.document_store import DocumentStore @@ -30,6 +30,11 @@ def plan_of(root): return plan_bundle(resolve_bundle(parse_bundle(root))) +def local_builds(plan): + """The images ``plan`` builds, which a run whose deploys all run on this machine builds here as it writes them.""" + return frozenset(write.id for write in plan.writes if isinstance(write.source, BuiltImage)) + + def local_store(): return get_config().local_namespace_document_store() diff --git a/tst/unit/bundle/conftest.py b/tst/unit/bundle/conftest.py index dccf4545..b1514c98 100644 --- a/tst/unit/bundle/conftest.py +++ b/tst/unit/bundle/conftest.py @@ -1,13 +1,11 @@ """Fixtures the bundle tests share.""" -from pathlib import Path import pytest from agent_env.artifact.artifacts import docker_image as docker_image_module -from agent_env.artifact.artifacts.docker_image import DockerImageArtifact -from agent_env.artifact.store import get_artifact_store from agent_env.bundle import materialize as materialize_module +from agent_env.providers.sandbox_providers import host_images from tst.unit.bundle._support import DIGEST, listing @@ -18,21 +16,17 @@ def docker_on_path(monkeypatch): @pytest.fixture def builds(monkeypatch, docker_on_path): - """Stands in for docker: each build is recorded with the files of its context, a link marked with a - trailing ``@``, and the image written as a docker_image document.""" - calls = [] + """Stands in for this machine's Docker: each build is recorded with its Dockerfile, the files of its context, a + link marked with a trailing ``@``, its tag and its platform, and Docker then holds the tag, labeled with its + sources.""" + calls, held = [], {} - def build(dockerfile, context, tag, *, platform): + def build(dockerfile, context, tag, *, platform, labels=None): calls.append({"build": (dockerfile.relative_to(context).as_posix(), listing(context), tag, platform)}) + held[tag] = (labels or {}).get(host_images._SOURCES_LABEL) - def put(id, *, description, image_name, build_context_path=None, dockerfile_path=None, platform=None): - dockerfile = Path(dockerfile_path).relative_to(build_context_path).as_posix() if dockerfile_path else None - calls[-1]["put"] = (id, image_name, listing(build_context_path), dockerfile) - return get_artifact_store().put_document(DockerImageArtifact( - id=id, description=description, image_name=image_name, tar_gz_s3_url=f"file:///{id}.tar.gz")) - - monkeypatch.setattr(materialize_module, "build_image", build) - monkeypatch.setattr(materialize_module.DockerImageArtifact, "put", put) + monkeypatch.setattr(host_images, "build_image", build) + monkeypatch.setattr(host_images, "_built_from", held.get) return calls diff --git a/tst/unit/bundle/env_toml_test.py b/tst/unit/bundle/env_toml_test.py index 3bc03ee1..5d9b881f 100644 --- a/tst/unit/bundle/env_toml_test.py +++ b/tst/unit/bundle/env_toml_test.py @@ -20,7 +20,7 @@ from agent_env.env.envs.website import WebsiteEnv from agent_env.providers.env_providers import env_provider as env_provider_module from agent_env.store.routing import namespace_routing -from tst.unit.bundle._support import RefusingStore, layout, local_store, plan_of +from tst.unit.bundle._support import RefusingStore, layout, local_builds, local_store, plan_of ROOT = "@local/~/triage" @@ -49,7 +49,8 @@ def _mcp(name, card=None): def _run(root, dry_run=False): - return materialize(plan_of(root), dry_run=dry_run) + plan = plan_of(root) + return materialize(plan, dry_run=dry_run, build_here=local_builds(plan)) def _summary(materialization): diff --git a/tst/unit/bundle/image_folder_test.py b/tst/unit/bundle/image_folder_test.py index 5b781a7f..8ade394b 100644 --- a/tst/unit/bundle/image_folder_test.py +++ b/tst/unit/bundle/image_folder_test.py @@ -12,9 +12,8 @@ from agent_env.artifact.store import get_artifact_store from agent_env.bundle.materialize import materialize from agent_env.env import Env -from agent_env.store.ids import image_repository from agent_env.store.routing import namespace_routing -from tst.unit.bundle._support import DIGEST, layout, plan_of +from tst.unit.bundle._support import DIGEST, layout, local_builds, plan_of ROOT = "@local/~/triage" IMAGE = f"{ROOT}/crm-image" @@ -43,7 +42,8 @@ def bundle_dir(tmp_path, monkeypatch, local_stores, cli_routing): def _run(root): - return materialize(plan_of(root)) + plan = plan_of(root) + return materialize(plan, build_here=local_builds(plan)) def _summary(materialization): @@ -63,7 +63,7 @@ def test_an_image_folder_is_built_once_and_everything_naming_it_is_written_over_ done = _run(bundle_dir) assert [call["build"][:3] for call in builds] == [ - ("Dockerfile", ["Dockerfile", "server.py"], f"{image_repository(IMAGE)}:bundle")] + ("Dockerfile", ["Dockerfile", "server.py"], DockerImageArtifact.get(IMAGE).image_name)] assert DockerImageArtifact.get(IMAGE).description == "built from artifacts/crm-image/Dockerfile" assert _images_named(done) == [(IMAGE, 1)] * 3 assert _summary(_run(bundle_dir))[IMAGE] == (1, True, ()) diff --git a/tst/unit/bundle/importer_test.py b/tst/unit/bundle/importer_test.py index 8811286a..33f1ead7 100644 --- a/tst/unit/bundle/importer_test.py +++ b/tst/unit/bundle/importer_test.py @@ -18,6 +18,7 @@ from agent_env.bundle import BundleError, parse_bundle from agent_env.bundle import importer as importer_module from agent_env.bundle.importer import ImportedEval, import_eval, import_task +from agent_env.bundle.installed import InstalledBundle from agent_env.bundle.materialize import materialize from agent_env.bundle.plan import plan_bundle from agent_env.bundle.resolve import resolve_bundle @@ -361,7 +362,7 @@ def test_a_folder_that_fails_its_check_is_not_moved_into_place_and_its_staging_i (store / "dest").mkdir() before = _left(store) - def refuse(resolved): + def refuse(resolved, **_): raise BundleError(["refused"]) monkeypatch.setattr(importer_module, "check_bundle", refuse) @@ -406,7 +407,7 @@ def test_a_destination_something_else_writes_during_the_import_is_left_as_it_wro if empty: dest.mkdir() - def land(resolved): + def land(resolved, **_): dest.mkdir(exist_ok=True) (dest / "theirs.txt").write_text("theirs") @@ -436,7 +437,7 @@ def test_a_folder_the_import_cant_write_in_is_refused_in_one_line(store): def test_sigterm_during_an_import_leaves_nothing_beside_its_destination(store, monkeypatch): before = _left(store) - monkeypatch.setattr(importer_module, "check_bundle", lambda resolved: os.kill(os.getpid(), signal.SIGTERM)) + monkeypatch.setattr(importer_module, "check_bundle", lambda resolved, **_: os.kill(os.getpid(), signal.SIGTERM)) result = CliRunner().invoke(cli, ["bundle", "init", str(store / "dest"), "--from-task", "triage"]) @@ -671,3 +672,22 @@ def test_the_cli_leaves_out_scoped_fields_unless_told_to_include_their_scope(sto "--include-scope user keeps it") in left.output.splitlines() assert "user scope" not in kept.output assert _steps(store / "kept", "scoped")[1]["litellm_base_url"] == "https://litellm.internal.example.com" + + +@pytest.mark.parametrize("pin", [None, 3]) +def test_an_installed_bundles_entity_floats_unpinned_so_the_import_runs_it_from_there(store, monkeypatch, pin): + kit = store / "site" / "kit" + (kit / "agents/solver").mkdir(parents=True) + (kit / "agents/solver/Dockerfile").write_text("FROM scratch\n") + monkeypatch.setattr(importer_module, "installed_bundles", + lambda: (InstalledBundle("kit", "demo-dist", "1.0", "demo", kit, None, None),)) + agent = "@local/demo-dist/kit/solver" + with namespace_routing(): + _task("@local/~/runs/t", [{"id": "agent", "type": "deploy_agent", "env_ids": [], "a2a_agent_id": agent, + "a2a_agent_version": pin}]) + imported = import_task("@local/~/runs/t", store / "imported", include_scopes=[FieldScope.USER]) + + assert [(ref.where, ref.id) for ref in imported.floating] == [("tasks/t.json: step 'agent': a2a_agent_id", agent)] + assert ("dropped" in imported.floating[0].why) == (pin is not None) and imported.pinned == () + assert _steps(imported.root, "t")[0]["a2a_agent_version"] is None + diff --git a/tst/unit/bundle/installed_entities_test.py b/tst/unit/bundle/installed_entities_test.py new file mode 100644 index 00000000..26af5b58 --- /dev/null +++ b/tst/unit/bundle/installed_entities_test.py @@ -0,0 +1,328 @@ +"""A bundle using an env, agent or artifact an installed bundle defines: by id, or as the configured default agent, +it's pulled into the plan as one of the bundle's own is, named after the bundle it comes from.""" + +import json +import subprocess +import sys +import textwrap +import time + +import pytest + +from agent_env.bundle import BundleError, BundleKind, dry_run_bundle, parse_bundle +from agent_env.bundle import preflight as preflight_module +from agent_env.bundle import run as run_module +from agent_env.bundle.installed import InstalledBundle +from agent_env.bundle.ledger import locked_ids +from agent_env.bundle.materialize import materialize +from agent_env.bundle.plan import check_bundle, plan_bundle +from agent_env.bundle.resolve import resolve_bundle +from tst.unit.bundle._support import RefusingStore, layout, local_builds + +ROOT = "@local/~/triage" +KIT = "@local/demo-dist/kit" +SOLVER = f"{KIT}/solver" +JUDGE = {"type": "rubrics_verifier", "prompt_id": "p", "verifier_id": "v", "criteria": [{"id": "c", "description": "ok"}]} +SERVERS = {"envs/crm/Dockerfile": "FROM scratch\n", "envs/crm/env.toml": 'environment_name = "crm"\n', + "envs/slack/Dockerfile": "FROM scratch\n", "envs/slack/env.toml": 'environment_name = "slack"\n', + "envs/suite/env.toml": 'type = "multi"\nmcp_server_envs = ["crm", "slack"]\n'} + + +@pytest.fixture +def bundle_dir(tmp_path, monkeypatch, local_stores, cli_routing): + monkeypatch.setenv("HOME", str(tmp_path)) + return tmp_path / "triage" + + +@pytest.fixture +def default_agent(monkeypatch): + def set_default(agent_id): + monkeypatch.setattr("agent_env.config.runtime.Config.get_default_a2a_agent_id", lambda self: agent_id) + + return set_default + + +def _installed(root, files, *, dist="demo-dist", name="kit", problem=None): + layout(root, files) + return InstalledBundle(name, dist, "1.0", "demo", None if problem else root, None, problem) + + +def _kit(tmp_path, files=None, **options): + return _installed(tmp_path / "site" / "kit", {"agents/solver/Dockerfile": "FROM scratch\n", **(files or {})}, + **options) + + +def _task(bundle_dir, steps, files=None): + layout(bundle_dir, {"tasks/t.json": json.dumps(steps), **(files or {})}) + return resolve_bundle(parse_bundle(bundle_dir)) + + +def _plan(resolved, *installed): + return plan_bundle(resolved, installed=lambda: installed) + + +def _problems(resolved, *installed): + with pytest.raises(BundleError) as caught: + _plan(resolved, *installed) + return caught.value.problems + + +def _writes(plan): + return [(write.kind, write.id) for write in plan.writes] + + +def test_an_installed_bundles_agent_named_by_id_is_written_by_the_run_after_its_image(bundle_dir, tmp_path): + kit = _kit(tmp_path) + plan = _plan(_task(bundle_dir, [{"id": "agent", "type": "deploy_agent", "a2a_agent_id": SOLVER}]), kit) + + assert _writes(plan) == [(BundleKind.ARTIFACT, f"{SOLVER}__agent_image"), (BundleKind.AGENT, SOLVER), + (BundleKind.TASK, f"{ROOT}/t")] + agent = plan.writes[1].source.entry + assert plan.writes[-1].needs == (("agent", SOLVER),) and plan.store_refs == () + assert (plan.bundle_of(agent).id_root, plan.path(agent)) == (KIT, "demo-dist/kit:agents/solver") + + +def test_without_the_installed_bundles_its_id_is_a_store_id(bundle_dir, tmp_path): + resolved = _task(bundle_dir, [{"id": "agent", "type": "deploy_agent", "a2a_agent_id": SOLVER}]) + + with pytest.raises(BundleError) as caught: + plan_bundle(resolved) + assert caught.value.problems == ( + f"tasks/t.json: step 'agent': a2a_agent_id: there is no agent {SOLVER!r} in the store",) + + +def test_a_step_falling_back_to_an_installed_default_agent_pulls_it_in(bundle_dir, tmp_path, default_agent): + default_agent(SOLVER) + plan = _plan(_task(bundle_dir, [ + {"id": "agent", "type": "deploy_agent"}, + {**JUDGE, "id": "judge"}, + {**JUDGE, "id": "llm", "use_agent_judge": False, "default_model": "m"}, + {**JUDGE, "id": "own", "agent_name": "default-agent"}, + ]), _kit(tmp_path)) + + assert _writes(plan) == [(BundleKind.ARTIFACT, f"{SOLVER}__agent_image"), (BundleKind.AGENT, SOLVER), + (BundleKind.TASK, f"{ROOT}/t")] + task = plan.writes[-1].source + assert [(ref.where, ref.local.id) for ref in task.references] == [ + ("step 'agent': a2a_agent_id (the default agent)", SOLVER), + ("step 'judge': judge_a2a_agent_id (the default agent)", SOLVER), + ("step 'llm': judge_a2a_agent_id (the default agent)", SOLVER), + ("step 'own': judge_a2a_agent_id (the default agent)", SOLVER)] + + +def test_a_default_agent_the_bundle_defines_is_written_by_the_run(bundle_dir, default_agent): + default_agent(f"{ROOT}/solver") + plan = _plan(_task(bundle_dir, [{"id": "agent", "type": "deploy_agent"}], + {"agents/solver/Dockerfile": "FROM scratch\n"})) + + assert plan.writes[-1].needs == (("agent", f"{ROOT}/solver"),) and plan.origins == {} + + +@pytest.mark.parametrize("default", [SOLVER, f"{ROOT}/solver"]) +def test_a_pin_on_a_default_agent_a_bundle_defines_is_refused(bundle_dir, tmp_path, default_agent, default): + default_agent(default) + resolved = _task(bundle_dir, [{"id": "agent", "type": "deploy_agent", "a2a_agent_version": 2}], + {"agents/solver/Dockerfile": "FROM scratch\n"}) + + [problem] = _problems(resolved, _kit(tmp_path)) + assert problem.startswith(f"tasks/t.json: step 'agent': a2a_agent_id (the default agent): {default!r} ") + assert "so it has no fixed version; drop" in problem + + +def test_a_default_agent_neither_bundle_defines_is_left_to_the_store(bundle_dir, tmp_path, default_agent): + default_agent("house-agent") + plan = _plan(_task(bundle_dir, [{"id": "agent", "type": "deploy_agent"}]), _kit(tmp_path)) + + assert _writes(plan) == [(BundleKind.TASK, f"{ROOT}/t")] and plan.writes[0].source.references == () + + +def test_an_installed_env_is_pulled_with_the_envs_it_names_and_its_artifacts_by_id(bundle_dir, tmp_path): + kit = _kit(tmp_path, {**SERVERS, "artifacts/world/crm/a.json": "{}"}) + plan = _plan(_task(bundle_dir, [ + {"id": "env", "type": "deploy_env", "env_id": f"{KIT}/suite"}, + {"id": "load", "type": "load_artifact", "env_id": f"{KIT}/suite", "artifact_id": f"{KIT}/world", + "depends_on": ["env"]}, + ]), kit) + + assert _writes(plan) == [ + (BundleKind.ARTIFACT, f"{KIT}/crm__env_image"), (BundleKind.ENV, f"{KIT}/crm"), + (BundleKind.ARTIFACT, f"{KIT}/slack__env_image"), (BundleKind.ENV, f"{KIT}/slack"), + (BundleKind.ENV, f"{KIT}/suite"), (BundleKind.ARTIFACT, f"{KIT}/world"), (BundleKind.TASK, f"{ROOT}/t")] + + +def test_a_pulled_env_is_checked_as_the_bundles_own_from_its_planned_config(bundle_dir, tmp_path): + kit = _kit(tmp_path, SERVERS) + resolved = _task(bundle_dir, [ + {"id": "env", "type": "deploy_env", "env_id": f"{KIT}/suite"}, + {"id": "load", "type": "load_artifact", "env_id": f"{KIT}/suite", "artifact_id": "world", "depends_on": ["env"]}, + ], {"artifacts/world/artifact.toml": 'type = "environment_universe"\n', "artifacts/world/crm/a.json": "{}", + "artifacts/world/slakc/a.json": "{}"}) + problem = ("tasks/t.json: step 'load': artifacts/world/slakc seeds an environment named 'slakc', and " + "demo-dist/kit:envs/suite has no server by that name ('crm', 'slack'); name it after the server it " + "seeds, or leave it out") + + assert _problems(resolved, kit) == (problem,) + + +def test_an_installed_bundle_naming_another_ones_entity_pulls_that_in_too(bundle_dir, tmp_path): + base = _installed(tmp_path / "site" / "base", {"artifacts/base/Dockerfile": "FROM scratch\n"}, + dist="base-dist", name="base") + kit = _kit(tmp_path, {"agents/coder/agent.toml": 'image = "@local/base-dist/base/base"\n'}) + plan = _plan(_task(bundle_dir, [{"id": "agent", "type": "deploy_agent", "a2a_agent_id": f"{KIT}/coder"}]), + kit, base) + + assert _writes(plan) == [(BundleKind.ARTIFACT, "@local/base-dist/base/base"), (BundleKind.AGENT, f"{KIT}/coder"), + (BundleKind.TASK, f"{ROOT}/t")] + assert plan.path(plan.writes[0].source.entry) == "base-dist/base:artifacts/base" + + +def test_a_pulled_entity_naming_an_image_folder_of_its_bundle_pulls_the_image(bundle_dir, tmp_path): + kit = _kit(tmp_path, {"artifacts/base/Dockerfile": "FROM scratch\n", "agents/coder/agent.toml": 'image = "base"\n'}) + plan = _plan(_task(bundle_dir, [{"id": "agent", "type": "deploy_agent", "a2a_agent_id": f"{KIT}/coder"}]), kit) + + assert _writes(plan) == [(BundleKind.ARTIFACT, f"{KIT}/base"), (BundleKind.AGENT, f"{KIT}/coder"), + (BundleKind.TASK, f"{ROOT}/t")] + + +def test_a_pulled_entity_naming_the_running_bundles_own_entity_uses_it(tmp_path, local_stores, cli_routing): + base = _installed(tmp_path / "site" / "base", { + "artifacts/base/Dockerfile": "FROM scratch\n", + "tasks/t.json": json.dumps([{"id": "agent", "type": "deploy_agent", "a2a_agent_id": f"{KIT}/coder"}])}, + dist="base-dist", name="base") + kit = _kit(tmp_path, {"agents/coder/agent.toml": 'image = "@local/base-dist/base/base"\n'}) + plan = _plan(resolve_bundle(parse_bundle(base.root, id_root=base.id_root)), kit, base) + + assert _writes(plan) == [(BundleKind.ARTIFACT, "@local/base-dist/base/base"), (BundleKind.AGENT, f"{KIT}/coder"), + (BundleKind.TASK, "@local/base-dist/base/t")] + assert [plan.path(write.source.entry) for write in plan.writes] == [ + "artifacts/base", "demo-dist/kit:agents/coder", "tasks/t.json"] + + +def test_the_bundles_own_entity_wins_over_an_installed_one_with_its_id(bundle_dir, tmp_path): + plan = _plan(_task(bundle_dir, [{"id": "agent", "type": "deploy_agent", "a2a_agent_id": SOLVER}], { + "agents/mine/agent.toml": f'id = "{SOLVER}"\n', "agents/mine/Dockerfile": "FROM scratch\n"}), _kit(tmp_path)) + + assert plan.origins == {} and plan.path(plan.writes[1].source.entry) == "agents/mine" + + +@pytest.mark.parametrize(("step", "problem"), [ + ({"a2a_agent_id": SOLVER, "a2a_agent_version": 2}, + f"{SOLVER!r} comes with the installed bundle demo-dist/kit, so it has no fixed version; drop the pin"), + ({"a2a_agent_id": f"{KIT}/nope"}, f"there is no agent '{KIT}/nope' in the installed bundle demo-dist/kit"), + ({"a2a_agent_id": f"{KIT}/crm"}, f"'{KIT}/crm' is the installed bundle demo-dist/kit's env, but this field takes " + "an agent"), +]) +def test_an_installed_id_that_names_nothing_it_can_be_is_refused_and_not_read_from_the_store(bundle_dir, tmp_path, + step, problem): + kit = _kit(tmp_path, SERVERS) + resolved = _task(bundle_dir, [{"id": "agent", "type": "deploy_agent", **step}]) + + assert _problems(resolved, kit) == (f"tasks/t.json: step 'agent': a2a_agent_id: {problem}",) + + +def test_an_installed_bundle_that_cant_be_read_is_one_problem_where_it_is_named(bundle_dir, tmp_path, default_agent): + default_agent(SOLVER) + broken = _kit(tmp_path, problem="its distribution's metadata can't be read") + unparsed = _kit(tmp_path, {"tasks/t.json": "["}, name="kit") + resolved = _task(bundle_dir, [{"id": "agent", "type": "deploy_agent"}]) + + assert _problems(resolved, broken) == ( + f"tasks/t.json: step 'agent': a2a_agent_id (the default agent): {SOLVER!r} comes with the installed bundle " + "demo-dist/kit, which can't be read: its distribution's metadata can't be read",) + [problem] = _problems(resolved, unparsed) + assert problem.startswith(f"tasks/t.json: step 'agent': a2a_agent_id (the default agent): {SOLVER!r} comes with " + "the installed bundle demo-dist/kit, which can't be read: tasks/t.json: ") + + +def test_a_pulled_entitys_own_problem_is_named_after_its_bundle(bundle_dir, tmp_path): + kit = _kit(tmp_path, {"agents/solver/agent.toml": "colour = 1\n"}) + [problem] = _problems(_task(bundle_dir, [{"id": "agent", "type": "deploy_agent", "a2a_agent_id": SOLVER}]), kit) + + assert problem.startswith("demo-dist/kit:agents/solver: agent.toml: unknown key 'colour'") + + +def test_checking_a_bundle_without_a_store_pulls_and_refuses_as_planning_does(bundle_dir, tmp_path, monkeypatch): + kit = _kit(tmp_path) + resolved = _task(bundle_dir, [{"id": "agent", "type": "deploy_agent", "a2a_agent_id": f"{KIT}/nope"}]) + monkeypatch.setattr("agent_env.config.runtime.Config.get_document_store", lambda self: RefusingStore()) + + with pytest.raises(BundleError) as caught: + check_bundle(resolved, installed=lambda: (kit,)) + assert caught.value.problems == ( + f"tasks/t.json: step 'agent': a2a_agent_id: there is no agent '{KIT}/nope' in the installed bundle " + "demo-dist/kit",) + check_bundle(_task(bundle_dir, [{"id": "agent", "type": "deploy_agent", "a2a_agent_id": SOLVER}]), + installed=lambda: (kit,)) + + +def test_a_dry_run_preflights_a_pulled_default_agent_and_names_its_image_after_its_bundle(bundle_dir, tmp_path, + default_agent, monkeypatch): + default_agent(SOLVER) + kit = _kit(tmp_path) + monkeypatch.setattr(run_module, "installed_bundles", lambda: (kit,)) + monkeypatch.setattr(preflight_module, "docker_unreachable", lambda: None) + _task(bundle_dir, [{"id": "agent", "type": "deploy_agent"}]) + said = [] + + dry = dry_run_bundle(bundle_dir, sandbox="local", on_progress=said.append) + + assert dry.images() == ("demo-dist/kit:agents/solver (Dockerfile image) v1: a build context, built with docker on " + "this machine",) + assert [dry.path(entry) for entry in dry.runs] == ["tasks/t.json"] + assert [line for line in said if "demo-dist/kit" in line] == [ + "demo-dist/kit:agents/solver (Dockerfile image): v1 (new)", "demo-dist/kit:agents/solver: v1 (new)"] + + +def _user(tmp_path, name): + return layout(tmp_path / name, {"tasks/t.json": json.dumps([ + {"id": "agent", "type": "deploy_agent", "a2a_agent_id": SOLVER}])}) + + +def _materialized(root, *installed, id_root=None): + plan = plan_bundle(resolve_bundle(parse_bundle(root, id_root=id_root)), installed=lambda: installed) + return materialize(plan, build_here=local_builds(plan)) + + +def _kits(materialization): + return {done.write.id: (done.version, done.reused, done.reasons) for done in materialization.writes + if done.write.id.startswith(f"{SOLVER}")} + + +def test_every_bundle_using_an_installed_agent_shares_one_write_of_it(bundle_dir, tmp_path, builds): + kit = _kit(tmp_path, {"tasks/smoke.json": json.dumps([{"id": "agent", "type": "deploy_agent", + "a2a_agent_id": "solver"}])}) + image = f"{SOLVER}__agent_image" + + assert _kits(_materialized(_user(tmp_path, "first"), kit)) == {image: (1, False, ("new",)), + SOLVER: (1, False, ("new",))} + assert _kits(_materialized(_user(tmp_path, "second"), kit)) == {image: (1, True, ()), SOLVER: (1, True, ())} + assert _kits(_materialized(kit.root, kit, id_root=KIT)) == {image: (1, True, ()), SOLVER: (1, True, ())} + + (kit.root / "agents/solver/Dockerfile").write_text("FROM scratch\nENV CHANGED=1\n") + assert _kits(_materialized(_user(tmp_path, "second"), kit)) == { + image: (2, False, ("files changed: Dockerfile",)), + SOLVER: (2, False, (f"artifact {image} is written anew (v1 → v2)",))} + + +def test_a_run_locks_the_entities_it_pulls_in_as_it_does_its_own(bundle_dir, tmp_path, builds): + plan = plan_bundle(resolve_bundle(parse_bundle(_user(tmp_path, "first"))), installed=lambda: (_kit(tmp_path),)) + held, done, waited = tmp_path / "held", tmp_path / "done", [] + holder = subprocess.Popen([sys.executable, "-c", textwrap.dedent(f""" + import pathlib, time + from agent_env.bundle.ledger import materializing + with materializing([{SOLVER!r}]): + pathlib.Path({str(held)!r}).write_text("held") + time.sleep(1.5) + pathlib.Path({str(done)!r}).write_text("done") + """)]) + try: + for _ in range(200): + if held.exists(): + break + time.sleep(0.05) + materialize(plan, on_wait=lambda: waited.append(done.exists())) + assert done.exists() + finally: + holder.wait() + + assert SOLVER in locked_ids(plan) and waited == [False] diff --git a/tst/unit/bundle/ledger_digest_test.py b/tst/unit/bundle/ledger_digest_test.py index a55c4ce7..0432b0dd 100644 --- a/tst/unit/bundle/ledger_digest_test.py +++ b/tst/unit/bundle/ledger_digest_test.py @@ -30,17 +30,17 @@ ]), } -# Recorded by agentenv-framework 0.9.1278. +# Recorded at SCHEME 2, which writes every image the bundle builds as its build context. RECORDED = { - "@local/~/triage/tickets__env_image": "sha256:b381bf1c841f024fe2dfeabb08e5a10cf2b3cc596563ba3e6ea5c6798af1c868", - "@local/~/triage/tickets": "sha256:724cfb95eed4f78c7f77dc1c94507277217860bcce6a773bc3156e77bc2f8521", - "@local/~/triage/solver__agent_image": "sha256:00fb2164bc6e1d8230ef6e2d437f640ba853acde7ab7c4f13485d3fb1b48d5d6", - "@local/~/triage/solver": "sha256:54bcabc92b285434ad23af91e2dcf1b04700d73a819016b1e62b66f7d0f3ec0e", - "@local/~/triage/greeting": "sha256:e0047a71d6e34c6f390f6b6bf416ff54765bbbc390b2bd7f34432ee79a691e66", - "@local/~/triage/note": "sha256:3e9de21964b7c4e78a240f8f7aaafb66a0f04a9a430b33b6e54b9869c9001a9d", - "@local/~/triage/shelf": "sha256:ab0dd7c0b9c41312caeb9e012a31e4b640d0c626ddf005a5d7607824f36aa674", - "@local/~/triage/t": "sha256:056ad6e1030f251c2a6f9e144270fd53d9053c5d4264e0654ba8e547b10abe7a", - "@local/~/triage/u": "sha256:f8d5445a95fed8bf4d03b713fa8f545b1a8da847db02c22a3806eec39e26db29", + "@local/~/triage/tickets__env_image": "sha256:c76d8bd3706e9785ff464cc56fd7cf0b2afd968003a1661568d36e2abefa70dd", + "@local/~/triage/tickets": "sha256:0732f87377c46dc5e0238adb4cadc705dc814fb89825536cdc83b28b086a9b33", + "@local/~/triage/solver__agent_image": "sha256:6d78d7ba2266a8144537f8493099b5b5b9440a20d4d5d806f9cddb39de443436", + "@local/~/triage/solver": "sha256:ef703cb708fba45b796f9413654cd0c8579ca4f6268cdedf177068619b1ad645", + "@local/~/triage/greeting": "sha256:386ee0e6beef93a08f3816336de4ae136ab76a6c2136159c430599728ca91c0e", + "@local/~/triage/note": "sha256:f427863a7d913875a1b87f10ea136e9bc50ec496bcaec277b11927f19ae3e6cd", + "@local/~/triage/shelf": "sha256:53359d8af25aec8090b42560071e31d8a15023be27148189f8376fedf3062955", + "@local/~/triage/t": "sha256:3477937456f6da7403b62c98f2e76d695349887aa8d17076542f21be277c141d", + "@local/~/triage/u": "sha256:b264dad989f8de065f0ea4026cab0dc67d8a51ad73621380bb9ed22ada0e51f2", } diff --git a/tst/unit/bundle/ledger_test.py b/tst/unit/bundle/ledger_test.py index de5acbda..510784c1 100644 --- a/tst/unit/bundle/ledger_test.py +++ b/tst/unit/bundle/ledger_test.py @@ -17,6 +17,7 @@ from agent_env.bundle import parse_bundle from agent_env.bundle import plan as plan_module from agent_env.bundle import resolve as resolve_module +from agent_env.bundle import ledger as ledger_module from agent_env.bundle.ledger import LEDGER_COLLECTION, Ledger, materializing from agent_env.bundle.materialize import _pinned from agent_env.config import configure, get_config @@ -294,7 +295,7 @@ def edited_meanwhile(): def test_a_new_digest_scheme_rewrites_everything_and_says_only_that(bundle_dir, monkeypatch): _run(bundle_dir) - monkeypatch.setattr("agent_env.bundle.ledger.SCHEME", 2) + monkeypatch.setattr("agent_env.bundle.ledger.SCHEME", ledger_module.SCHEME + 1) assert {check.reasons for check in _run(bundle_dir).values()} == {("the ledger's digest scheme changed",)} @@ -563,7 +564,7 @@ def test_a_second_run_writing_any_of_the_same_ids_waits_for_the_first_to_finish( import pathlib, time from agent_env.bundle import parse_bundle from agent_env.bundle.ledger import materializing - with materializing(parse_bundle(pathlib.Path({str(first)!r}), **{root!r})): + with materializing(entry.id for entry in parse_bundle(pathlib.Path({str(first)!r}), **{root!r}).entries): pathlib.Path({str(held)!r}).write_text("held") time.sleep(1.5) pathlib.Path({str(done)!r}).write_text("done") @@ -574,7 +575,8 @@ def test_a_second_run_writing_any_of_the_same_ids_waits_for_the_first_to_finish( break time.sleep(0.05) waited = [] - with materializing(parse_bundle(second, **root), on_wait=lambda: waited.append(True)): + with materializing((entry.id for entry in parse_bundle(second, **root).entries), + on_wait=lambda: waited.append(True)): assert done.exists() assert waited == [True] finally: diff --git a/tst/unit/bundle/materialize_test.py b/tst/unit/bundle/materialize_test.py index 6e5b6826..172bfdab 100644 --- a/tst/unit/bundle/materialize_test.py +++ b/tst/unit/bundle/materialize_test.py @@ -1,6 +1,5 @@ """Materializing a planned bundle: what gets written, what is reused, and what is refused before any write.""" -import contextlib import json import subprocess import sys @@ -31,12 +30,13 @@ from agent_env.env.env import Env from agent_env.eval import Eval, EvalTask from agent_env.store import Filter -from agent_env.store.ids import image_repository +from agent_env.providers.sandbox_providers import host_images +from agent_env.store.base import NotFoundError from agent_env.store.routing import disable_namespace_routing from agent_env.task import Task from agent_env.task_step.task_step import TaskStep from agent_env.utils.build_context import BuildContext -from tst.unit.bundle._support import DIGEST, RefusingStore, layout, local_store, plan_of +from tst.unit.bundle._support import DIGEST, RefusingStore, layout, local_builds, local_store, plan_of ROOT = "@local/~/triage" LAYOUT = { @@ -144,7 +144,8 @@ def _steps(root, steps): def _run(root, dry_run=False): - return materialize(plan_of(root), dry_run=dry_run) + plan = plan_of(root) + return materialize(plan, dry_run=dry_run, build_here=local_builds(plan)) def _summary(materialization): @@ -444,19 +445,20 @@ def test_an_agent_folder_with_a_dockerfile_is_built_and_the_agent_written_over_i _steps(bundle_dir, [{"id": "agent", "type": "deploy_agent", "env_ids": [], "a2a_agent_id": "solver"}]) image = f"{ROOT}/solver__agent_image" announced = [] + plan = plan_of(bundle_dir) - first = materialize(plan_of(bundle_dir), on_build=lambda write: announced.append(write.id)) + first = materialize(plan, on_build=lambda write: announced.append(write.id), build_here=local_builds(plan)) - tag = f"{image_repository(image)}:bundle" - assert builds == [{"build": ("Dockerfile", ["Dockerfile", "run.sh"], tag, None), - "put": (image, tag, ["Dockerfile", "run.sh"], "Dockerfile")}] + written = DockerImageArtifact.get(image, 1) + assert written.context_only + assert builds == [{"build": ("Dockerfile", ["Dockerfile", "run.sh"], written.image_name, None)}] assert announced == [image] assert {id: summary[:2] for id, summary in _summary(first).items() if "solver" in id} == { image: (1, False), f"{ROOT}/solver": (1, False)} agent = A2AAgent.get(f"{ROOT}/solver") assert (agent.docker_image_artifact.id, agent.docker_image_artifact.version) == (image, 1) - again = materialize(plan_of(bundle_dir), on_build=lambda write: announced.append(write.id)) + again = materialize(plan, on_build=lambda write: announced.append(write.id), build_here=local_builds(plan)) assert len(builds) == 1 and announced == [image] assert _summary(again)[image] == (1, True, ()) and _summary(again)[f"{ROOT}/solver"] == (1, True, ()) @@ -481,7 +483,6 @@ def test_an_image_is_built_from_the_files_the_ledger_hashes_so_a_reused_one_matc context = [".dockerignore", "Dockerfile", "agent.toml", "lib/run.sh", "run.sh"] assert [call["build"][1] for call in builds] == [context] - assert builds[0]["put"][2] == context assert again[f"{ROOT}/solver__agent_image"] == (1, True, ()) @@ -507,32 +508,29 @@ def test_an_image_written_as_its_build_context_is_uploaded_with_no_build_and_no_ image = f"{ROOT}/solver__agent_image" announced = [] - first = materialize(plan_of(bundle_dir), contexts=frozenset({image}), on_build=lambda write: announced.append(write.id)) + materialize(plan_of(bundle_dir), on_build=lambda write: announced.append(write.id)) written = DockerImageArtifact.get(image, 1) folder = bundle_dir / "agents/solver" - assert builds == [] and announced == [] and first.contexts == {image} - assert written.context_only and (written.dockerfile_path, written.platform) == ("Dockerfile", "linux/amd64") - assert written.source_digest == BuildContext.of(folder, folder / "Dockerfile").source_digest("linux/amd64") + assert builds == [] and announced == [] + assert written.context_only and (written.dockerfile_path, written.platform) == ("Dockerfile", None) + assert written.source_digest == BuildContext.of(folder, folder / "Dockerfile").source_digest(None) assert A2AAgent.get(f"{ROOT}/solver").docker_image_artifact.version == 1 - assert _summary(materialize(plan_of(bundle_dir), contexts=frozenset({image})))[image] == (1, True, ()) + assert _summary(materialize(plan_of(bundle_dir)))[image] == (1, True, ()) -def test_an_image_switched_between_built_here_and_a_build_context_is_rewritten_with_its_agent(bundle_dir, builds): +def test_an_image_is_one_version_whether_its_deploys_run_here_or_elsewhere(bundle_dir, builds): layout(bundle_dir, {"agents/solver/Dockerfile": "FROM scratch\nCOPY run.sh /\n", "agents/solver/run.sh": "echo hi\n"}) _steps(bundle_dir, [{"id": "agent", "type": "deploy_agent", "env_ids": [], "a2a_agent_id": "solver"}]) image = f"{ROOT}/solver__agent_image" _run(bundle_dir) - as_context = _summary(materialize(plan_of(bundle_dir), contexts=frozenset({image}))) - built_again = _summary(_run(bundle_dir)) + elsewhere = _summary(materialize(plan_of(bundle_dir))) + here_again = _summary(_run(bundle_dir)) - assert as_context[image] == (2, False, ("now written as a build context, built where it runs, not an image built " - "on this machine",)) - assert as_context[f"{ROOT}/solver"] == (2, False, (f"artifact {image} is written anew (v1 → v2)",)) - assert built_again[image] == (3, False, ("now written as an image built on this machine, not a build context, " - "built where it runs",)) - assert len(builds) == 2 + assert elsewhere[image] == elsewhere[f"{ROOT}/solver"] == (1, True, ()) + assert here_again[image] == here_again[f"{ROOT}/solver"] == (1, True, ()) + assert len(builds) == 1 def test_a_dry_run_predicts_a_rebuild_and_its_agents_rewrite_without_building(bundle_dir, builds): @@ -564,12 +562,12 @@ def test_an_agent_toml_edit_rebuilds_the_image_too_since_a_dockerfile_can_copy_i assert edited[f"{ROOT}/solver"][:2] == (2, False) -def test_a_failed_build_is_a_bundle_problem_naming_the_agent(bundle_dir, builds, monkeypatch): - def fail(dockerfile, context, tag, *, platform): +def test_a_failed_build_is_a_bundle_problem_naming_the_agent_and_writes_no_image(bundle_dir, builds, monkeypatch): + def fail(dockerfile, context, tag, *, platform, labels=None): output = "\n".join(f"#{n} step" for n in range(60)) raise materialize_module.DockerBuildError(f"docker build of {tag} failed (exit 1):\n{output}\nERROR: failed to solve") - monkeypatch.setattr(materialize_module, "build_image", fail) + monkeypatch.setattr(host_images, "build_image", fail) layout(bundle_dir, {"agents/solver/Dockerfile": "FROM scratch\n"}) _steps(bundle_dir, [{"id": "agent", "type": "deploy_agent", "env_ids": [], "a2a_agent_id": "solver"}]) @@ -577,54 +575,41 @@ def fail(dockerfile, context, tag, *, platform): lines = problem.splitlines() assert lines[0].startswith("agents/solver: docker build of ") and lines[0].endswith(" failed (exit 1):") assert lines[1:] == [*(f"#{n} step" for n in range(21, 60)), "ERROR: failed to solve"] + with pytest.raises(NotFoundError): + DockerImageArtifact.get(f"{ROOT}/solver__agent_image") -def test_a_failed_push_is_a_bundle_problem_naming_the_agent(bundle_dir, builds, monkeypatch): - def fail(id, **kwargs): - raise RuntimeError("could not start the local registry on port 5000: port is already allocated") - - monkeypatch.setattr(materialize_module.DockerImageArtifact, "put", fail) +def test_an_image_the_ledger_reuses_is_built_again_once_this_machines_docker_no_longer_holds_it(bundle_dir, builds): layout(bundle_dir, {"agents/solver/Dockerfile": "FROM scratch\n"}) _steps(bundle_dir, [{"id": "agent", "type": "deploy_agent", "env_ids": [], "a2a_agent_id": "solver"}]) + image = f"{ROOT}/solver__agent_image" + _run(bundle_dir) + host_images._built_from.__self__.clear() # the builds fixture's Docker, pruned - assert _problems(lambda: _run(bundle_dir)) == ( - "agents/solver: could not start the local registry on port 5000: port is already allocated",) + assert _summary(_run(bundle_dir, dry_run=True))[image] == (1, True, ()) and len(builds) == 1 + assert _summary(_run(bundle_dir))[image] == (1, True, ()) and len(builds) == 2 -@_RUN_OR_DRY_RUN -def test_an_image_to_build_without_docker_on_path_is_refused_before_anything_is_written( - bundle_dir, monkeypatch, dry_run, -): - monkeypatch.setattr(materialize_module.shutil, "which", lambda name: None) +def test_a_build_context_that_cant_be_written_is_a_bundle_problem_naming_the_agent(bundle_dir, builds, monkeypatch): + def fail(id, **kwargs): + raise RuntimeError("the object store is unreachable") + + monkeypatch.setattr(materialize_module.DockerImageArtifact, "put_context", fail) layout(bundle_dir, {"agents/solver/Dockerfile": "FROM scratch\n"}) _steps(bundle_dir, [{"id": "agent", "type": "deploy_agent", "env_ids": [], "a2a_agent_id": "solver"}]) - assert _problems(lambda: _run(bundle_dir, dry_run)) == ( - "agents/solver: building its image from Dockerfile needs docker, and it isn't on PATH",) - assert not local_store().path.exists() - - -def test_whether_an_image_needs_docker_is_decided_once_another_run_writing_it_is_done(bundle_dir, builds, monkeypatch): - """Another run may be building the image; once its lock is released, the ledger can reuse what it built.""" - events = [] - locked = materialize_module.materializing + assert _problems(lambda: _run(bundle_dir)) == ("agents/solver: the object store is unreachable",) - @contextlib.contextmanager - def materializing(bundle, on_wait=None): - with locked(bundle, on_wait): - events.append("locked") - yield - monkeypatch.setattr(materialize_module, "materializing", materializing) - check = materialize_module._refuse_builds_without_docker - monkeypatch.setattr(materialize_module, "_refuse_builds_without_docker", - lambda plan, ledger, contexts: events.append("docker checked") or check(plan, ledger, contexts)) +def test_an_image_built_here_without_docker_on_path_is_one_problem_and_writes_no_image(bundle_dir, monkeypatch): + monkeypatch.setenv("PATH", "") layout(bundle_dir, {"agents/solver/Dockerfile": "FROM scratch\n"}) _steps(bundle_dir, [{"id": "agent", "type": "deploy_agent", "env_ids": [], "a2a_agent_id": "solver"}]) - _run(bundle_dir) - - assert events == ["locked", "docker checked"] + (problem,) = _problems(lambda: _run(bundle_dir)) + assert problem.startswith("agents/solver: docker build of ") and problem.endswith("docker is not on PATH") + with pytest.raises(NotFoundError): + DockerImageArtifact.get(f"{ROOT}/solver__agent_image") @_RUN_OR_DRY_RUN @@ -916,7 +901,7 @@ def test_a_second_run_of_the_bundle_waits_for_the_first_to_finish_writing(bundle import pathlib, time from agent_env.bundle import parse_bundle from agent_env.bundle.ledger import materializing - with materializing(parse_bundle(pathlib.Path({str(bundle_dir)!r}))): + with materializing(entry.id for entry in parse_bundle(pathlib.Path({str(bundle_dir)!r})).entries): pathlib.Path({str(held)!r}).write_text("held") time.sleep(1.5) pathlib.Path({str(done)!r}).write_text("done") diff --git a/tst/unit/bundle/preflight_test.py b/tst/unit/bundle/preflight_test.py index 58d792c2..2b6f3947 100644 --- a/tst/unit/bundle/preflight_test.py +++ b/tst/unit/bundle/preflight_test.py @@ -218,40 +218,40 @@ def test_a_remote_image_runs_anywhere_and_a_local_one_on_the_local_provider(bund assert problem.startswith("tasks/t.json: step 'helper': deploys agent 'helper''s image") -# Images the bundle builds, written in the form their deploys run +# Images the bundle builds, each written as its build context, built where each deploy runs BUILT = "@local/~/triage/solver__agent_image" -CONTEXT = "a build context, which each deploy builds where it runs, for linux/amd64" +CONTEXT = "a build context, which each deploy builds where it runs, for its sandbox's platform" +BUILT_HERE = "a build context, built with docker on this machine" @pytest.mark.parametrize("sandbox, form", [ - ("local", "built on this machine"), ("modal_vm", CONTEXT), ("modal", CONTEXT), ("modal_vm,modal", CONTEXT), + ("local", BUILT_HERE), ("modal_vm", CONTEXT), ("modal", CONTEXT), ("modal_vm,modal", CONTEXT), + ("modal_vm,local", BUILT_HERE), ]) -def test_an_agent_the_bundle_builds_is_built_here_for_the_local_provider_and_a_build_context_for_those_that_build_it( +def test_an_agent_the_bundle_builds_is_its_build_context_wherever_it_deploys_and_built_here_for_a_deploy_here( bundle_dir, docker_on_path, sandbox, form): - """A VM builds it, and Modal builds it before creating the container that runs it.""" + """A VM builds it, Modal builds it before creating the container that runs it, and a run builds it with this + machine's Docker for a deploy on this machine, which a chain may fall back to.""" layout(bundle_dir, {"agents/solver/Dockerfile": "FROM scratch\n"}) _task(bundle_dir, [AGENT]) dry = dry_run_bundle(bundle_dir, sandbox=sandbox) - assert dry.materialization.contexts == (set() if sandbox == "local" else {BUILT}) + assert dry.built_here == ({BUILT} if form == BUILT_HERE else set()) assert dry.images() == (f"agents/solver (Dockerfile image) v1: {form}",) -@pytest.mark.parametrize("sandbox, form", [ - ("local", "built on this machine"), ("modal_vm", CONTEXT), ("modal", CONTEXT), -]) -def test_an_image_folder_an_agent_names_is_written_in_the_form_its_deploys_run( - bundle_dir, docker_on_path, sandbox, form): +@pytest.mark.parametrize("sandbox, form", [("local", BUILT_HERE), ("modal_vm", CONTEXT), ("modal", CONTEXT)]) +def test_an_image_folder_an_agent_names_is_built_where_its_deploys_run(bundle_dir, docker_on_path, sandbox, form): layout(bundle_dir, {"artifacts/base/Dockerfile": "FROM scratch\n", "agents/solver/agent.toml": 'image = "base"\n'}) _task(bundle_dir, [AGENT]) dry = dry_run_bundle(bundle_dir, sandbox=sandbox) - assert dry.materialization.contexts == (set() if sandbox == "local" else {"@local/~/triage/base"}) + assert dry.built_here == ({"@local/~/triage/base"} if form == BUILT_HERE else set()) assert dry.images() == (f"artifacts/base (Dockerfile image) v1: {form}",) @@ -303,30 +303,27 @@ def test_a_gateway_is_refused_on_a_provider_that_creates_no_vm_and_has_no_privat ] -MIXED = ("the bundle builds it from agents/solver/Dockerfile on this machine for the local provider, and as a build " - "context for one that builds it, and a run writes it one way; run its deploys on one kind, such as " - "--sandbox local or --sandbox modal_vm") +def test_an_image_only_a_step_the_walk_doesnt_model_uses_is_built_where_it_runs_and_needs_no_docker_here(bundle_dir, + monkeypatch): + monkeypatch.setattr(preflight_module, "docker_unreachable", lambda: "docker isn't on PATH") + layout(bundle_dir, {"agents/solver/Dockerfile": "FROM scratch\n"}) + _task(bundle_dir, [ + {"id": "box", "type": "deploy_sandbox", "sandbox_name": "box", "sandbox_mode": "vm", "sandbox_type": "modal_vm"}, + {"id": "install", "type": "install_agent", "sandbox_name": "box", "a2a_agent_id": "solver"}]) + + dry = dry_run_bundle(bundle_dir) + assert dry.built_here == set() and dry.images() == (f"agents/solver (Dockerfile image) v1: {CONTEXT}",) -def test_an_image_the_bundle_builds_is_refused_when_a_run_deploys_it_both_here_and_on_vms(bundle_dir): + +def test_one_version_of_an_image_the_bundle_builds_deploys_both_here_and_on_vms(bundle_dir, docker_on_path): layout(bundle_dir, {"agents/solver/Dockerfile": "FROM scratch\n"}) _task(bundle_dir, [{**AGENT, "sandbox_type": "local"}, {**AGENT, "id": "again", "agent_name": "again", "sandbox_type": "modal_vm"}]) - assert _problems(lambda: dry_run_bundle(bundle_dir)) == [ - "tasks/t.json: step 'agent': deploys agent '@local/~/triage/solver''s image on the 'local' sandbox provider, " - f"and tasks/t.json: step 'again' on the 'modal_vm' one: {MIXED}", - ] - + dry = dry_run_bundle(bundle_dir) -def test_an_image_the_bundle_builds_is_refused_on_a_chain_that_falls_back_from_vms_to_the_local_provider(bundle_dir): - layout(bundle_dir, {"agents/solver/Dockerfile": "FROM scratch\n"}) - _task(bundle_dir, [AGENT]) - - assert _problems(lambda: dry_run_bundle(bundle_dir, sandbox="modal_vm,local")) == [ - "tasks/t.json: step 'agent': deploys agent '@local/~/triage/solver''s image on the 'modal_vm,local' sandbox " - f"provider: {MIXED}", - ] + assert dry.built_here == {BUILT} and dry.images() == (f"agents/solver (Dockerfile image) v1: {BUILT_HERE}",) def test_a_sandbox_image_in_this_machines_registry_is_refused_on_another_provider(bundle_dir): @@ -399,19 +396,14 @@ def _context_image(id): "be built, not pulled; run it on a provider that builds it, such as --sandbox modal") -@pytest.mark.parametrize("sandbox, refused", [("modal_vm", False), ("modal", False), ("modal_vm,modal", False), - ("local", True), ("modal_vm,local", False)]) -def test_an_agent_image_that_is_only_a_build_context_runs_where_a_vm_or_modal_builds_it(bundle_dir, sandbox, refused): - """A chain skips the local provider, which runs an image by name, so one that builds it is enough.""" +@pytest.mark.parametrize("sandbox", ["modal_vm", "modal", "modal_vm,modal", "local", "modal_vm,local"]) +def test_an_agent_image_that_is_only_a_build_context_runs_where_a_vm_modal_or_this_machine_builds_it(bundle_dir, + sandbox): with namespace_routing(): A2AAgent.put(id="solver", docker_image_artifact=_context_image("solver-image")) _task(bundle_dir, [AGENT]) - if refused: - assert _problems(lambda: dry_run_bundle(bundle_dir, sandbox=sandbox)) == [ - "tasks/t.json: step 'agent': deploys agent 'solver''s image " + BY_NAME.format(sandbox=sandbox, id="solver-image")] - else: - assert [entry.name for entry in dry_run_bundle(bundle_dir, sandbox=sandbox).runs] == ["t"] + assert [entry.name for entry in dry_run_bundle(bundle_dir, sandbox=sandbox).runs] == ["t"] @pytest.mark.parametrize("ref, problem", [ @@ -443,16 +435,17 @@ def test_an_env_image_that_is_only_a_build_context_runs_on_a_gateway_vm_and_on_m assert [entry.name for entry in dry_run_bundle(bundle_dir, sandbox="modal").runs] == ["t"] -@pytest.mark.parametrize("sandbox", ["local", "modal_vm"]) -def test_a_lone_server_whose_image_is_only_a_build_context_is_refused_on_any_provider_but_modal(bundle_dir, sandbox): +def test_a_lone_server_whose_image_is_only_a_build_context_runs_where_modal_or_this_machine_builds_it(bundle_dir): with namespace_routing(): MCPServerEnv.put(id="crm", docker_image_artifact=_context_image("crm-image"), environment_name="crm", env_provider_type="server") _task(bundle_dir, [{"id": "env", "type": "deploy_env", "env_id": "crm"}]) - assert _problems(lambda: dry_run_bundle(bundle_dir, sandbox=sandbox)) == [ - "tasks/t.json: step 'env': deploys env 'crm''s image 'crm-image' " + BY_NAME.format(sandbox=sandbox, id="crm-image")] - assert [entry.name for entry in dry_run_bundle(bundle_dir, sandbox="modal").runs] == ["t"] + assert _problems(lambda: dry_run_bundle(bundle_dir, sandbox="modal_vm")) == [ + "tasks/t.json: step 'env': deploys env 'crm''s image 'crm-image' " + BY_NAME.format(sandbox="modal_vm", + id="crm-image")] + for sandbox in ("modal", "local"): + assert [entry.name for entry in dry_run_bundle(bundle_dir, sandbox=sandbox).runs] == ["t"] def _built_here(id): @@ -600,16 +593,16 @@ def test_containers_on_the_local_provider_need_docker_and_a_vm_sandbox_doesnt(bu ] -def test_a_chain_that_skips_the_local_provider_doesnt_need_docker(bundle_dir, monkeypatch): - """The local provider runs an image by name, so a deploy of one that is only a build context skips it for a VM.""" - probes = [] - monkeypatch.setattr(preflight_module, "docker_unreachable", lambda: probes.append(1) or "docker isn't on PATH") +def test_a_chain_that_falls_back_to_the_local_provider_needs_docker_to_build_an_image_that_is_only_its_context( + bundle_dir, monkeypatch): + """The local provider builds an image that is only a build context with this machine's Docker.""" + monkeypatch.setattr(preflight_module, "docker_unreachable", lambda: "docker isn't on PATH") with namespace_routing(): A2AAgent.put(id="solver", docker_image_artifact=_context_image("solver-image")) _task(bundle_dir, [AGENT]) - assert [entry.name for entry in dry_run_bundle(bundle_dir, sandbox="modal_vm,local").runs] == ["t"] - assert probes == [] + assert _problems(lambda: dry_run_bundle(bundle_dir, sandbox="modal_vm,local")) == [ + "the local sandbox provider runs containers for tasks/t.json: step 'agent', and docker isn't on PATH"] class _DeploysItself(Env): @@ -862,7 +855,7 @@ def _env_folder(root, name, toml="", kind="mcp_server"): @pytest.mark.parametrize("sandbox, toml", [("modal_vm", ""), ("modal", ""), ("modal", 'env_provider_type = "server"\n')]) -def test_an_image_the_bundle_builds_for_an_env_is_a_build_context_where_its_server_is_built(bundle_dir, sandbox, toml): +def test_an_image_the_bundle_builds_for_an_env_is_built_where_its_server_is(bundle_dir, sandbox, toml): """A gateway VM builds it, and Modal builds it for its gateway's container or a lone server's.""" _env_folder(bundle_dir, "crm", toml) _task(bundle_dir, [{"id": "env", "type": "deploy_env", "env_id": "crm"}]) @@ -870,7 +863,7 @@ def test_an_image_the_bundle_builds_for_an_env_is_a_build_context_where_its_serv dry = dry_run_bundle(bundle_dir, sandbox=sandbox) - assert dry.materialization.contexts == {f"{BUNDLE}/crm__env_image"} + assert dry.built_here == set() and dry.images() == (f"envs/crm (Dockerfile image) v1: {CONTEXT}",) def test_an_image_the_bundle_builds_for_a_lone_server_is_refused_where_the_server_runs_by_name(bundle_dir): diff --git a/tst/unit/bundle/run_test.py b/tst/unit/bundle/run_test.py index 7152bd33..96e71c2f 100644 --- a/tst/unit/bundle/run_test.py +++ b/tst/unit/bundle/run_test.py @@ -10,11 +10,12 @@ from click.testing import CliRunner import agent_env.bundle.run as run_module -from agent_env.bundle import materialize as materialize_module +from agent_env.providers.sandbox_providers import host_images from agent_env.artifact.artifacts.docker_image import DockerImageArtifact from agent_env.artifact.artifacts.file import FileArtifact from agent_env.artifact.store import get_artifact_store from agent_env.bundle import BundleError, Outcome, TaskRun, dry_run_bundle, parse_bundle, run_bundle +from agent_env.bundle.preflight import Preflight from agent_env.bundle.materialize import materialize from agent_env.bundle.plan import plan_bundle from agent_env.bundle.resolve import resolve_bundle @@ -147,14 +148,15 @@ def bundle_dir(tmp_path, monkeypatch, local_stores): @pytest.fixture def docker(monkeypatch): - """Stands in for docker: each build's tag is recorded, and the image written as a docker_image document.""" - builds = [] - monkeypatch.setattr(materialize_module.shutil, "which", lambda name: f"/usr/bin/{name}") - monkeypatch.setattr(materialize_module, "build_image", - lambda dockerfile, context, tag, *, platform: builds.append(tag)) - monkeypatch.setattr(materialize_module.DockerImageArtifact, "put", lambda id, **kwargs: get_artifact_store( - ).put_document(DockerImageArtifact(id=id, description="d", image_name=kwargs["image_name"], - tar_gz_s3_url=f"file:///{id}.tar.gz"))) + """Stands in for this machine's Docker: each build's tag is recorded, and Docker then holds it.""" + builds, held = [], {} + + def build(dockerfile, context, tag, *, platform, labels=None): + builds.append(tag) + held[tag] = (labels or {}).get(host_images._SOURCES_LABEL) + + monkeypatch.setattr(host_images, "build_image", build) + monkeypatch.setattr(host_images, "_built_from", held.get) return builds @@ -318,7 +320,11 @@ def broken(line): assert calls == ["tasks/a.json: v1, unchanged"] -def test_a_run_says_before_it_builds_an_image_and_names_the_image_apart_from_its_agent(bundle_dir, docker): +def test_a_run_says_before_it_builds_an_image_and_names_the_image_apart_from_its_agent(bundle_dir, docker, + monkeypatch): + """As it does for an image a deploy on this machine runs, which preflight names.""" + monkeypatch.setattr(run_module, "preflight_run", + lambda plan, tasks, sandbox: Preflight(frozenset(), (), frozenset({f"{ROOT}/solver__agent_image"}))) layout(bundle_dir, BUILT_AGENT) lines = [] @@ -561,7 +567,7 @@ def test_the_cli_dry_run_reads_another_bundles_entity_and_says_why_an_agent_pinn ) -def test_the_cli_dry_run_lists_an_image_it_would_build_and_builds_nothing(bundle_dir, docker, quiet_logs): +def test_the_cli_dry_run_lists_an_image_it_would_write_and_builds_nothing(bundle_dir, docker, quiet_logs): layout(bundle_dir, BUILT_AGENT) result = CliRunner().invoke(cli, ["run", str(bundle_dir), "--task", "agent", "--dry-run"]) @@ -574,7 +580,8 @@ def test_the_cli_dry_run_lists_an_image_it_would_build_and_builds_nothing(bundle "tasks/agent.json: v1 (new)\n" "\n" "Images:\n" - " agents/solver (Dockerfile image) v1: built on this machine\n" + " agents/solver (Dockerfile image) v1: a build context, which each deploy builds where it runs, for its " + "sandbox's platform\n" "Would run:\n" " tasks/agent.json v1\n" + DRY_RUN diff --git a/tst/unit/providers/sandbox_providers/host_images_test.py b/tst/unit/providers/sandbox_providers/host_images_test.py new file mode 100644 index 00000000..7ad68d88 --- /dev/null +++ b/tst/unit/providers/sandbox_providers/host_images_test.py @@ -0,0 +1,183 @@ +"""Images built from their build context on this machine: natively, once per sources, labeled with them, and saved once +for a VM here to load.""" + +import gzip +import io +import threading +import time + +import pytest + +from agent_env.artifact.artifacts.docker_image import DockerImageArtifact +from agent_env.providers.sandbox_providers import host_images +from agent_env.providers.sandbox_providers.local_sandbox import LocalSandbox +from agent_env.providers.sandbox_providers.sandbox import VmSandbox +from agent_env.utils.docker_build import DockerBuildError + + +@pytest.fixture +def docker(monkeypatch): + """This machine's Docker: each build is recorded with its Dockerfile, the files of its context, tag, platform and + labels, and Docker then holds the tag, labeled.""" + builds, held = [], {} + + def build(dockerfile, context, tag, *, platform, labels=None): + builds.append((dockerfile.relative_to(context).as_posix(), + sorted(p.relative_to(context).as_posix() for p in context.rglob("*") if p.is_file()), + tag, platform, labels)) + held[tag] = labels[host_images._SOURCES_LABEL] + + monkeypatch.setattr(host_images, "build_image", build) + monkeypatch.setattr(host_images, "_built_from", held.get) + return builds, held + + +@pytest.fixture +def context(tmp_path): + folder = tmp_path / "ctx" + folder.mkdir() + (folder / "Dockerfile").write_text("FROM scratch\nCOPY run.sh /\n") + (folder / "run.sh").write_text("echo hi\n") + return folder + + +def _put(context, id="solver__agent_image"): + return DockerImageArtifact.put_context(id, description="d", context_path=str(context), + dockerfile_path=str(context / "Dockerfile")) + + +def test_an_image_is_built_natively_from_its_stored_context_labeled_with_its_sources_and_once(local_stores, context, + docker): + builds, _ = docker + image = _put(context) + + host_images.build(image) + host_images.build(image) + + assert builds == [("Dockerfile", ["Dockerfile", "run.sh"], image.image_name, None, + {host_images._SOURCES_LABEL: image.source_digest})] + assert host_images.held(image) and host_images.built_here(image.image_name) + + +def test_an_image_docker_holds_from_other_sources_is_built_again(local_stores, context, docker): + builds, held = docker + image = _put(context) + held[image.image_name] = "sha256:" + "0" * 64 + + host_images.build(image) + + assert len(builds) == 1 and host_images.held(image) + + +def test_a_staged_build_says_so_first_only_when_it_builds(local_stores, context, docker): + builds, _ = docker + said = [] + sources, name = DockerImageArtifact.context_identity("solver__agent_image", str(context), str(context / "Dockerfile")) + + for _ in range(2): + host_images.build_staged(context, "Dockerfile", name, sources, lambda: said.append(name)) + + assert said == [name] and len(builds) == 1 + assert name == _put(context).image_name + + +def test_deploys_building_one_image_at_once_build_it_once(local_stores, context, docker, monkeypatch): + builds, held = docker + image = _put(context) + slow = host_images.build_image + + def build(*args, **kwargs): + time.sleep(0.3) + slow(*args, **kwargs) + + monkeypatch.setattr(host_images, "build_image", build) + threads = [threading.Thread(target=host_images.build, args=(image,)) for _ in range(4)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + assert len(builds) == 1 and held[image.image_name] == image.source_digest + + +def test_a_failed_build_raises_what_docker_says(local_stores, context, monkeypatch): + def fail(dockerfile, context, tag, *, platform, labels=None): + raise DockerBuildError(f"docker build of {tag} failed (exit 1):\nboom") + + monkeypatch.setattr(host_images, "build_image", fail) + monkeypatch.setattr(host_images, "_built_from", lambda name: None) + + with pytest.raises(DockerBuildError, match="boom"): + host_images.build(_put(context)) + + +def test_no_docker_on_path_holds_nothing(monkeypatch): + monkeypatch.setenv("PATH", "") + + assert not host_images.built_here("local/solver-0123:abc") + + +@pytest.mark.asyncio +async def test_a_local_sandbox_pulls_only_images_this_machine_didnt_build(monkeypatch, docker): + _, held = docker + held["local/solver-0123:abc"] = "sha256:" + "1" * 64 + pulled = [] + + async def pull(self, names): + pulled.extend(names) + + monkeypatch.setattr(VmSandbox, "pull_images", pull) + await LocalSandbox().pull_images(["local/solver-0123:abc", "ghcr.io/acme/tool:1"]) + + assert pulled == ["ghcr.io/acme/tool:1"] + + +@pytest.mark.asyncio +async def test_a_local_sandbox_builds_a_context_with_this_machines_docker(local_stores, context, docker): + builds, _ = docker + image = _put(context) + + await LocalSandbox().build_images([image, image]) + + assert [build[2:4] for build in builds] == [(image.image_name, None)] + + +class _Save: + """``docker save``, writing the name it saves.""" + + def __init__(self, argv, **kwargs): + self.stdout, self.stderr = io.BytesIO(argv[-1].encode()), io.BytesIO() + + def wait(self): + return 0 + + +def test_images_of_equal_sources_under_two_names_are_saved_once_each_so_each_loads_under_its_own(local_stores, context, + docker, monkeypatch): + monkeypatch.setattr(host_images.subprocess, "Popen", _Save) + first, second = _put(context, "solver__agent_image"), _put(context, "helper__agent_image") + + urls = [host_images._saved_here(image) for image in (first, second, first)] + + assert first.source_digest == second.source_digest and urls[0] != urls[1] and urls[2] == urls[0] + store = local_stores.get_object_store() + assert [gzip.decompress(store.get(url)).decode() for url in urls[:2]] == [first.image_name, second.image_name] + + +@pytest.mark.asyncio +async def test_a_local_sandbox_given_a_platform_builds_as_a_vm_does(local_stores, context, docker, monkeypatch): + builds, _ = docker + built_in_vm = [] + + async def build_in_vm(self, artifacts): + built_in_vm.extend(artifact.image_name for artifact in artifacts) + + monkeypatch.setattr(VmSandbox, "build_images", build_in_vm) + sandbox = LocalSandbox() + sandbox.platform = "linux/amd64" + image = _put(context) + + await sandbox.build_images([image]) + + assert builds == [] and built_in_vm == [image.image_name] + diff --git a/tst/unit/providers/sandbox_providers/provider_capabilities_test.py b/tst/unit/providers/sandbox_providers/provider_capabilities_test.py index 524734a1..e8f7ccb8 100644 --- a/tst/unit/providers/sandbox_providers/provider_capabilities_test.py +++ b/tst/unit/providers/sandbox_providers/provider_capabilities_test.py @@ -41,7 +41,7 @@ @pytest.mark.parametrize("cls, on_this_machine, creates_vms, sandbox, container, private_network", [ - (LocalSandboxProvider, True, True, NAME, NAME, False), + (LocalSandboxProvider, True, True, NAME_OR_CONTEXT, NAME_OR_CONTEXT, False), (ModalSandboxProvider, False, False, NAME_OR_CONTEXT, NAME_OR_CONTEXT, True), (ModalVmSandboxProvider, False, True, LOADABLE, NAME, False), (E2BSandboxProvider, False, True, LOADABLE, NAME, False), @@ -124,7 +124,7 @@ class _VmsAgain(_Vms): (_ContainersOnly, NAME), (_BuildingContainers, NAME_OR_CONTEXT), # create_sandbox is one of its containers (_ContainerAgents, NAME), (_ContainerAgentsAgain, NAME), # declared, and kept by a subclass (_VmsAgain, LOADABLE), # derived again for a subclass that declares nothing - (LocalSandboxProvider, NAME), + (LocalSandboxProvider, NAME_OR_CONTEXT), # its containers build an image that is only its context ], ids=lambda value: getattr(value, "__name__", getattr(value, "name", value))) def test_an_agents_sandbox_takes_what_create_sandbox_gives_unless_the_provider_declares_otherwise(cls, accepts): assert cls.SANDBOX_ACCEPTS is accepts diff --git a/tst/unit/providers/sandbox_providers/smol_vm_provider_test.py b/tst/unit/providers/sandbox_providers/smol_vm_provider_test.py index 274d504d..e3cafb63 100644 --- a/tst/unit/providers/sandbox_providers/smol_vm_provider_test.py +++ b/tst/unit/providers/sandbox_providers/smol_vm_provider_test.py @@ -13,6 +13,9 @@ pytest.importorskip("smol") +from agent_env.artifact.artifacts.docker_image import DockerImageArtifact +from agent_env.providers.sandbox_providers import host_images +from agent_env.providers.sandbox_providers.sandbox import VmSandbox from agent_env.providers.sandbox_providers.smol_vm import provider as module from agent_env.providers.sandbox_providers.smol_vm import sandbox as sandbox_module from agent_env.providers.sandbox_providers.smol_vm.sandbox import SmolVmSandbox @@ -315,3 +318,57 @@ async def test_terminate_sandbox_deletes_a_listed_machine_without_setting_it_up( assert deleted == ([True] if listed else []) assert connect.called is listed setup.assert_not_called() + + +@pytest.mark.asyncio +async def test_a_smol_vm_of_this_machines_platform_loads_the_build_this_machine_saves(monkeypatch): + image = DockerImageArtifact(id="solver-image", version=1, description="d", image_name="local/solver-0123:abc", + build_context_object_url="file:///ctx.tar.gz", dockerfile_path="Dockerfile") + loaded, built = [], [] + + async def saved(artifact): + return f"file:///saved/{artifact.image_name}.tar.gz" + + async def load(self, artifacts): + loaded.extend(artifact.tar_gz_object_url for artifact in artifacts) + + async def build_in_vm(self, artifacts): + built.extend(artifact.image_name for artifact in artifacts) + + monkeypatch.setattr(sandbox_module, "saved_here", saved) + monkeypatch.setattr(SmolVmSandbox, "_load_tarballs", load) + monkeypatch.setattr(VmSandbox, "build_images", build_in_vm) + vm = object.__new__(SmolVmSandbox) + + vm.platform = host_images.host_platform() + await vm.build_images([image]) + vm.platform = "linux/riscv64" + await vm.build_images([image]) + + assert loaded == ["file:///saved/local/solver-0123:abc.tar.gz"] and built == ["local/solver-0123:abc"] + + +@pytest.mark.asyncio +async def test_an_arm64_smol_vm_on_an_arm64_machine_runs_a_bundle_image_built_here(local_stores, tmp_path, monkeypatch): + """A bundle writes its image as a context recording no platform, which an arm64 VM runs, from this machine's + native build.""" + (tmp_path / "Dockerfile").write_text("FROM scratch\n") + image = DockerImageArtifact.put_context("solver-image", description="d", context_path=str(tmp_path), + dockerfile_path=str(tmp_path / "Dockerfile"), platform=None) + loaded = [] + + async def saved(artifact): + return "file:///saved/solver.tar.gz" + + async def load(self, artifacts): + loaded.extend(artifact.tar_gz_object_url for artifact in artifacts) + + monkeypatch.setattr(sandbox_module, "host_platform", lambda: "linux/arm64") + monkeypatch.setattr(sandbox_module, "saved_here", saved) + monkeypatch.setattr(SmolVmSandbox, "_load_tarballs", load) + vm = object.__new__(SmolVmSandbox) + vm.platform = "linux/arm64" + + await vm.load_docker_images([image]) + + assert image.platform is None and loaded == ["file:///saved/solver.tar.gz"] diff --git a/tst/unit/providers/sandbox_providers/vm_sandbox_test.py b/tst/unit/providers/sandbox_providers/vm_sandbox_test.py index e1a79bf8..50689403 100644 --- a/tst/unit/providers/sandbox_providers/vm_sandbox_test.py +++ b/tst/unit/providers/sandbox_providers/vm_sandbox_test.py @@ -765,6 +765,20 @@ async def test_a_build_takes_the_recorded_dockerfile_and_platform_else_linux_amd assert "docker build --platform linux/arm64 -f Dockerfile -t local/b-0123456789ab:v1 ." in builds[1] +@pytest.mark.asyncio +async def test_a_context_recording_no_platform_is_built_for_the_vms_own(signing_store): + """As a bundle writes its images: a VM of any platform runs one, built for its own.""" + image = _context_image("a", "sha256:" + "1" * 64, platform=None) + arm = _RecordingVmSandbox() + arm.platform = "linux/arm64" + + assert arm.load_problems([image]) == [] + await arm.load_docker_images([image]) + + assert any("docker build --platform linux/arm64 -f Dockerfile -t local/a-0123456789ab:v1 ." in script + for script in arm.scripts) + + class _BuildFailsVm(_RecordingVmSandbox): async def exec_with_output(self, *args): if args[:2] == ("sudo", "bash") and "docker build" in script_run(args): diff --git a/tst/unit/task_step/test_entity_refs.py b/tst/unit/task_step/test_entity_refs.py index eff99261..f91c3ab1 100644 --- a/tst/unit/task_step/test_entity_refs.py +++ b/tst/unit/task_step/test_entity_refs.py @@ -5,7 +5,7 @@ import pytest from agent_env.artifact.registry import get_artifact_registry -from agent_env.entity.refs import EntityKind, EntityRef, parse_toml_ref, ref_sites +from agent_env.entity.refs import EntityKind, EntityRef, default_agent_refs, parse_toml_ref, ref_sites from agent_env.task_step.registry import _builtin_registry from agent_env.task_step.snapshot_utils.snapshot_series import SnapshotConfig from agent_env.task_step.task_step import TaskStep @@ -208,6 +208,25 @@ def test_malformed_declarations_fail_where_they_are_written(bad): bad() +@pytest.mark.parametrize("step,unset", [ + ({"type": "deploy_agent"}, ["a2a_agent_id"]), + ({"type": "deploy_agent", "a2a_agent_id": None}, ["a2a_agent_id"]), + ({"type": "deploy_agent", "a2a_agent_id": "solver"}, []), + ({"type": "rubrics_verifier"}, ["judge_a2a_agent_id"]), + ({"type": "rubrics_verifier", "use_agent_judge": True, "agent_name": None}, ["judge_a2a_agent_id"]), + ({"type": "rubrics_verifier", "judge_a2a_agent_id": "judge"}, []), + ({"type": "rubrics_verifier", "agent_name": "solver"}, ["judge_a2a_agent_id"]), + ({"type": "rubrics_verifier", "use_agent_judge": False}, ["judge_a2a_agent_id"]), + ({"type": "verify_a2a_agent_card", "a2a_agent_id": "solver"}, []), + ({"type": "install_agent", "sandbox_name": "box"}, ["a2a_agent_id"]), + ({"type": "install_agent", "sandbox_name": "box", "a2a_agent_id": "solver"}, []), +]) +def test_an_unset_agent_field_falls_back_to_the_default_agent(step, unset): + refs = _builtin_registry()[step["type"]].entity_refs + + assert [ref.path for ref in default_agent_refs(refs, step)] == unset + + def test_each_kind_has_a_constructor_and_only_artifacts_take_a_type(): kinds = [EntityRef.env("e").kind, EntityRef.agent("a").kind, EntityRef.artifact("f").kind, EntityRef.task("t").kind] assert kinds == list(EntityKind) diff --git a/tst/unit/task_step/unwritten_local_agent_test.py b/tst/unit/task_step/unwritten_local_agent_test.py new file mode 100644 index 00000000..49cdb3a2 --- /dev/null +++ b/tst/unit/task_step/unwritten_local_agent_test.py @@ -0,0 +1,49 @@ +"""A step deploying an @local agent the store doesn't hold says that a bundle run writes one, not only that it isn't +found: outside a bundle run, nothing has written it.""" + +import pytest + +from agent_env.a2a_agent import A2AAgent +from agent_env.store.base import NotFoundError +from agent_env.task_step.context import PromptResponse, TaskStepContext +from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep + +AGENT = "@local/agentenv-framework/agents/claude-code" +WRITTEN_BY_A_RUN = r"there is no agent '@local/agentenv-framework/agents/claude-code' in the store: an @local agent is " \ + r"written by `agent-env run` of the bundle that defines it or uses it" + + +@pytest.fixture(autouse=True) +def empty_store(monkeypatch): + def missing(cls, id, version=None): + raise NotFoundError(f"A2AAgent {id} not found") + + monkeypatch.setattr(A2AAgent, "get", classmethod(missing)) + + +@pytest.mark.asyncio +async def test_deploying_an_unwritten_local_default_agent_names_the_run_that_writes_one(monkeypatch): + monkeypatch.setattr("agent_env.config.runtime.Config.get_default_a2a_agent_id", lambda self: AGENT) + + with pytest.raises(NotFoundError, match=WRITTEN_BY_A_RUN + r".*point a2a_agent_id or \[agents\] default_a2a"): + await DeployAgentTaskStep(id="deploy", version=None, agent_name="solver").execute(TaskStepContext()) + + +@pytest.mark.asyncio +async def test_a_store_agent_that_isnt_there_is_reported_as_the_store_reports_it(): + with pytest.raises(NotFoundError, match="^A2AAgent solver not found$"): + await DeployAgentTaskStep(id="deploy", version=None, a2a_agent_id="solver").execute(TaskStepContext()) + + +@pytest.mark.asyncio +async def test_a_judge_deploying_an_unwritten_local_agent_names_the_run_that_writes_one(): + step = RubricsVerifierTaskStep(id="judge", version=1, criteria=[{"id": "done", "description": "done"}], + prompt_id="p1", verifier_id="v", use_trajectory=False, judge_a2a_agent_id=AGENT) + context = TaskStepContext( + prompt_responses=[PromptResponse(prompt_id="p1", response="done", prompt_text="do it")], + metadata={"user_overrides": {"judge_litellm_api_key": "k", "judge_litellm_base_url": "http://litellm"}}, + ) + + with pytest.raises(NotFoundError, match=WRITTEN_BY_A_RUN + r".*point judge_a2a_agent_id"): + await step.execute(context)