diff --git a/AGENTS.md b/AGENTS.md index e11b9828..265eada1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -99,7 +99,7 @@ and deserialized through a registry. | Package | Contents | |---|---| | `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset` and `load_file_artifact_universe`. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`), `GatewayEnv` (`gateway_server`). `env/gateway/gateway.py` is the env-side gateway server, container-only code. | -| `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Built-ins: `CliArtifact` (`cli`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/build_context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | +| `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Built-ins: `CliArtifact` (`cli`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | | `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, and more). `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the `@local` namespace: while the CLI runs, an `@local/…` id and what an `@local` task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | | `config/` | The `Config` singleton (`get_config`, `configure`, `reset_config`) in `config/runtime.py`, file discovery in `config/loader.py`, and `load_impl`, which resolves `module:Class` pointers. `agent_env.store` re-exports the config names for compatibility. | diff --git a/src/agent_env/artifact/artifacts/docker_image.py b/src/agent_env/artifact/artifacts/docker_image.py index 2c3e4443..d61a6491 100644 --- a/src/agent_env/artifact/artifacts/docker_image.py +++ b/src/agent_env/artifact/artifacts/docker_image.py @@ -23,8 +23,8 @@ from agent_env.store.image_store.local_registry_image_store import LocalRegistryImageStore from agent_env.store.image_store.oci_registry_credentials import is_loopback_host, names_registry, registry_host_from_ref from agent_env.store.image_store.registry_api import pin_digest, pinned_from -from agent_env.utils.build_context import BuildContext -from agent_env.utils.docker_build import DEFAULT_BUILD_PLATFORM, normalize_platform +from agent_env.utils.docker.context import BuildContext +from agent_env.utils.docker.build import DEFAULT_BUILD_PLATFORM, normalize_platform logger = logging.getLogger(__name__) diff --git a/src/agent_env/bundle/materialize.py b/src/agent_env/bundle/materialize.py index 9029eed2..14a94cb6 100644 --- a/src/agent_env/bundle/materialize.py +++ b/src/agent_env/bundle/materialize.py @@ -36,7 +36,7 @@ from agent_env.task import Task from agent_env.task_step.registry import get_task_step_registry from agent_env.task_step.task_step import TaskStep -from agent_env.utils.docker_build import DockerBuildError +from agent_env.utils.docker.build import DockerBuildError from ._fs import with_article from .authoring import AuthoringContext, build_context_files diff --git a/src/agent_env/bundle/preflight.py b/src/agent_env/bundle/preflight.py index a4384585..3c251e07 100644 --- a/src/agent_env/bundle/preflight.py +++ b/src/agent_env/bundle/preflight.py @@ -71,7 +71,7 @@ from agent_env.task_step.task_steps.deploy_env import DeployEnvTaskStep from agent_env.task_step.task_steps.deploy_sandbox import DeploySandboxTaskStep from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep -from agent_env.utils.docker_build import DEFAULT_BUILD_PLATFORM, docker_unreachable +from agent_env.utils.docker.build import DEFAULT_BUILD_PLATFORM, docker_unreachable from .parse import BundleError, BundleKind from .plan import Plan diff --git a/src/agent_env/cli/__init__.py b/src/agent_env/cli/__init__.py index a2b4f10b..9356b217 100644 --- a/src/agent_env/cli/__init__.py +++ b/src/agent_env/cli/__init__.py @@ -7,7 +7,7 @@ from agent_env.plugins._cli import load_cli_plugins, load_cli_root_options from agent_env.store.base import NotFoundError from agent_env.store.routing import namespace_routing -from agent_env.utils.docker_build import DockerBuildError +from agent_env.utils.docker.build import DockerBuildError from .a2a_agent import a2a_agent from .artifact import artifact diff --git a/src/agent_env/cli/a2a_agent/put.py b/src/agent_env/cli/a2a_agent/put.py index d1a15678..407c411e 100644 --- a/src/agent_env/cli/a2a_agent/put.py +++ b/src/agent_env/cli/a2a_agent/put.py @@ -7,7 +7,7 @@ from agent_env.artifact import DockerImageArtifact from agent_env.cli.utils import build_platform_option, detect_env_metadata, refuse_unwritable_ids from agent_env.store.ids import derive_id, image_repository -from agent_env.utils.docker_build import DEFAULT_BUILD_PLATFORM, build_image +from agent_env.utils.docker.build import DEFAULT_BUILD_PLATFORM, build_image @click.command() diff --git a/src/agent_env/cli/env/mcp_server.py b/src/agent_env/cli/env/mcp_server.py index 82696df4..86fab3aa 100644 --- a/src/agent_env/cli/env/mcp_server.py +++ b/src/agent_env/cli/env/mcp_server.py @@ -16,7 +16,7 @@ resolve_environment_name, ) from agent_env.utils.card_naming import card_name_from_github, card_name_from_source -from agent_env.utils.docker_build import DEFAULT_BUILD_PLATFORM, build_image +from agent_env.utils.docker.build import DEFAULT_BUILD_PLATFORM, build_image from agent_env.env import Env, MCPServerEnv from agent_env.store.ids import derive_id, image_repository diff --git a/src/agent_env/cli/env/website.py b/src/agent_env/cli/env/website.py index f10930fd..9045413b 100644 --- a/src/agent_env/cli/env/website.py +++ b/src/agent_env/cli/env/website.py @@ -16,7 +16,7 @@ resolve_environment_name, ) from agent_env.utils.card_naming import card_name_from_github, card_name_from_source -from agent_env.utils.docker_build import DEFAULT_BUILD_PLATFORM, build_image +from agent_env.utils.docker.build import DEFAULT_BUILD_PLATFORM, build_image from agent_env.env.envs.website import WebsiteEnv from agent_env.store.ids import derive_id, image_repository diff --git a/src/agent_env/cli/utils.py b/src/agent_env/cli/utils.py index 21a0b72d..0489733a 100644 --- a/src/agent_env/cli/utils.py +++ b/src/agent_env/cli/utils.py @@ -10,8 +10,8 @@ from agent_env.providers.env_providers.env_provider import _env_provider_class from agent_env.providers.env_providers.env_server_provider import EnvironmentServerProvider from agent_env.store.base import NotFoundError -from agent_env.utils.build_metadata import detect_base_metadata, detect_env_metadata # noqa: F401 re-exported for the put commands -from agent_env.utils.docker_build import DEFAULT_BUILD_PLATFORM +from agent_env.utils.docker.metadata import detect_base_metadata, detect_env_metadata # noqa: F401 re-exported for the put commands +from agent_env.utils.docker.build import DEFAULT_BUILD_PLATFORM @contextmanager diff --git a/src/agent_env/env/bootstrap.py b/src/agent_env/env/bootstrap.py index 02263c5a..acd6eab1 100644 --- a/src/agent_env/env/bootstrap.py +++ b/src/agent_env/env/bootstrap.py @@ -27,9 +27,9 @@ from agent_env.store.local_state import holding_locks from agent_env.store.object_store import LocalFilesystemObjectStore from agent_env.store.routing import configured_store -from agent_env.utils.build_context import BuildContext -from agent_env.utils.build_metadata import detect_env_metadata -from agent_env.utils.docker_build import build_image, docker_unreachable +from agent_env.utils.docker.context import BuildContext +from agent_env.utils.docker.metadata import detect_env_metadata +from agent_env.utils.docker.build import build_image, docker_unreachable _ENV_PACKAGE = Path(__file__).parent GATEWAY_DOCKERFILE = _ENV_PACKAGE / "gateway" / "Dockerfile" diff --git a/src/agent_env/providers/sandbox_providers/host_images.py b/src/agent_env/providers/sandbox_providers/host_images.py index 0ce2f494..d5b0dd54 100644 --- a/src/agent_env/providers/sandbox_providers/host_images.py +++ b/src/agent_env/providers/sandbox_providers/host_images.py @@ -23,7 +23,7 @@ from agent_env.artifact.artifacts.docker_image import DockerImageArtifact from agent_env.config import get_config from agent_env.store.local_state import holding_locks -from agent_env.utils.docker_build import build_image, normalize_platform +from agent_env.utils.docker.build import build_image, normalize_platform logger = logging.getLogger(__name__) diff --git a/src/agent_env/providers/sandbox_providers/modal_image_build.py b/src/agent_env/providers/sandbox_providers/modal_image_build.py index c7eca715..3fad4d3a 100644 --- a/src/agent_env/providers/sandbox_providers/modal_image_build.py +++ b/src/agent_env/providers/sandbox_providers/modal_image_build.py @@ -5,9 +5,7 @@ import asyncio import glob -import json import logging -import re import tarfile import tempfile import weakref @@ -17,7 +15,8 @@ import modal from agent_env.config import get_config -from agent_env.utils.build_context import extract, ignore_file +from agent_env.utils.docker.context import extract, ignore_file +from agent_env.utils.docker.dockerfile import Instruction, copy_arguments, instructions, opens_heredoc if TYPE_CHECKING: from agent_env.artifact.artifacts.docker_image import DockerImageArtifact @@ -76,7 +75,6 @@ def fmt_exc(e: BaseException) -> str: return f"{type_name}: {msg}" if msg else type_name -_COPY_OR_ADD = re.compile(r"\s*(COPY|ADD)\s+(.*)", re.IGNORECASE | re.DOTALL) # The flags COPY takes as ADD does; ADD's own (--checksum, --keep-git-dir, --unpack) keep an ADD as it is. _COPY_FLAGS = ("--chown=", "--chmod=", "--link") @@ -86,43 +84,29 @@ def modal_dockerfile(text: str, context: Path) -> str: a JSON-form COPY or ADD whose paths hold no whitespace in shell form, which Modal's context parser reads, and an ADD of files and folders in ``context``, none of them a tar archive, as the COPY it is, since Modal's ADD only fetches a URL. Every other line is left as written.""" - lines, instruction = [], [] - for line in text.splitlines(keepends=True): - instruction.append(line) - if not line.rstrip("\r\n").endswith("\\"): - lines.append(_for_modal("".join(instruction), context)) - instruction = [] - return "".join(lines + instruction) - - -def _for_modal(instruction: str, context: Path) -> str: - match = _COPY_OR_ADD.fullmatch(re.sub(r"\\\r?\n", " ", instruction).strip()) - if match is None: - return instruction - keyword, rest = match.group(1).upper(), match.group(2).strip() - flags = [] - while rest.startswith("--"): - flag, _, rest = rest.partition(" ") - flags.append(flag) - rest = rest.strip() - if rest.startswith("["): - try: - args = json.loads(rest) - except ValueError: - return instruction - if not (isinstance(args, list) and len(args) >= 2 - and all(isinstance(arg, str) and arg and not any(c.isspace() for c in arg) for arg in args)): - return instruction - elif keyword == "ADD" and not rest.startswith("<<") and not any(c in rest for c in "\"'"): - args = rest.split() - else: - return instruction + return "".join(_for_modal(instruction, context) for instruction in instructions(text)) + + +def _for_modal(instruction: Instruction, context: Path) -> str: + args = copy_arguments(instruction) + if args is None or args.heredoc: + return instruction.text + paths = (*args.sources, args.destination) + if args.json_form: + # Written in shell form, a path with whitespace would split, and one like < bool: diff --git a/src/agent_env/providers/sandbox_providers/sandbox.py b/src/agent_env/providers/sandbox_providers/sandbox.py index 422dd336..d5f0429d 100644 --- a/src/agent_env/providers/sandbox_providers/sandbox.py +++ b/src/agent_env/providers/sandbox_providers/sandbox.py @@ -24,7 +24,7 @@ from agent_env.config import get_config from agent_env.store.image_store.oci_registry_credentials import registry_host_from_ref -from agent_env.utils.docker_build import DEFAULT_BUILD_PLATFORM +from agent_env.utils.docker.build import DEFAULT_BUILD_PLATFORM from agent_env.utils.paths import validate_relative_filename if TYPE_CHECKING: diff --git a/src/agent_env/providers/sandbox_providers/smol_vm/sandbox.py b/src/agent_env/providers/sandbox_providers/smol_vm/sandbox.py index 561aba7e..bfc99bce 100644 --- a/src/agent_env/providers/sandbox_providers/smol_vm/sandbox.py +++ b/src/agent_env/providers/sandbox_providers/smol_vm/sandbox.py @@ -16,7 +16,7 @@ from agent_env.providers.sandbox_providers.local_sandbox import LocalSandboxProvider from agent_env.store.object_store.local.store import LocalFilesystemObjectStore from agent_env.store.object_store.local.tls import local_ca -from agent_env.utils.docker_build import normalize_platform +from agent_env.utils.docker.build import normalize_platform from agent_env.providers.sandbox_providers.sandbox_provider import SANDBOX_MODE_VM _PORTS_FILE = "/storage/agentenv-ports.json" diff --git a/src/agent_env/utils/docker/__init__.py b/src/agent_env/utils/docker/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/agent_env/utils/docker_build.py b/src/agent_env/utils/docker/build.py similarity index 100% rename from src/agent_env/utils/docker_build.py rename to src/agent_env/utils/docker/build.py diff --git a/src/agent_env/utils/build_context.py b/src/agent_env/utils/docker/context.py similarity index 91% rename from src/agent_env/utils/build_context.py rename to src/agent_env/utils/docker/context.py index c00ad88f..714df6dc 100644 --- a/src/agent_env/utils/build_context.py +++ b/src/agent_env/utils/docker/context.py @@ -59,7 +59,7 @@ def of(cls, root: Path, dockerfile: Path | None = None) -> BuildContext: named = None if dockerfile is not None and _inside(top, Path(os.path.realpath(dockerfile))): named = _inside(top, dockerfile) - matcher = FilePatternMatcher.from_file(ignores) if ignores else None + matcher = ignore_matcher(ignores.read_text(encoding="utf-8")) if ignores else None walk = _Walk(top, matcher, kept) walk.run() if walk.problems: @@ -104,7 +104,7 @@ def extract(archive: Path, out: Path) -> None: of ``out`` is left out, as nothing there is the build's. Raises ValueError on anything else, which no context written by ``BuildContext.write`` holds: a hard link, a device, a name that leads out of ``out``.""" with tarfile.open(archive, "r:gz") as tar: - tar.extractall(out, filter=_context_member) + tar.extractall(out, filter=context_member) class _Walk: @@ -206,15 +206,15 @@ def hexdigest(self) -> str: return self._digest.hexdigest() -def _context_member(member: tarfile.TarInfo, out: str) -> tarfile.TarInfo | None: - """``member`` as ``extract`` writes it, None to leave it out.""" +def context_member(member: tarfile.TarInfo, out: str, *, links_out: bool = False) -> tarfile.TarInfo | None: + """``member`` as ``extract`` writes it; None for a link leading out of ``out``, unless ``links_out``.""" if not (member.isreg() or member.isdir() or member.issym()): raise ValueError(f"{member.name!r} in the build context is neither a file, a folder nor a link") try: member = tarfile.tar_filter(member, out) except tarfile.FilterError as e: raise ValueError(f"{member.name!r} in the build context: {e}") from e - if member.issym(): + if member.issym() and not links_out: top = os.path.realpath(out) target = os.path.realpath(os.path.join(top, os.path.dirname(member.name), member.linkname)) if os.path.commonpath([top, target]) != top: @@ -229,6 +229,17 @@ def ignore_file(top: Path, dockerfile: Path | None) -> Path | None: return next((path for path in (*candidates, top / ".dockerignore") if path.is_file()), None) +def ignore_names(dockerfile: str) -> tuple[str, str]: + """``ignore_file``'s candidates, in order, for a Dockerfile at ``dockerfile`` in the context.""" + return f"{dockerfile}.dockerignore", ".dockerignore" + + +def ignore_matcher(ignore_file: str) -> FilePatternMatcher: + """A matcher for an ignore file's patterns, as docker build reads them: a line opening with ``#`` is a comment.""" + lines = ignore_file.removeprefix("\ufeff").splitlines() + return FilePatternMatcher(*(line for line in lines if not line.startswith("#"))) + + def _inside(top: Path, path: Path) -> str | None: """``path``'s name in the context at ``top``, the folders above it resolved but not the file itself, so a link keeps its own name; None when it's outside the context.""" diff --git a/src/agent_env/utils/docker/dockerfile.py b/src/agent_env/utils/docker/dockerfile.py new file mode 100644 index 00000000..2c55b11c --- /dev/null +++ b/src/agent_env/utils/docker/dockerfile.py @@ -0,0 +1,293 @@ +"""A Dockerfile's instructions as ``docker build`` reads them, its COPY and ADD arguments, and what a build of it reads +of its context.""" + +from __future__ import annotations + +import csv +import json +import posixpath +import re +from dataclasses import dataclass + +_DIRECTIVE = re.compile(r"#\s*([A-Za-z][A-Za-z0-9_-]*)\s*=\s*(\S+)\s*") +# A word opening a heredoc, as BuildKit reads one: a file descriptor, <<, a -, and its delimiter, quoted or not. A +# here-string (<<<) is left to the shell. +_HEREDOC = re.compile(r"\d*<<(-?)([^<]+)") +_HEREDOC_KEYWORDS = frozenset({"RUN", "COPY", "ADD"}) +# The COPY and ADD flags that change how a source is written, not which one it is. +_SOURCE_KEEPING_FLAGS = ("--chown=", "--chmod=", "--link", "--checksum=", "--keep-git-dir", "--unpack") +# The RUN --mount types that never read the build context. +_OFF_CONTEXT_MOUNTS = frozenset({"cache", "tmpfs", "secret", "ssh"}) +# The Dockerfile frontend this reading follows; another may read the context its own way. +_DOCKERFILE_SYNTAX = re.compile(r"(?:docker\.io/)?docker/dockerfile(?:-upstream)?(?::[\w.-]+)?(?:@sha256:[0-9a-f]{64})?") + + +@dataclass(frozen=True) +class Instruction: + """An instruction, or a line between them (``keyword`` None); all their ``text`` joined is the Dockerfile.""" + + keyword: str | None # upper case + arguments: str # on one line, continued lines joined, heredoc bodies left out + text: str + heredocs: tuple[str, ...] = () # each heredoc's body, in order + complete: bool = True # False when a heredoc it opens is never closed + + +@dataclass(frozen=True) +class ContextReads: + """The context paths a build reads, each of which may hold Docker's wildcards, or why it reads all of it.""" + + paths: tuple[str, ...] = () + whole: str | None = None + + +@dataclass(frozen=True) +class CopyArguments: + """A COPY or ADD's flags as written (``--name`` or ``--name=value``), sources and destination.""" + + keyword: str + flags: tuple[str, ...] + sources: tuple[str, ...] # a heredoc's ``< list[Instruction]: + """``dockerfile``'s instructions and the lines between them, in order, as ``docker build`` splits them.""" + lines = dockerfile.splitlines(keepends=True) + parsed, at = _directives(lines) + escape = parsed.get("escape", "\\") + # The escape character ending a line, spaces or tabs after it, joins the next line on as it is. + continued = re.compile(re.escape(escape) + r"[ \t]*$") + found = [Instruction(None, "", line) for line in lines[:at]] + while at < len(lines): + line = lines[at] + if not line.strip() or line.lstrip().startswith("#"): + found.append(Instruction(None, "", line)) + at += 1 + continue + text, parts = [], [] + while at < len(lines): + line = lines[at] + text.append(line) + at += 1 + if parts and (not line.strip() or line.lstrip().startswith("#")): + continue + body = line.rstrip("\r\n") + if not (end := continued.search(body)): + parts.append(body) + break + parts.append(body[:end.start()]) + keyword, *rest = "".join(parts).split(None, 1) + keyword, arguments = keyword.upper(), rest[0].strip() if rest else "" + heredocs, complete = [], True + if keyword in _HEREDOC_KEYWORDS: + for strip_tabs, name in _heredoc_names(arguments): + body, closed = [], False + while at < len(lines): + line = lines[at] + text.append(line) + at += 1 + if (line.rstrip("\r\n").lstrip("\t") if strip_tabs else line.rstrip("\r\n")) == name: + closed = True + break + body.append(line) + heredocs.append("".join(body)) + complete = complete and closed + found.append(Instruction(keyword, arguments, "".join(text), tuple(heredocs), complete)) + return found + + +def directives(dockerfile: str) -> dict[str, str]: + """The parser directives ``dockerfile`` opens with, by lower-case name.""" + return _directives(dockerfile.splitlines())[0] + + +def _directives(lines: list[str]) -> tuple[dict[str, str], int]: + """The parser directives ``lines`` open with, and how many lines they take.""" + found, at = {}, 0 + while at < len(lines) and (directive := _DIRECTIVE.fullmatch(lines[at].strip())): + found[directive.group(1).lower()] = directive.group(2) + at += 1 + return found, at + + +def leading_flags(arguments: str) -> tuple[list[str], str]: + """The ``--flag`` words ``arguments`` begins with, split on any whitespace outside quotes, and the rest of it.""" + flags, at = [], 0 + while arguments.startswith("--", at): + end, quote = at, None + while end < len(arguments) and (quote or not arguments[end].isspace()): + char = arguments[end] + quote = (None if char == quote else quote) if quote else (char if char in "\"'" else None) + end += 1 + flags.append(arguments[at:end]) + at = end + while at < len(arguments) and arguments[at].isspace(): + at += 1 + return flags, arguments[at:] + + +def _json_strings(arguments: str) -> list[str] | None: + """``arguments`` as an exec-form JSON list of strings; None when ``docker build`` reads it as shell form.""" + if not arguments.startswith("["): + return None + try: + parsed = json.loads(arguments) + except ValueError: + return None + return parsed if isinstance(parsed, list) and all(isinstance(arg, str) for arg in parsed) else None + + +def opens_heredoc(word: str) -> bool: + """Whether ``word``, written in a shell-form instruction, would open a heredoc.""" + return _heredoc(word) is not None + + +def _heredoc_names(arguments: str) -> list[tuple[str, str]]: + """Each heredoc ``arguments`` opens, as (its ``-``, its delimiter): each word that opens one, as BuildKit reads it.""" + return [heredoc for word in _words(arguments) if (heredoc := _heredoc(word))] + + +def _heredoc(word: str) -> tuple[str, str] | None: + """The heredoc ``word`` opens, as (its ``-``, its delimiter, unquoted); None when it opens none.""" + if not (match := _HEREDOC.fullmatch(word)): + return None + delimiter, quote, at = [], None, 0 + while at < len(match.group(2)): + char = match.group(2)[at] + if char == "\\" and quote != "'" and at + 1 < len(match.group(2)): + at += 1 + delimiter.append(match.group(2)[at]) + elif char == quote: + quote = None + elif char in "\"'" and quote is None: + quote = char + else: + delimiter.append(char) + at += 1 + return (match.group(1), "".join(delimiter)) if delimiter else None + + +def _words(arguments: str) -> list[str]: + """``arguments`` split into words on whitespace outside quotes, each as written, quotes and escapes kept.""" + words, word, quote, at = [], [], None, 0 + while at < len(arguments): + char = arguments[at] + if char == "\\" and quote != "'" and at + 1 < len(arguments): + word.append(arguments[at:at + 2]) + at += 2 + continue + if quote: + quote = None if char == quote else quote + elif char in "\"'": + quote = char + elif char.isspace(): + if word: + words.append("".join(word)) + word = [] + at += 1 + continue + word.append(char) + at += 1 + return words + (["".join(word)] if word else []) + + +def copy_arguments(instruction: Instruction) -> CopyArguments | None: + """A COPY or ADD's arguments; None for another instruction, a quoted shell-form path, or fewer than two paths.""" + if instruction.keyword not in ("COPY", "ADD"): + return None + flags, rest = leading_flags(instruction.arguments) + args = _json_strings(rest) + json_form = args is not None + if not json_form: + if any(quote in rest for quote in "\"'"): + return None + args = rest.split() + if len(args) < 2: + return None + sources = tuple(source for source in args[:-1] if json_form or _heredoc(source) is None) + return CopyArguments(instruction.keyword, tuple(flags), sources, args[-1], json_form, + heredoc=len(sources) < len(args) - 1) + + +def context_reads(dockerfile: str) -> ContextReads: + """The context paths a build of ``dockerfile`` reads: its local COPY and ADD sources and bind-mount sources.""" + if (syntax := directives(dockerfile).get("syntax")) and not _DOCKERFILE_SYNTAX.fullmatch(syntax): + return ContextReads(whole=f"# syntax={syntax} isn't followed") + paths: list[str] = [] + for instruction in instructions(dockerfile): + if not instruction.complete: + return ContextReads(whole=f"a heredoc in {instruction.keyword} never ends, so the rest can't be read") + if instruction.keyword in ("COPY", "ADD"): + reads = _copy_reads(instruction) + elif instruction.keyword == "RUN": + reads = _mount_reads(instruction) + else: + continue + if reads.whole: + return reads + paths.extend(reads.paths) + return ContextReads(paths=tuple(dict.fromkeys(paths))) + + +def _copy_reads(instruction: Instruction) -> ContextReads: + """The context paths a COPY or ADD reads.""" + args = copy_arguments(instruction) + if args is None: + return ContextReads(whole=f"{instruction.keyword} {instruction.arguments} can't be read") + if any(flag.startswith("--from=") for flag in args.flags): + return ContextReads() + if unknown := next((flag for flag in args.flags if not flag.startswith(_SOURCE_KEEPING_FLAGS)), None): + return ContextReads(whole=f"{args.keyword} {unknown} isn't followed") + paths = [] + for source in args.sources: + if args.keyword == "ADD" and ("://" in source or source.startswith("git@")): + continue + if why := _unread(source): + return ContextReads(whole=f"{args.keyword} {source} {why}") + if (path := _context_path(source)) is None: + return ContextReads(whole=f"{args.keyword} {source} copies all of it") + paths.append(path) + return ContextReads(paths=tuple(paths)) + + +def _mount_reads(instruction: Instruction) -> ContextReads: + """The context paths a RUN's bind mounts read.""" + paths = [] + for flag in leading_flags(instruction.arguments)[0]: + if not flag.startswith("--mount="): + continue + # A mount's options are one CSV record, as docker build reads them, so a quoted field holds a comma or space. + options = {} + for field in next(csv.reader([flag.removeprefix("--mount=")])): + key, _, value = field.partition("=") + options[key.strip().lower()] = value.strip().strip("\"'") + kind = options.get("type", "bind").lower() + if kind in _OFF_CONTEXT_MOUNTS or (kind == "bind" and "from" in options): + continue + if kind != "bind": + return ContextReads(whole=f"RUN --mount type={kind} isn't followed") + source = options.get("source") or options.get("src") or "." + if why := _unread(source): + return ContextReads(whole=f"RUN --mount source {source} {why}") + if (path := _context_path(source)) is None: + return ContextReads(whole=f"RUN --mount binds {source}, all of it") + paths.append(path) + return ContextReads(paths=tuple(paths)) + + +def _unread(source: str) -> str | None: + """Why ``source`` can't be read before a build, as docker build expands and unescapes it first; None when it can.""" + if "$" in source: + return "names a variable" + if "\\" in source or "`" in source: + return "holds an escape" + return None + + +def _context_path(source: str) -> str | None: + """``source``'s path in the context; None for the context itself, or a path that would leave it.""" + path = posixpath.normpath(source.lstrip("/")) + return None if path == "." or path == ".." or path.startswith("../") else path diff --git a/src/agent_env/utils/build_metadata.py b/src/agent_env/utils/docker/metadata.py similarity index 100% rename from src/agent_env/utils/build_metadata.py rename to src/agent_env/utils/docker/metadata.py diff --git a/src/agent_env/utils/docker/prune.py b/src/agent_env/utils/docker/prune.py new file mode 100644 index 00000000..1cad5878 --- /dev/null +++ b/src/agent_env/utils/docker/prune.py @@ -0,0 +1,283 @@ +"""The part of a stored build context each image's build reads, written into a folder of its own.""" + +from __future__ import annotations + +import bisect +import functools +import heapq +import os +import posixpath +import re +import shutil +import tarfile +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path, PurePosixPath + +from modal import FilePatternMatcher + +from agent_env.utils.docker.context import context_member, ignore_matcher, ignore_names +from agent_env.utils.docker.dockerfile import ContextReads, context_reads +from agent_env.utils.paths import LEAVINGS + +_WILDCARD = re.compile(r"[*?\[]") +_LARGEST = 5 + + +@dataclass(frozen=True) +class Pruned: + """What ``write_pruned`` wrote into one folder, and why it wrote all the context sends when it did.""" + + dockerfile: str + files: int + size: int + whole: str | None + largest: tuple[tuple[str, int], ...] = () + + +def write_pruned(archive: Path, images: Mapping[Path, str | None]) -> dict[Path, Pruned]: + """Write into each folder what its Dockerfile's build reads of the context tar.gz (None: the one it finds).""" + links, dockerfiles, texts = _read(archive, images) + plans = {folder: _plan(dockerfile, links, texts) for folder, dockerfile in dockerfiles.items()} + written = _write(archive, {folder: kept for folder, (kept, _) in plans.items()}) + return {folder: Pruned(dockerfiles[folder], len(files), sum(files.values()), plans[folder][1], + tuple(heapq.nlargest(_LARGEST, files.items(), key=lambda file: file[1]))) + for folder, files in written.items()} + + +def _read(archive: Path, + images: Mapping[Path, str | None]) -> tuple[dict[str, str | None], dict[Path, str], dict[str, str]]: + """The archive's names to their link targets, each image's Dockerfile, and its Dockerfile and ignore file texts.""" + dockerfiles = {folder: posixpath.normpath(dockerfile) for folder, dockerfile in images.items() if dockerfile} + wanted = {name for dockerfile in dockerfiles.values() for name in (dockerfile, *ignore_names(dockerfile))} + links: dict[str, str | None] = {} + texts: dict[str, str] = {} + with tarfile.open(archive, "r:gz") as tar: + for member in tar: + if (name := _member_name(member)) is None: + continue + links[name] = member.linkname if member.issym() else None + if name in wanted and member.isreg(): + texts[name] = tar.extractfile(member).read().decode("utf-8", "replace") + for folder in images.keys() - dockerfiles.keys(): + dockerfiles[folder] = _found_dockerfile(links) + wanted |= {dockerfiles[folder], *ignore_names(dockerfiles[folder])} + if unread := {name for name in wanted if name in links and not links[name] and name not in texts}: + texts |= _texts(archive, unread) + # A Dockerfile or ignore file stored as a link is read where it leads, as docker build reads it. + linked = {name: _resolve(name, links)[0] for name in wanted if links.get(name)} + if unread := {target for target in linked.values() if target not in texts}: + texts |= _texts(archive, unread) + return links, dockerfiles, texts | {name: texts[target] for name, target in linked.items() if target in texts} + + +def _plan(dockerfile: str, links: Mapping[str, str | None], texts: Mapping[str, str]) -> tuple[set[str], str | None]: + """The names a build of ``dockerfile`` keeps, and why it keeps all the context sends when it does.""" + if dockerfile not in texts: + raise ValueError(f"the build context has no {dockerfile}") + ignore = next((name for name in ignore_names(dockerfile) if name in texts), None) + matcher = ignore_matcher(texts[ignore]) if ignore else None + reads, crossed = _through_links(context_reads(texts[dockerfile]), links) + # A source the reading found that names nothing here is likelier a misreading than a build that fails. + if missing := next((path for path in reads.paths if not any(_matches(path, name) for name in links)), None): + reads, crossed = ContextReads(whole=f"{missing} matches nothing in the context"), set() + return _kept(links, reads, dockerfile, matcher, crossed), reads.whole + + +def _write(archive: Path, kept: Mapping[Path, set[str]]) -> dict[Path, dict[str, int]]: + """Write each folder's kept names from the archive, and each folder's files by name with their sizes.""" + # By name, since a name the archive holds twice is written over: what's left is its last entry. + written: dict[Path, dict[str, int]] = {folder: {} for folder in kept} + with tarfile.open(archive, "r:gz") as tar: + for member in tar: + name = _member_name(member) + first: Path | None = None + for folder in (folder for folder, names in kept.items() if name in names): + if member.isreg() and first is not None: + _check_inside(folder, name) + (folder / name).parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(first, folder / name) + else: + folder.mkdir(parents=True, exist_ok=True) + tar.extract(member, folder, filter=_member) + first = folder / name if member.isreg() else None + if member.isreg(): + written[folder][name] = member.size + else: + written[folder].pop(name, None) + # A folder the archive only implies, as the parent of what it holds, has no member of its own. + for folder, names in kept.items(): + for name in names: + if not os.path.lexists(folder / name): + _check_inside(folder, name) + (folder / name).mkdir(parents=True, exist_ok=True) + return written + + +def _check_inside(folder: Path, name: str) -> None: + """Raise ValueError when ``name`` in ``folder``, through any link written there, would land outside it.""" + top = os.path.realpath(folder) + if os.path.commonpath([top, os.path.realpath(folder / name)]) != top: + raise ValueError(f"{name!r} in the build context leads out of {folder}") + + +def _member(member: tarfile.TarInfo, out: str) -> tarfile.TarInfo | None: + """``member`` as ``write_pruned`` writes it: a link as it is, wherever it leads, and at the time of writing.""" + if (member := context_member(member, out, links_out=True)) is None: + return None + # A stored context's times are fixed, and docker build skips sending a file whose time and size it has seen before. + return member.replace(mtime=None, deep=False) + + +def _found_dockerfile(links: Mapping[str, str | None]) -> str: + """The Dockerfile an image that names none builds: the context's ``Dockerfile``, else the only one in it.""" + if "Dockerfile" in links: + return "Dockerfile" + found = sorted(name for name in links if posixpath.basename(name) == "Dockerfile") + if len(found) != 1: + raise ValueError(f"the build context names no Dockerfile and holds {len(found)}" + f"{': ' + ', '.join(found) if found else ''}; name the one it builds") + return found[0] + + +def _texts(archive: Path, names: set[str]) -> dict[str, str]: + """The text of each regular file ``names`` names in the tar.gz at ``archive``.""" + texts = {} + with tarfile.open(archive, "r:gz") as tar: + for member in tar: + if member.isreg() and (name := _member_name(member)) in names: + texts[name] = tar.extractfile(member).read().decode("utf-8", "replace") + return texts + + +def _through_links(reads: ContextReads, links: Mapping[str, str | None]) -> tuple[ContextReads, set[str]]: + """``reads`` with each source that names nothing followed through the links on its path, and the links crossed.""" + if reads.whole is not None: + return reads, set() + paths, crossed = [], set() + for path in reads.paths: + if _WILDCARD.search(path) or any(_matches(path, name) for name in links): + paths.append(path) + continue + resolved, hops = _resolve(path, links) + paths.append(resolved) + crossed |= hops + return ContextReads(paths=tuple(paths)), crossed + + +def _resolve(path: str, links: Mapping[str, str | None]) -> tuple[str, set[str]]: + """``path`` with its links followed as docker build follows them, and the links crossed; as it is when they loop.""" + resolved, crossed = "", set() + for part in path.split("/"): + resolved = posixpath.join(resolved, part) + while links.get(resolved) is not None: + if resolved in crossed: + return path, set() + crossed.add(resolved) + resolved = _link_target(resolved, links) + return resolved, crossed + + +def _link_target(name: str, links: Mapping[str, str | None]) -> str: + """Where the link ``name`` leads, as docker build follows it in the context: ``/`` and ``..`` stop at its root.""" + target = links[name] + parts = [] if target.startswith("/") or "/" not in name else name.split("/")[:-1] + for part in target.split("/"): + if part == "..": + parts = parts[:-1] + elif part not in ("", "."): + parts.append(part) + return "/".join(parts) + + +def _kept(links: Mapping[str, str | None], reads: ContextReads, dockerfile: str, matcher: FilePatternMatcher | None, + crossed: set[str] = frozenset()) -> set[str]: + """The names in ``links``, and the folders they imply, that ``write_pruned`` keeps for ``reads``.""" + def sent(name: str) -> bool: + return not LEAVINGS.intersection(name.split("/")) and (matcher is None or not matcher(Path(name))) + + implied = {str(parent) for name in links for parent in PurePosixPath(name).parents if str(parent) != "."} + candidates = links.keys() | implied + if reads.whole is not None: + kept = {name for name in candidates if sent(name)} + else: + kept = {name for name in candidates if sent(name) and any(_matches(path, name) for path in reads.paths)} + # docker build reads the Dockerfile and ignore file where they lead, whatever the ignore file says. Both ignore files + # stay, since docker build's frontends differ on which one an empty .dockerignore leaves in effect. + for name in (dockerfile, *ignore_names(dockerfile)): + resolved, hops = _resolve(name, links) + kept |= {other for other in (name, resolved, *hops) if other in links} + # A link a source only crosses is needed for its path, not for all it leads to. + pending = [name for name in kept if links.get(name)] + kept |= {name for name in crossed if sent(name)} + names = sorted(links) + while pending: + # A target is followed through the links on its way, which stay for its path. + target, hops = _resolve(_link_target(pending.pop(), links), links) + kept |= {hop for hop in hops if sent(hop)} + for other in _at_or_under(names, target) if target else names: + if other not in kept and sent(other): + kept.add(other) + if links[other]: + pending.append(other) + return kept | {str(parent) for name in kept for parent in PurePosixPath(name).parents if str(parent) in links} + + +def _at_or_under(names: list[str], path: str) -> list[str]: + """The names in ``names``, sorted, that are ``path`` or under it.""" + at = bisect.bisect_left(names, path) + found = [path] if at < len(names) and names[at] == path else [] + start = bisect.bisect_left(names, f"{path}/") + end = bisect.bisect_left(names, f"{path}0") # "0" sorts right after "/" + return found + names[start:end] + + +def _matches(path: str, name: str) -> bool: + """Whether ``name`` is ``path`` or under it, each part matched as Docker matches a COPY source's.""" + want, have = path.split("/"), name.split("/") + return len(have) >= len(want) and all( + part == glob if not _WILDCARD.search(glob) else bool((pattern := _part_pattern(glob)) and pattern.fullmatch(part)) + for part, glob in zip(have, want)) + + +@functools.lru_cache(maxsize=1024) +def _part_pattern(glob: str) -> re.Pattern[str] | None: + """A path part's wildcard as docker build matches it; None when docker build refuses it as malformed.""" + out, at = [], 0 + while at < len(glob): + char = glob[at] + if char == "*": + out.append("[^/]*") + elif char == "?": + out.append("[^/]") + elif char == "[": + at += 1 + negated = at < len(glob) and glob[at] == "^" + at += negated + ranges = [] + while not (at < len(glob) and glob[at] == "]" and ranges): + # A class's characters as Go's filepath.Match reads them: a range's ends neither - nor ], low to high. + if at + 1 >= len(glob) or glob[at] in "-]": + return None + low = high = glob[at] + at += 1 + if glob[at] == "-": + if at + 2 >= len(glob) or glob[at + 1] in "-]": + return None + high, at = glob[at + 1], at + 2 + if low > high: + return None + ranges.append(re.escape(low) if low == high else f"{re.escape(low)}-{re.escape(high)}") + out.append(f"[{'^' if negated else ''}{''.join(ranges)}]") + else: + out.append(re.escape(char)) + at += 1 + return re.compile("".join(out)) + + +def _member_name(member: tarfile.TarInfo) -> str | None: + """``member``'s path in the context; None for the context's own folder. Raises ValueError for one leading out.""" + name = posixpath.normpath(member.name.lstrip("/")) + if name == ".." or name.startswith("../"): + raise ValueError(f"{member.name!r} in the build context leads out of it") + return None if name == "." else name diff --git a/tst/integration/env/envs/build_context_local_test.py b/tst/integration/env/envs/build_context_local_test.py index ad2604e4..dedc04dc 100644 --- a/tst/integration/env/envs/build_context_local_test.py +++ b/tst/integration/env/envs/build_context_local_test.py @@ -18,7 +18,7 @@ from agent_env.env.envs.mcp_server import MCPServerEnv from agent_env.providers.sandbox_providers import sandbox from agent_env.providers.sandbox_providers.local_sandbox import LocalSandbox -from agent_env.utils.docker_build import normalize_platform +from agent_env.utils.docker.build import normalize_platform from tst.integration.env.envs.server_env_local_test import local_stack as local_stack from tst.integration.store.local_parallel_deploy_test import _docker, _seed_bootstrap_envs diff --git a/tst/integration/utils/__init__.py b/tst/integration/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tst/integration/utils/docker/__init__.py b/tst/integration/utils/docker/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tst/integration/utils/docker/prune_test.py b/tst/integration/utils/docker/prune_test.py new file mode 100644 index 00000000..862aa6e3 --- /dev/null +++ b/tst/integration/utils/docker/prune_test.py @@ -0,0 +1,150 @@ +"""An image built from what ``write_pruned`` keeps holds what one built from the whole context does. Needs Docker.""" + +import hashlib +import io +import shutil +import subprocess +import tarfile +import uuid +from pathlib import Path + +import pytest + +from agent_env.utils.docker.prune import write_pruned +from tst.util.capabilities import missing_capability_reason + +pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] + +BASE = "mirror.gcr.io/library/python:3.12-slim@sha256:423ed6ab25b1921a477529254bfeeabf5855151dc2c3141699a1bfc852199fbf" +DOCKERFILE = f"""FROM {BASE} AS build +COPY tools/gen.sh /gen.sh +RUN sh /gen.sh > /built.txt + +FROM {BASE} +COPY --from=build /built.txt /out/built.txt +COPY app/ /app/ +COPY conf/*.cfg /etc/svc/ +ADD vendor.tar.gz /vendor/ +COPY current/ /release/ +COPY current/app.txt /through-link.txt +COPY lit/[!a].txt /lit/ +RUN --network=none --mount=type=bind,"source=my dir",target=/m cp /m/f /out/spaced.txt +RUN --mount=type=cache,target=/root/.cache --mount=type=BIND,source=scripts,target=/scripts cp /scripts/run.sh /out/run.sh +""" +# What the build reads, beside a context ten times bigger it never does. +CONTEXT = { + "server/real.Dockerfile": DOCKERFILE, + "lit/!.txt": "bang\n", "lit/a.txt": "a\n", "lit/b.txt": "b\n", "my dir/f": "spaced\n", + ".dockerignore": "app/tests\n**/*.pyc\n", + "tools/gen.sh": "echo generated\n", "app/main.py": "print('main')\n", "app/util.py": "u = 1\n", + "app/tests/test_main.py": "assert True\n", "app/cache.pyc": "compiled", "conf/a.cfg": "a=1\n", + "conf/b.cfg": "b=2\n", "conf/readme.md": "not a cfg\n", "scripts/run.sh": "exec python /app/main.py\n", + "releases/v2/app.txt": "release 2\n", "releases/v1/app.txt": "release 1\n", + **{f"websites/site{n}/index.html": f"

{n}

" * 2000 for n in range(10)}, +} +VENDOR = {"lib/vendored.py": "V = True\n"} + + +def _add(tar: tarfile.TarFile, name: str, data: bytes, mode: int = 0o644) -> None: + info = tarfile.TarInfo(name) + info.size, info.mode = len(data), mode + tar.addfile(info, io.BytesIO(data)) + + +def _stored_context(path: Path) -> Path: + """The context as a store might hold it, ignored files and all, with a vendored archive and a release link.""" + vendor = io.BytesIO() + with tarfile.open(fileobj=vendor, mode="w:gz") as tar: + for name, text in VENDOR.items(): + _add(tar, name, text.encode()) + with tarfile.open(path, "w:gz") as tar: + for name, text in CONTEXT.items(): + _add(tar, name, text.encode(), 0o755 if name.endswith(".sh") else 0o644) + _add(tar, "vendor.tar.gz", vendor.getvalue()) + for name, target in (("current", "releases/v2"), ("server/Dockerfile", "real.Dockerfile")): + link = tarfile.TarInfo(name) + link.type, link.linkname = tarfile.SYMTYPE, target + tar.addfile(link) + return path + + +def _build(context: Path, tag: str) -> None: + result = subprocess.run(["docker", "build", "-q", "-f", str(context / "server" / "Dockerfile"), "-t", tag, + str(context)], capture_output=True, text=True) + assert result.returncode == 0, result.stderr + + +def _image_files(tag: str) -> dict[str, tuple[int, str]]: + """Each file, folder and link the image holds under what the build wrote, by its path: its mode and content.""" + container = subprocess.run(["docker", "create", tag], capture_output=True, text=True, check=True).stdout.strip() + try: + exported = subprocess.run(["docker", "export", container], capture_output=True, check=True).stdout + finally: + subprocess.run(["docker", "rm", container], capture_output=True) + found = {} + with tarfile.open(fileobj=io.BytesIO(exported)) as tar: + for member in tar: + if member.name.split("/", 1)[0] not in ("app", "out", "etc", "vendor", "release", "through-link.txt", "lit"): + continue + content = hashlib.sha256(tar.extractfile(member).read()).hexdigest() if member.isreg() else member.linkname + found[member.name] = (member.mode, content) + return found + + +@pytest.fixture +def docker(): + if shutil.which("docker") is None or subprocess.run(["docker", "info"], capture_output=True).returncode != 0: + pytest.skip(missing_capability_reason("docker_daemon")) + tags = [] + yield tags + for tag in tags: + subprocess.run(["docker", "rmi", "-f", tag], capture_output=True) + + +def test_an_image_built_from_the_pruned_context_holds_what_one_from_the_whole_context_does(tmp_path, docker): + archive = _stored_context(tmp_path / "ctx.tar.gz") + full = tmp_path / "full" + with tarfile.open(archive) as tar: + tar.extractall(full, filter="tar") + + pruned = write_pruned(archive, {tmp_path / "pruned": "server/Dockerfile"}) + + kept = {str(path.relative_to(tmp_path / "pruned")) for path in (tmp_path / "pruned").rglob("*") + if path.is_file() or path.is_symlink()} + assert kept == {"server/Dockerfile", "server/real.Dockerfile", ".dockerignore", "tools/gen.sh", "app/main.py", + "app/util.py", "conf/a.cfg", "conf/b.cfg", "vendor.tar.gz", "current", "releases/v2/app.txt", + "scripts/run.sh", "lit/!.txt", "lit/a.txt", "my dir/f"} + assert pruned[tmp_path / "pruned"].whole is None + suffix = uuid.uuid4().hex[:8] + docker.extend([f"prune-full-{suffix}", f"prune-kept-{suffix}"]) + _build(full, docker[0]) + _build(tmp_path / "pruned", docker[1]) + + built = _image_files(docker[1]) + assert built == _image_files(docker[0]) + assert {"app/main.py", "out/built.txt", "out/run.sh", "etc/svc/a.cfg", "vendor/lib/vendored.py", + "release/app.txt", "through-link.txt", "lit/!.txt", "lit/a.txt", "out/spaced.txt"} <= set(built) + assert "lit/b.txt" not in built + assert not any(name.startswith("app/tests") or name.endswith(".pyc") for name in built) + + +def test_folders_of_one_name_written_from_two_stored_contexts_each_build_their_own_files(tmp_path, docker): + # docker build skips sending a file whose path, size and time match one it was sent before, as a stored context's + # fixed times would make them, and builds the old file. + name = f"gmail-{uuid.uuid4().hex[:8]}" + built = [] + for version in ("1", "2"): + archive = tmp_path / f"v{version}.tar.gz" + with tarfile.open(archive, "w:gz") as tar: + _add(tar, "Dockerfile", f"FROM {BASE}\nCOPY version.txt /version.txt\n".encode()) + _add(tar, "version.txt", f"version {version}\n".encode()) + folder = tmp_path / f"bundle-v{version}" / "envs" / name + write_pruned(archive, {folder: "Dockerfile"}) + docker.append(f"prune-{name}-v{version}") + result = subprocess.run(["docker", "build", "-q", "--no-cache", "-t", docker[-1], str(folder)], + capture_output=True, text=True) + assert result.returncode == 0, result.stderr + built.append(subprocess.run(["docker", "run", "--rm", docker[-1], "cat", "/version.txt"], capture_output=True, + text=True, check=True).stdout) + + assert built == ["version 1\n", "version 2\n"] diff --git a/tst/unit/artifact/test_docker_image_build_context.py b/tst/unit/artifact/test_docker_image_build_context.py index 98458069..0dfc0c07 100644 --- a/tst/unit/artifact/test_docker_image_build_context.py +++ b/tst/unit/artifact/test_docker_image_build_context.py @@ -12,7 +12,7 @@ from agent_env.artifact.store import get_artifact_store from agent_env.store.image_store.local_registry_image_store import LocalRegistryImageStore from agent_env.store.image_store.oci_registry_credentials import names_registry -from agent_env.utils.build_context import BuildContext +from agent_env.utils.docker.context import BuildContext SERVER = {"Dockerfile": "FROM python:3.12\nCOPY . /app\n", "src/app.py": "print(1)\n", ".dockerignore": "*.log\n", "debug.log": "noise"} diff --git a/tst/unit/bundle/materialize_test.py b/tst/unit/bundle/materialize_test.py index 172bfdab..102c0414 100644 --- a/tst/unit/bundle/materialize_test.py +++ b/tst/unit/bundle/materialize_test.py @@ -35,7 +35,7 @@ from agent_env.store.routing import disable_namespace_routing from agent_env.task import Task from agent_env.task_step.task_step import TaskStep -from agent_env.utils.build_context import BuildContext +from agent_env.utils.docker.context import BuildContext from tst.unit.bundle._support import DIGEST, RefusingStore, layout, local_builds, local_store, plan_of ROOT = "@local/~/triage" diff --git a/tst/unit/cli/build_platform_test.py b/tst/unit/cli/build_platform_test.py index 8b365d18..f4a217d6 100644 --- a/tst/unit/cli/build_platform_test.py +++ b/tst/unit/cli/build_platform_test.py @@ -9,7 +9,7 @@ from agent_env.cli import cli from agent_env.env.envs.website_browser import WEBSITE_BROWSER_IMAGE_TAG -from agent_env.utils.docker_build import DockerBuildError +from agent_env.utils.docker.build import DockerBuildError gateway = importlib.import_module("agent_env.cli.env.gateway") # the packages export the click commands service_db = importlib.import_module("agent_env.cli.env.service_db") diff --git a/tst/unit/cli/sandbox_url_relocation_test.py b/tst/unit/cli/sandbox_url_relocation_test.py index 2b774333..7ab59047 100644 --- a/tst/unit/cli/sandbox_url_relocation_test.py +++ b/tst/unit/cli/sandbox_url_relocation_test.py @@ -11,7 +11,7 @@ from click.testing import CliRunner from unittest.mock import MagicMock, patch -from agent_env.utils.docker_build import DockerBuildError +from agent_env.utils.docker.build import DockerBuildError def test_old_cli_path_no_longer_re_exports_the_helper(): diff --git a/tst/unit/providers/sandbox_providers/host_images_test.py b/tst/unit/providers/sandbox_providers/host_images_test.py index 7ad68d88..39f91348 100644 --- a/tst/unit/providers/sandbox_providers/host_images_test.py +++ b/tst/unit/providers/sandbox_providers/host_images_test.py @@ -12,7 +12,7 @@ from agent_env.providers.sandbox_providers import host_images from agent_env.providers.sandbox_providers.local_sandbox import LocalSandbox from agent_env.providers.sandbox_providers.sandbox import VmSandbox -from agent_env.utils.docker_build import DockerBuildError +from agent_env.utils.docker.build import DockerBuildError @pytest.fixture diff --git a/tst/unit/providers/sandbox_providers/modal_context_build_test.py b/tst/unit/providers/sandbox_providers/modal_context_build_test.py index f1e3ac15..310e653b 100644 --- a/tst/unit/providers/sandbox_providers/modal_context_build_test.py +++ b/tst/unit/providers/sandbox_providers/modal_context_build_test.py @@ -146,13 +146,14 @@ async def test_a_failed_modal_build_keeps_modals_pointer_to_its_build_log(local_ @pytest.mark.parametrize("line, built", [ ('COPY ["a.txt", "/x/a.txt"]', "COPY a.txt /x/a.txt"), + ('COPY\t["a.txt", "/t/"]', "COPY a.txt /t/"), ('copy --chown=1:1 ["a.txt", "sub", "/y/"]', "COPY --chown=1:1 a.txt sub /y/"), ("ADD a.txt /added/", "COPY a.txt /added/"), ("ADD sub \\\n /added/sub/", "COPY sub /added/sub/"), ("ADD *.txt /globbed/", "COPY *.txt /globbed/"), ('ADD ["a.txt", "/j/"]', "COPY a.txt /j/"), ("ADD --chmod=755 a.txt /c", "COPY --chmod=755 a.txt /c"), -], ids=["json-copy", "json-copy-flags", "add-file", "add-folder-continued", "add-glob", "json-add", "add-chmod"]) +], ids=["json-copy", "json-copy-tab", "json-copy-flags", "add-file", "add-folder-continued", "add-glob", "json-add", "add-chmod"]) def test_what_modal_cant_take_is_written_as_its_docker_equivalent(tmp_path, line, built): (tmp_path / "a.txt").write_text("a") (tmp_path / "sub").mkdir() @@ -161,12 +162,19 @@ def test_what_modal_cant_take_is_written_as_its_docker_equivalent(tmp_path, line f"FROM python\n{built}\nRUN true\n" +def test_a_comment_among_the_lines_of_a_rewritten_instruction_is_kept_above_it(tmp_path): + (tmp_path / "sub").mkdir() + + assert modal_image_build.modal_dockerfile("FROM python\nADD sub \\\n # the vendored copy\n /added/\n", + tmp_path) == "FROM python\n # the vendored copy\nCOPY sub /added/\n" + + @pytest.mark.parametrize("line", [ 'COPY ["with space.txt", "/z/"]', "ADD archive.tar.gz /unpacked/", "ADD https://example.com/f /f", "ADD --checksum=sha256:abc a.txt /c", "ADD missing.txt /m", "COPY --from=build /out /out", "COPY a.txt /x/", - 'ADD "a.txt" /q/', "ADD < Path: @@ -81,6 +81,16 @@ def test_a_dockerfiles_own_ignore_file_replaces_the_contexts(tmp_path): assert not any(entry.name.startswith("data") for entry in context.entries) +def test_the_ignore_file_is_read_as_docker_reads_it_its_bom_dropped_and_a_line_opening_with_a_hash_a_comment(tmp_path): + # docker build sends "# notes.txt" and leaves out a.log and "#raw.txt" for this ignore file. + root = _tree(tmp_path / "ctx", {**SERVER, ".dockerignore": "*.log\n# notes.txt\n #raw.txt\n", "a.log": "", + "# notes.txt": "", "#raw.txt": ""}) + + names = {entry.name for entry in BuildContext.of(root, root / "Dockerfile").entries} + + assert "# notes.txt" in names and "a.log" not in names and "#raw.txt" not in names + + def test_what_the_os_and_python_leave_behind_is_dropped(tmp_path): root = _tree(tmp_path / "ctx", {**SERVER, ".DS_Store": "", "app/__pycache__/main.cpython-312.pyc": ""}) diff --git a/tst/unit/utils/docker/dockerfile_test.py b/tst/unit/utils/docker/dockerfile_test.py new file mode 100644 index 00000000..d5f4d613 --- /dev/null +++ b/tst/unit/utils/docker/dockerfile_test.py @@ -0,0 +1,219 @@ +"""A Dockerfile read into instructions as docker build reads it, a COPY or ADD's arguments, and what it reads.""" + +import pytest + +from agent_env.utils.docker.dockerfile import (ContextReads, context_reads, copy_arguments, directives, instructions, + leading_flags) + +DOCKERFILE = """# syntax=docker/dockerfile:1 +FROM python:3.12-slim AS base +# a comment +RUN apt-get update \\ + # a comment among the continued lines + && apt-get install -y curl + +COPY --chown=1:1 a.txt \\ + b/ /app/ +RUN < /q\nRUN ["sh", "-c", "cat < /a\n' + 'COPY a.txt /a.txt\n') + + assert [(i.keyword, i.heredocs) for i in found if i.keyword] == [("FROM", ()), ("RUN", ()), ("RUN", ()), ("RUN", ()), + ("COPY", ())] + + +@pytest.mark.parametrize(("line", "heredocs"), [ + ("RUN < /x\n", ()), +], ids=["dotted-delimiter", "quoted-dotted", "file-descriptor", "no-space-before"]) +def test_a_heredoc_is_a_word_opening_with_lt_lt_and_its_delimiter_is_the_rest_of_that_word(line, heredocs): + # docker build closes < /out\nCOPY a.txt /a.txt\nword\n") + + assert [(i.keyword, i.heredocs) for i in found if i.keyword] == [("FROM", ()), ("RUN", ()), ("COPY", ()), + ("WORD", ())] + + +def test_a_heredoc_left_open_runs_to_the_end_and_says_so(): + found = [i for i in instructions("FROM x\nRUN < /q\n' + 'RUN ["sh", "-c", "cat < /out\nCOPY a.txt /a.txt\n") == ContextReads(paths=("a.txt",)) + + +@pytest.mark.parametrize(("line", "why"), [ + ("COPY \\[x].txt /x/", "COPY \\[x].txt holds an escape"), + ('COPY ["a\\\\b", "/x/"]', "COPY a\\b holds an escape"), + ("RUN --mount=type=bind,source=a\\b,target=/m ls", "RUN --mount source a\\b holds an escape"), +], ids=["shell", "json", "mount"]) +def test_a_source_holding_an_escape_keeps_all_of_the_context(line, why): + # docker build unescapes a source before matching it: COPY \[x].txt copies x.txt, not [x].txt. + assert context_reads(f"FROM x\n{line}\n") == ContextReads(whole=why) + + +def test_a_heredoc_left_open_reads_all_of_the_context_rather_than_miss_a_copy_it_swallowed(): + assert context_reads("FROM x\nRUN < Path: + """A context tar.gz holding ``files`` and ``links``, everything in it as another tool might have written it.""" + with tarfile.open(path, "w:gz") as tar: + for name, text in files.items(): + info = tarfile.TarInfo(name) + data = text.encode() + info.size, info.mode = len(data), 0o755 if name in executable else 0o644 + tar.addfile(info, io.BytesIO(data)) + for name, target in (links or {}).items(): + info = tarfile.TarInfo(name) + info.type, info.linkname = tarfile.SYMTYPE, target + tar.addfile(info) + return path + + +def _files(folder: Path) -> dict[str, str]: + return {str(path.relative_to(folder)): path.read_text() for path in sorted(folder.rglob("*")) + if path.is_file() and not path.is_symlink()} + + +SERVERS = { + "__init__.py": "", "BaseService/base.py": "base", "wheels/x.whl": "wheel", "gsuite/g.py": "g", + "gmail/Dockerfile": "FROM python\nCOPY __init__.py BaseService/ gsuite/ gmail/ wheels/ /app/\n", + "gmail/main.py": "gmail", "slack/Dockerfile": "FROM python\nCOPY BaseService/ /app/BaseService/\nCOPY slack/ /s/\n", + "slack/main.py": "slack", "websites/huge.bin": "x" * 4096, +} + + +def test_each_image_gets_only_what_its_dockerfile_reads_from_one_archive_they_share(tmp_path): + archive = _archive(tmp_path / "ctx.tar.gz", SERVERS, executable=("gmail/main.py",)) + + pruned = write_pruned(archive, {tmp_path / "gmail": "gmail/Dockerfile", tmp_path / "slack": "slack/Dockerfile"}) + + assert _files(tmp_path / "gmail") == {name: SERVERS[name] for name in ( + "BaseService/base.py", "__init__.py", "gmail/Dockerfile", "gmail/main.py", "gsuite/g.py", "wheels/x.whl")} + assert _files(tmp_path / "slack") == {name: SERVERS[name] for name in ( + "BaseService/base.py", "slack/Dockerfile", "slack/main.py")} + assert (tmp_path / "gmail" / "gmail" / "main.py").stat().st_mode & 0o111 + assert pruned[tmp_path / "slack"].files == 3 and pruned[tmp_path / "slack"].whole is None + assert pruned[tmp_path / "slack"].size == sum(len(SERVERS[name]) for name in ( + "BaseService/base.py", "slack/Dockerfile", "slack/main.py")) + + +def test_what_the_ignore_file_leaves_out_stays_out_and_a_dockerfiles_own_ignore_file_replaces_the_contexts(tmp_path): + files = { + "Dockerfile": "FROM python\nCOPY app/ /app/\n", ".dockerignore": "app/tests\n!app/tests/keep.py\n", + "app/main.py": "m", "app/tests/t.py": "t", "app/tests/keep.py": "k", + "sub/Dockerfile": "FROM python\nCOPY app/ /app/\n", "sub/Dockerfile.dockerignore": "# only notes left out\n**/*.md\n", + "app/notes.md": "n", + } + archive = _archive(tmp_path / "ctx.tar.gz", files) + + write_pruned(archive, {tmp_path / "root": "Dockerfile", tmp_path / "sub": "sub/Dockerfile"}) + + assert set(_files(tmp_path / "root")) == {"Dockerfile", ".dockerignore", "app/main.py", "app/tests/keep.py", + "app/notes.md"} + assert set(_files(tmp_path / "sub")) == {"sub/Dockerfile", "sub/Dockerfile.dockerignore", ".dockerignore", + "app/main.py", "app/tests/t.py", "app/tests/keep.py"} + + +def test_an_empty_dockerfiles_own_ignore_file_leaves_everything_in_and_both_ignore_files_stay(tmp_path): + # docker build's own frontend reads an empty Dockerfile.dockerignore as leaving nothing out; the docker/dockerfile:1 + # frontend reads the context's .dockerignore instead. Both files stay, so either frontend reads what it would. + files = {"Dockerfile": "FROM python\nCOPY app/ /app/\n", "Dockerfile.dockerignore": "", ".dockerignore": "**/*.md\n", + "app/main.py": "m", "app/notes.md": "n"} + archive = _archive(tmp_path / "ctx.tar.gz", files) + + write_pruned(archive, {tmp_path / "out": "Dockerfile"}) + + assert set(_files(tmp_path / "out")) == {"Dockerfile", "Dockerfile.dockerignore", ".dockerignore", "app/main.py", + "app/notes.md"} + + +def test_a_dockerfile_that_reads_all_of_the_context_gets_what_docker_would_send_and_says_why(tmp_path): + files = {"Dockerfile": "FROM python\nCOPY . /app\n", ".dockerignore": "*.log\n", "a.py": "a", "b.log": "b"} + archive = _archive(tmp_path / "ctx.tar.gz", files) + + pruned = write_pruned(archive, {tmp_path / "out": "Dockerfile"}) + + assert set(_files(tmp_path / "out")) == {"Dockerfile", ".dockerignore", "a.py"} + assert pruned[tmp_path / "out"].whole == "COPY . copies all of it" + + +def test_a_kept_link_keeps_what_it_points_at_and_one_leading_out_of_the_context_is_written_as_it_is(tmp_path): + files = {"Dockerfile": "FROM python\nCOPY current/ /app/\nCOPY lib/ /lib/\n", "v2/app.py": "2", "v1/app.py": "1", + "lib/a.py": "a"} + archive = _archive(tmp_path / "ctx.tar.gz", files, links={"current": "v2", "lib/out": "../../../etc/passwd", + "lib/abs": "/usr/share/zoneinfo/UTC"}) + + write_pruned(archive, {tmp_path / "out": "Dockerfile"}) + + assert set(_files(tmp_path / "out")) == {"Dockerfile", "v2/app.py", "lib/a.py"} + assert (tmp_path / "out" / "current").is_symlink() + # docker build copies a link as a link, wherever it leads. + assert [(tmp_path / "out" / "lib" / name).readlink().as_posix() for name in ("out", "abs")] == [ + "../../../etc/passwd", "/usr/share/zoneinfo/UTC"] + + +def test_a_link_is_followed_as_docker_build_follows_it_its_root_standing_for_slash_and_any_dotdot_above_it(tmp_path): + # docker build copies a.txt, conf/f.cfg and a.txt again for this Dockerfile. + files = {"Dockerfile": "FROM python\nCOPY sub/deep/up /x/up\nCOPY sub/abs/f.cfg /x/\nCOPY sub/top/a.txt /x/top\n", + "a.txt": "a", "b.txt": "b", "conf/f.cfg": "f", "conf/g.cfg": "g"} + archive = _archive(tmp_path / "ctx.tar.gz", files, links={"sub/deep/up": "../../../a.txt", "sub/abs": "/conf", + "sub/top": "../../../../.."}) + + pruned = write_pruned(archive, {tmp_path / "out": "Dockerfile"})[tmp_path / "out"] + + assert pruned.whole is None + assert set(_files(tmp_path / "out")) == {"Dockerfile", "a.txt", "conf/f.cfg"} + assert all((tmp_path / "out" / name).is_symlink() for name in ("sub/deep/up", "sub/abs", "sub/top")) + + +def test_nothing_is_written_through_a_link_that_leads_out_of_the_folder(tmp_path): + archive = tmp_path / "ctx.tar.gz" + with tarfile.open(archive, "w:gz") as tar: + dockerfile = b"FROM python\nCOPY . /app\n" + info = tarfile.TarInfo("Dockerfile") + info.size = len(dockerfile) + tar.addfile(info, io.BytesIO(dockerfile)) + link = tarfile.TarInfo("evil") + link.type, link.linkname = tarfile.SYMTYPE, str(tmp_path / "elsewhere") + tar.addfile(link) + through = tarfile.TarInfo("evil/planted") + through.size = 1 + tar.addfile(through, io.BytesIO(b"x")) + + with pytest.raises(ValueError, match="evil/planted"): + write_pruned(archive, {tmp_path / "out": "Dockerfile"}) + assert not (tmp_path / "elsewhere" / "planted").exists() + + +def test_a_source_reached_through_a_link_keeps_the_link_and_what_it_leads_to(tmp_path): + files = {"Dockerfile": "FROM python\nCOPY current/app.py /app.py\n", "releases/v2/app.py": "2", + "releases/v1/app.py": "1"} + archive = _archive(tmp_path / "ctx.tar.gz", files, links={"current": "stable", "stable": "releases/v2"}) + + pruned = write_pruned(archive, {tmp_path / "out": "Dockerfile"}) + + assert set(_files(tmp_path / "out")) == {"Dockerfile", "releases/v2/app.py"} + assert (tmp_path / "out" / "current").is_symlink() and (tmp_path / "out" / "stable").is_symlink() + assert pruned[tmp_path / "out"].whole is None + + +def test_a_link_loop_or_one_out_of_the_context_in_a_source_keeps_all_of_it(tmp_path): + files = {"Dockerfile": "FROM python\nCOPY loop/a.py /a.py\nCOPY out/b.py /b.py\n", "other.py": "o"} + archive = _archive(tmp_path / "ctx.tar.gz", files, links={"loop": "loop2", "loop2": "loop", "out": "../elsewhere"}) + + pruned = write_pruned(archive, {tmp_path / "out": "Dockerfile"}) + + assert pruned[tmp_path / "out"].whole == "loop/a.py matches nothing in the context" + + +@pytest.mark.parametrize(("glob", "kept"), [ + ("[!a].txt", {"!.txt", "a.txt"}), ("[^a].txt", {"!.txt", "b.txt"}), ("[a-b].txt", {"a.txt", "b.txt"}), + ("[!-a].txt", {"!.txt", "a.txt"}), ("?.txt", {"!.txt", "a.txt", "b.txt"}), +], ids=["bang-is-a-character", "caret-negates", "range", "range-from-bang", "any-one"]) +def test_a_wildcard_class_reads_as_gos_filepath_match_reads_it(tmp_path, glob, kept): + files = {"Dockerfile": f"FROM python\nCOPY {glob} /x/\n", "!.txt": "!", "a.txt": "a", "b.txt": "b", "[x].txt": "x"} + archive = _archive(tmp_path / "ctx.tar.gz", files) + + write_pruned(archive, {tmp_path / "out": "Dockerfile"}) + + assert set(_files(tmp_path / "out")) == {"Dockerfile", *kept} + + +@pytest.mark.parametrize("glob", ["[a", "[z-a].txt", "[^z-a].txt", "[-a].txt", "[a-].txt", "[a-b-c].txt", "[]a].txt"], + ids=["unclosed", "reversed", "negated-reversed", "leading-dash", "trailing-dash", "two-ranges", + "leading-bracket"]) +def test_a_wildcard_docker_build_refuses_keeps_all_of_the_context_rather_than_fail(tmp_path, glob): + # docker build refuses each of these as a syntax error in the pattern. + files = {"Dockerfile": f"FROM python\nCOPY {glob} /x/\n", "a.txt": "a", "b.txt": "b", "z.txt": "z"} + archive = _archive(tmp_path / "ctx.tar.gz", files) + + assert write_pruned(archive, {tmp_path / "out": "Dockerfile"})[tmp_path / "out"].whole == ( + f"{glob} matches nothing in the context") + + +def test_a_dockerfile_stored_as_a_link_is_read_where_it_leads(tmp_path): + files = {"docker/real.Dockerfile": "FROM python\nCOPY app/ /app/\n", "app/main.py": "m", "other.py": "o"} + archive = _archive(tmp_path / "ctx.tar.gz", files, links={"Dockerfile": "docker/real.Dockerfile"}) + + pruned = write_pruned(archive, {tmp_path / "out": "Dockerfile"}) + + assert set(_files(tmp_path / "out")) == {"docker/real.Dockerfile", "app/main.py"} + assert (tmp_path / "out" / "Dockerfile").is_symlink() and pruned[tmp_path / "out"].whole is None + + +def test_wildcards_match_one_part_of_a_path_as_dockers_do(tmp_path): + files = {"Dockerfile": "FROM python\nCOPY *.txt [^b]*.cfg /x/\n", "a.txt": "a", "sub/c.txt": "c", "a.cfg": "a", + "b.cfg": "b"} + archive = _archive(tmp_path / "ctx.tar.gz", files) + + write_pruned(archive, {tmp_path / "out": "Dockerfile"}) + + assert set(_files(tmp_path / "out")) == {"Dockerfile", "a.txt", "a.cfg"} + + +def test_what_the_os_or_python_leaves_behind_in_a_copied_folder_is_left_out(tmp_path): + files = {"Dockerfile": "FROM python\nCOPY lib/ /lib/\n", "lib/mod.py": "m", "lib/__pycache__/mod.cpython-312.pyc": "c", + "lib/.DS_Store": "d"} + archive = _archive(tmp_path / "ctx.tar.gz", files) + + write_pruned(archive, {tmp_path / "out": "Dockerfile"}) + + assert set(_files(tmp_path / "out")) == {"Dockerfile", "lib/mod.py"} + + +def test_a_source_that_matches_nothing_in_the_context_keeps_all_of_it_and_says_which(tmp_path): + files = {"Dockerfile": "FROM python\nCOPY app/ /app/\nCOPY gone.txt /gone\n", "app/main.py": "m", "other.py": "o"} + archive = _archive(tmp_path / "ctx.tar.gz", files) + + pruned = write_pruned(archive, {tmp_path / "out": "Dockerfile"}) + + assert set(_files(tmp_path / "out")) == {"Dockerfile", "app/main.py", "other.py"} + assert pruned[tmp_path / "out"].whole == "gone.txt matches nothing in the context" + + +def test_a_chain_of_kept_links_keeps_what_its_last_link_points_at(tmp_path): + files = {"Dockerfile": "FROM python\nCOPY current /app\n", "releases/v3/app.py": "3", "releases/v2/app.py": "2"} + archive = _archive(tmp_path / "ctx.tar.gz", files, links={"current": "stable", "stable": "releases/v3"}) + + write_pruned(archive, {tmp_path / "out": "Dockerfile"}) + + assert set(_files(tmp_path / "out")) == {"Dockerfile", "releases/v3/app.py"} + assert (tmp_path / "out" / "stable").is_symlink() + + +def test_an_image_naming_no_dockerfile_builds_the_contexts_or_the_only_one_in_it(tmp_path): + root = _archive(tmp_path / "root.tar.gz", {"Dockerfile": "FROM python\nCOPY a /a\n", "a": "a", + "sub/Dockerfile": "FROM python\n"}) + nested = tmp_path / "nested.tar.gz" + with tarfile.open(nested, "w:gz") as tar: # the only Dockerfile, written three times as a stored context had it + for text in ("FROM python\n", "FROM python\nCOPY b /b\n", "FROM python\nCOPY app/ /app/\n"): + info = tarfile.TarInfo("claude_code/Dockerfile") + info.size = len(text) + tar.addfile(info, io.BytesIO(text.encode())) + info = tarfile.TarInfo("app/main.py") + info.size = 1 + tar.addfile(info, io.BytesIO(b"m")) + + from_root = write_pruned(root, {tmp_path / "root": None})[tmp_path / "root"] + from_nested = write_pruned(nested, {tmp_path / "nested": None})[tmp_path / "nested"] + + assert from_root.dockerfile == "Dockerfile" and set(_files(tmp_path / "root")) == {"Dockerfile", "a"} + assert from_nested.dockerfile == "claude_code/Dockerfile" + assert _files(tmp_path / "nested") == {"claude_code/Dockerfile": "FROM python\nCOPY app/ /app/\n", + "app/main.py": "m"} + + +@pytest.mark.parametrize(("files", "problem"), [ + ({"a/Dockerfile": "FROM x\n", "b/Dockerfile": "FROM x\n"}, "holds 2: a/Dockerfile, b/Dockerfile; name the one"), + ({"a.py": "a"}, "names no Dockerfile and holds 0; name the one it builds"), +], ids=["two", "none"]) +def test_an_image_naming_no_dockerfile_is_refused_when_the_context_doesnt_settle_which(tmp_path, files, problem): + archive = _archive(tmp_path / "ctx.tar.gz", files) + + with pytest.raises(ValueError, match=problem): + write_pruned(archive, {tmp_path / "out": None}) + + +def test_each_image_names_its_largest_files_for_the_report(tmp_path): + files = {"Dockerfile": "FROM python\nCOPY data/ /data/\n", + **{f"data/f{n}.bin": "x" * (100 * n) for n in range(1, 8)}, "big-unread.bin": "x" * 10_000} + archive = _archive(tmp_path / "ctx.tar.gz", files) + + pruned = write_pruned(archive, {tmp_path / "out": "Dockerfile"})[tmp_path / "out"] + + assert pruned.largest == (("data/f7.bin", 700), ("data/f6.bin", 600), ("data/f5.bin", 500), ("data/f4.bin", 400), + ("data/f3.bin", 300)) + assert pruned.dockerfile == "Dockerfile" + + +def test_a_name_the_archive_holds_twice_counts_once_at_the_size_left_in_the_folder(tmp_path): + archive = tmp_path / "ctx.tar.gz" + with tarfile.open(archive, "w:gz") as tar: + for name, text in (("Dockerfile", "FROM python\nCOPY data/ /data/\n"), ("data/a.bin", "x" * 5000), + ("data/b.bin", "x" * 100), ("data/a.bin", "x" * 10)): + info = tarfile.TarInfo(name) + info.size = len(text) + tar.addfile(info, io.BytesIO(text.encode())) + + pruned = write_pruned(archive, {tmp_path / "out": "Dockerfile"})[tmp_path / "out"] + + assert (tmp_path / "out" / "data" / "a.bin").stat().st_size == 10 + assert (pruned.files, pruned.size) == (3, 30 + 100 + 10) + assert pruned.largest == (("data/b.bin", 100), ("Dockerfile", 30), ("data/a.bin", 10)) + + +def test_the_dockerfiles_link_target_is_kept_even_when_the_ignore_file_leaves_it_out(tmp_path): + # docker build reads the Dockerfile and its ignore file where they lead, whatever the ignore file says. + files = {"docker/real.Dockerfile": "FROM python\nCOPY app/ /app/\n", "docker/Dockerfile.dockerignore": "docker\n", + "app/main.py": "m"} + archive = _archive(tmp_path / "ctx.tar.gz", files, links={"docker/Dockerfile": "real.Dockerfile"}) + + write_pruned(archive, {tmp_path / "out": "docker/Dockerfile"}) + + assert set(_files(tmp_path / "out")) == {"docker/real.Dockerfile", "docker/Dockerfile.dockerignore", "app/main.py"} + + +@pytest.mark.parametrize("dockerfile", ["FROM python\nCOPY . /app\n", "FROM python\nCOPY sub/ /app/\n"], + ids=["whole", "copied"]) +def test_a_folder_the_archive_only_implies_is_written_when_the_ignore_file_leaves_out_all_it_holds(tmp_path, dockerfile): + # docker build sends sub/ as an empty folder here, and COPY sub/ copies it. + archive = _archive(tmp_path / "ctx.tar.gz", {"Dockerfile": dockerfile, ".dockerignore": "sub/*.md\n", + "sub/notes.md": "n"}) + + write_pruned(archive, {tmp_path / "out": "Dockerfile"}) + + assert (tmp_path / "out" / "sub").is_dir() and not any((tmp_path / "out" / "sub").iterdir()) + + +def test_a_kept_links_target_is_followed_through_the_links_on_its_way(tmp_path): + # docker build copies releases/v2/app/main.py for COPY current here. + files = {"Dockerfile": "FROM python\nCOPY current /app\n", "releases/v2/app/main.py": "2", "releases/v1/app/old.py": "1"} + archive = _archive(tmp_path / "ctx.tar.gz", files, links={"current": "releases/latest/app", "releases/latest": "v2"}) + + write_pruned(archive, {tmp_path / "out": "Dockerfile"}) + + assert set(_files(tmp_path / "out")) == {"Dockerfile", "releases/v2/app/main.py"} + assert (tmp_path / "out" / "current").is_symlink() and (tmp_path / "out" / "releases" / "latest").is_symlink() + + +def test_no_folder_sharing_the_archive_is_written_through_a_link_that_leads_out_of_it(tmp_path): + # The link leads into the first folder's own safe/, but out of the second one, into a folder beside it. + (tmp_path / "b" / "one" / "safe").mkdir(parents=True) + archive = tmp_path / "ctx.tar.gz" + with tarfile.open(archive, "w:gz") as tar: + dockerfile = b"FROM python\nCOPY . /app\n" + info = tarfile.TarInfo("Dockerfile") + info.size = len(dockerfile) + tar.addfile(info, io.BytesIO(dockerfile)) + folder = tarfile.TarInfo("safe") + folder.type, folder.mode = tarfile.DIRTYPE, 0o755 + tar.addfile(folder) + link = tarfile.TarInfo("dir") + link.type, link.linkname = tarfile.SYMTYPE, "../one/safe" + tar.addfile(link) + planted = tarfile.TarInfo("dir/file") + planted.size = 1 + tar.addfile(planted, io.BytesIO(b"x")) + + with pytest.raises(ValueError, match="'dir/file' in the build context leads out of"): + write_pruned(archive, {tmp_path / "a" / "one": "Dockerfile", tmp_path / "b" / "two": "Dockerfile"}) + assert not (tmp_path / "b" / "one" / "safe" / "file").exists() + + +def test_a_member_whose_name_leads_out_of_the_context_is_refused_before_anything_is_written(tmp_path): + archive = _archive(tmp_path / "ctx.tar.gz", {"Dockerfile": "FROM python\nCOPY . /app\n", + "../outside/__pycache__/payload": "x"}) + + with pytest.raises(ValueError, match="leads out of it"): + write_pruned(archive, {tmp_path / "out" / "here": "Dockerfile"}) + assert not (tmp_path / "out" / "outside").exists() and not (tmp_path / "out" / "here").exists() + + +def test_each_file_is_written_at_the_time_of_writing_not_the_stored_contexts(tmp_path): + # A stored context's files carry the time 0; docker build would read a rewritten one as unchanged. + archive = _archive(tmp_path / "ctx.tar.gz", SERVERS) + before = time.time() + + write_pruned(archive, {tmp_path / "gmail": "gmail/Dockerfile", tmp_path / "slack": "slack/Dockerfile"}) + + for name in ("gmail/BaseService/base.py", "slack/BaseService/base.py", "slack/slack/main.py"): + assert (tmp_path / name).stat().st_mtime >= before - 1 + + +def test_a_dockerfile_missing_from_the_archive_is_refused(tmp_path): + archive = _archive(tmp_path / "ctx.tar.gz", {"a.py": "a"}) + + with pytest.raises(ValueError, match="the build context has no gmail/Dockerfile"): + write_pruned(archive, {tmp_path / "out": "gmail/Dockerfile"}) + + +def test_a_member_no_context_holds_is_refused(tmp_path): + archive = tmp_path / "ctx.tar.gz" + with tarfile.open(archive, "w:gz") as tar: + dockerfile = b"FROM python\nCOPY . /app\n" + info = tarfile.TarInfo("Dockerfile") + info.size = len(dockerfile) + tar.addfile(info, io.BytesIO(dockerfile)) + hard = tarfile.TarInfo("hard") + hard.type, hard.linkname = tarfile.LNKTYPE, "Dockerfile" + tar.addfile(hard) + + with pytest.raises(ValueError, match="neither a file, a folder nor a link"): + write_pruned(archive, {tmp_path / "out": "Dockerfile"})