diff --git a/.env.example b/.env.example index b1efe88..2dfefc2 100644 --- a/.env.example +++ b/.env.example @@ -63,13 +63,17 @@ OPENROUTER_API_KEY= JINA_API_KEY= # --------------------------------------------------------------------------- -# 5. Container proxy (not supported by current restricted task policies) +# 5. Runtime egress (see docs/network-policy.md) # --------------------------------------------------------------------------- -# Leave these empty. A general proxy can bypass Harbor's destination allowlist. +# General container proxies bypass allowlists; leave empty for restricted tasks. CONTAINER_PROXY= CONTAINER_NO_PROXY= -# Optional: comma-separated reachable upstream IPv4 DNS servers for the Docker -# isolation sidecar. Bypasses an unreliable host DNS stub without opening HTTP -# egress; only DNS port 53 to these IPs is exempted in allowlist mode. +# Optional direct-mode upstream IPv4 DNS servers, comma-separated. CONTAINER_DNS= +# Optional image override; defaults to hanhainebula/search-swe-egress:1.0.0. +# Pulled automatically if missing locally. +EGRESS_IMAGE= +# Optional direct/proxy JSON path. Requires CONTAINER_DNS, CONTAINER_PROXY and +# EGRESS_IMAGE to be empty. Leave empty for default direct access. +EGRESS_CONFIG= diff --git a/README.md b/README.md index 3b75a15..cd1506a 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,9 @@ For other runs, fill only the matching sections already present in `.env`: `OPENROUTER_API_KEY`, or `JINA_API_KEY` only for the tasks identified by the comments in `.env.example`. +If a proxy is required, set `EGRESS_CONFIG` in `.env` following the +[network guide](docs/network-policy.md); otherwise leave it empty. + The [evaluation guide](docs/evaluation.md) documents credential isolation, custom endpoints, proxies, and the complete per-task matrix. @@ -194,7 +197,7 @@ hardware matrix plus GPU, network-policy, and custom-provider options. | --- | --- | | [Quick start guide](docs/quickstart.md) | A first CPU evaluation, end to end | | [Evaluation guide](docs/evaluation.md) | Per-task credentials, coding agents, GPU, network policy, and custom providers | -| [Network policy](docs/network-policy.md) | Harbor egress modes and exact per-task host allowlists | +| [Network policy](docs/network-policy.md) | Per-task allowlists, default direct gateway, and optional proxy egress | | [Asset guide](docs/assets.md) | Downloading, verifying, and restoring fixed data and models | | [Benchmark design](docs/benchmark.md) | Evaluation, repository layout, and data provenance | | [Contributing guide](docs/contributing.md) | Task-authoring workflow, validation, and PR expectations | diff --git a/README_zh.md b/README_zh.md index ba9c077..1962237 100644 --- a/README_zh.md +++ b/README_zh.md @@ -98,6 +98,8 @@ VERIFIER_OPENAI_API_KEY=YOUR_DEEPSEEK_KEY - 可选 submission API：只在 `.env.example` 注释所列任务确实使用时，填写 `TASK_1_1_OPENROUTER_API_KEY`、`OPENROUTER_API_KEY` 或 `JINA_API_KEY`。 +需要代理时，按[网络配置](docs/network-policy.md)设置 `.env` 的 `EGRESS_CONFIG`，否则留空。 + 凭证隔离、自定义服务地址、网络权限和完整的逐任务配置矩阵见 [评测指南](docs/evaluation.md)。 @@ -182,7 +184,7 @@ gateway、订阅 OAuth、Bedrock、Vertex、ACP 和自定义 Claude settings。 | --- | --- | | [快速开始指南](docs/quickstart.md) | 完整的一次 CPU 评测流程 | | [评测指南](docs/evaluation.md) | 各任务凭证、编码智能体、GPU、网络策略和自定义模型服务 | -| [网络权限](docs/network-policy.md) | Harbor 网络模式与各任务精确的 host allowlist | +| [网络权限](docs/network-policy.md) | 各任务的 host allowlist、默认直连网关与可选代理出口 | | [资源说明](docs/assets.md) | 固定数据与模型的下载、校验和恢复 | | [基准设计](docs/benchmark.md) | 评测方式、仓库结构和数据来源 | | [贡献指南](docs/contributing.md) | 任务创作流程、验证要求和 PR 说明 | diff --git a/docs/evaluation.md b/docs/evaluation.md index b3c77d9..d7fad87 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -260,13 +260,16 @@ Compose overlays request the real GPU. Harbor 0.22.0's Docker backend rejects `gpus = 1` during its own preflight and does not translate that field into the Compose GPU request. The shared launcher therefore supplies `--override-gpus 0` automatically while leaving the truthful -task metadata and Compose reservations intact. If invoking `harbor run --env -docker` directly with that Harbor version, add the same override. Recheck this -workaround when upgrading Harbor. +task metadata and Compose reservations intact. Direct Harbor invocations using +`--env scripts.harbor_environments:PhaseScopedDocker` need the same override +with that Harbor version. Recheck this workaround when upgrading Harbor. ## Runtime network enforcement -Every current task uses a restricted agent or verifier phase. Keep +Every current task uses a restricted agent or verifier phase, so the launcher +automatically selects the direct gateway and pulls its image if missing. For an +upstream HTTP(S) proxy, configure `EGRESS_CONFIG` using the +[network guide](network-policy.md); proxy mode rejects public phases. Keep `CONTAINER_PROXY` unset: a general proxy would let the proxy choose arbitrary destinations and would defeat Harbor's hostname policy, so the launcher rejects it. Configure image-pull and Docker build proxies separately at the Docker diff --git a/docs/network-policy.md b/docs/network-policy.md index 98388d5..d0de862 100644 --- a/docs/network-policy.md +++ b/docs/network-policy.md @@ -1,10 +1,111 @@ # Runtime network policy -Search-SWE uses Harbor 0.22.0's native `network_mode` and `allowed_hosts` -fields. `allowed_hosts` contains hostnames only—not URLs, ports, or paths—and is -valid only with `network_mode = "allowlist"`. Docker enforcement uses Harbor's -egress-control sidecar. If the Docker host cannot enforce the requested policy, -Harbor rejects the run instead of silently granting public access. +Search-SWE uses Harbor 0.22.0's `network_mode` and `allowed_hosts` fields. +Allowlists contain exact lowercase hostnames, not URLs, ports or paths, and +apply separately to each task and execution phase. Restricted tasks use an +independent trusted gateway for each agent/verifier environment, including when +an upstream proxy is configured. CPU/GPU task images remain unchanged. + +## Default: direct access + +No extra setup is required. The launcher uses +`hanhainebula/search-swe-egress:1.0.0`, pulls it if missing locally, and reuses it +on later runs. Leave `EGRESS_IMAGE` and `EGRESS_CONFIG` empty. `EGRESS_IMAGE` or +`--egress-image` can select a matching local build or repository digest. +Images are validated against this checkout before task containers start; +pull failures or incompatible images stop startup. Downloads use the Docker +daemon's network/proxy settings, not the task's proxy configuration. + +Direct mode uses Docker DNS through the trusted gateway. If needed, set +`CONTAINER_DNS` (or `--container-dns`) to comma-separated reachable IPv4 DNS +servers. Only allowed host queries are forwarded; task processes cannot query +upstream or Docker DNS directly. Changing DNS does not repair TLS or API errors. + +## Networks requiring a proxy + +Use an HTTP(S) proxy with CONNECT support and an HTTPS DNS-over-HTTPS (DoH) +endpoint accessible through it. Create a JSON file outside task mounts and +build contexts, replacing the example address and hostname below: + +```json +{ + "version": 1, + "mode": "proxy", + "image": "hanhainebula/search-swe-egress:1.0.0", + "upstream": {"url": "http://192.0.2.10:8080"}, + "dns": {"doh_url": "https://resolver.example/dns-query"} +} +``` + +Point `.env` to the file, leaving other overrides empty: + +```dotenv +EGRESS_CONFIG=/absolute/path/to/egress.json +EGRESS_IMAGE= +CONTAINER_DNS= +CONTAINER_PROXY= +``` + +`--egress-config PATH` overrides `EGRESS_CONFIG`. Relative CLI paths resolve +from the current directory; environment-variable paths resolve from the checkout. +Explicit config requires `image` and cannot be combined with image, DNS or general +container-proxy overrides. Do not add the proxy to task allowlists. + +The proxy must be reachable **from Docker**; host `127.0.0.1` is not the host +inside a container. There is no fixed VPN product/port or automatic discovery. +SOCKS-only and TUN-only endpoints are not supported by this configuration. +For a hostname URL such as `https://proxy.example:8443`, also set +`upstream.address` to its reachable IPv4 address. Certificates are validated +against the URL hostname; custom CAs and client certificates are unsupported. +Loopback, link-local and task-local endpoints are rejected. Proxy URLs cannot +contain credentials, queries or fragments; DoH URLs cannot contain query parameters. + +For proxy authentication, set `upstream.auth_file` to a JSON file containing +`{"username": "...", "password": "..."}`. Keep it outside task mounts/build +contexts, owned by the launcher user, with mode `0600` and parent mode `0700`. +Relative auth paths resolve from the config file; symlinks/hard links are rejected. +Only the trusted gateway receives these credentials. + +Explicit direct config uses `"mode": "direct"`, no `upstream`, and +`"dns": {"servers": ["192.0.2.53"]}` with a reachable resolver (an optional UDP +port is accepted). An empty `dns` object selects Docker DNS. `version` and +`image` are still required. The launcher's `--dry-run` checks policy/config +compatibility without pulling images, reading proxy credentials or calling APIs. + +## Supported runtime + +- Native Linux amd64, local rootful Docker through a Unix socket, Harbor 0.22.0, + and IPv4. Docker Desktop, remote/rootless Docker and userns remapping are unsupported. +- Direct mode supports public and restricted phases. **Proxy mode rejects any + public phase**. +- Custom network topologies, extra capabilities/devices, host/control mounts, + external/shared volumes and kept containers are rejected. Named volumes must + be project-local plain Docker volumes. +- Authorization checks HTTP destinations and TLS SNI, not encrypted paths, + bodies or Host headers/domain fronting at an approved origin. + +## Isolation and recovery + +The gateway starts in no-network mode, applies the phase baseline, then allows +task services to start. Task services cannot modify the firewall or forge trusted +traffic marks. Policy changes revoke old connections. Proxy, DNS or control +failure closes egress; there is no unrestricted fallback. A 30-second host-renewed +lease bounds revocation after loss of host control; trusted host clocks are required. +An already accepted upstream API operation cannot be undone by revocation. + +Normal teardown removes each environment's owned resources and private settings, +while preserving images and host datasets. For leftovers after a failed cleanup: + +```bash +python scripts/egress_cleanup.py +python scripts/egress_cleanup.py --directory /path/from/list +python scripts/egress_cleanup.py --directory /path/from/list --remove +``` + +These list, preview and remove one verified inactive instance respectively. +Recovery refuses active owners, foreign resources or a different Docker daemon. +Host logs under `egress/.log` record policies and kernel counters; +these counters are not HTTP request counts. ## Current task matrix @@ -40,7 +141,8 @@ The repository launcher derives and supplies the coding-model host. When invoking Harbor directly on a non-public task, add the matching hostname: ```bash -harbor run --path tasks/TASK_ID --env docker \ +PYTHONPATH="$PWD${PYTHONPATH:+:$PYTHONPATH}" harbor run --path tasks/TASK_ID \ + --env scripts.harbor_environments:PhaseScopedDocker \ --agent scripts.harbor_agents:PreinstalledCodex --model MODEL_ID \ --ak version=0.147.0 \ --allow-agent-host MODEL_API_HOST @@ -50,7 +152,8 @@ For the supported official-Anthropic Claude Code mode, use the pinned wrapper, an environment reference rather than a literal secret, and the fixed host: ```bash -harbor run --path tasks/TASK_ID --env docker \ +PYTHONPATH="$PWD${PYTHONPATH:+:$PYTHONPATH}" harbor run --path tasks/TASK_ID \ + --env scripts.harbor_environments:PhaseScopedDocker \ --agent scripts.harbor_agents:PreinstalledClaudeCode \ --model ANTHROPIC_MODEL_ID \ --ak version=2.1.273 \ @@ -63,6 +166,11 @@ Bedrock selectors. Reproduce that sanitization when bypassing it; custom Anthropic endpoints and alternate Claude authentication modes are not part of the supported configuration. +These commands use the default direct gateway image and pull it if missing locally. +To select another matching image, add `--environment-kwarg egress_image=YOUR_IMAGE`. Manually using +`--env docker` selects installed Harbor and does not receive this checkout's +capability removal or old-connection revocation fixes. + Do not use `--allow-environment-host` for a coding-model endpoint: that changes the environment baseline rather than only the agent phase. Do not add package registries, source-code hosts, wildcard domains, or a general HTTP proxy to a @@ -74,29 +182,20 @@ Image pulls and Docker build downloads happen before untrusted task execution and are configured at the Docker daemon/build layer; they are not task runtime egress permissions. -## Optional upstream DNS for isolated Docker runs - -If Docker's embedded resolver intermittently times out forwarding to the host -DNS stub (for example, `127.0.0.53`), configure reachable upstream IPv4 DNS -servers in the local `.env`: - -```dotenv -CONTAINER_DNS=198.18.254.30,198.18.254.31 -``` - -These example addresses belong to the diagnosed host's network; do not assume -they work elsewhere. `--container-dns IP,IP` overrides the environment setting. -Leave it unset to retain Harbor's default DNS behavior. - -The launcher generates a Docker Compose overlay for the isolation sidecar, -shared by the task processes. It retains Harbor's original policy helper and -adds only TCP/UDP port 53 exceptions to the configured DNS IPs during nonempty -allowlist policies. `deny-all` removes those exceptions; ordinary API traffic -still uses the original hostname allowlist. This is not an HTTP proxy and does -not enable `CONTAINER_PROXY`. It neither changes host DNS nor restarts Docker. - -Overlays and a snapshot of the installed Harbor helper are retained in -`/tmp/searchswe-dns-*` for the lifetime of the run. Do not remove them while a -run is active (including its separate verifier); they may be removed afterward. -Changing DNS does not fix unrelated TLS, API rate-limit, or authentication -failures. Verify permitted and denied destinations after upgrading Harbor. +## Regression requirements + +The [build and regression guide](../environments/egress/README.md#verification) +covers these enforcement requirements: + +- Untrusted services cannot forge the gateway's firewall exemption marks or + modify its rules; capabilities and namespace boundaries must enforce this. +- Tightening a policy terminates previously authorized tunnels and pooled + connections, not merely rejects new connections. +- Agent, verifier, baseline, public, and no-network transitions are tested; + phase allowlists must not be replaced with their union. +- Destination resolution works through the intended upstream path, including + when local DNS returns NXDOMAIN, times out, or gives an incorrect address. +- Failure of the proxy, resolver, or policy update fails closed, without a + direct/unfiltered fallback. TLS certificate validation remains enabled. +- The upstream endpoint is operator-configured, with no machine-specific IP, + port, VPN product, or public DNS provider required by task packages. diff --git a/docs/quickstart.md b/docs/quickstart.md index b789e56..201435b 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -62,6 +62,9 @@ configuration groups and only the verifier receives the `VERIFIER_*` values. their keys are optional and are not needed for an implementation that uses only the provided corpus and local runtime. +If a proxy is required, set `EGRESS_CONFIG` in `.env` following the +[network guide](network-policy.md); otherwise leave it empty. + The local `.env` is ignored by Git. Do not commit or print credentials. On a multi-user Unix host, restrict it after adding credentials: diff --git a/environments/egress/Dockerfile b/environments/egress/Dockerfile new file mode 100644 index 0000000..30adbb2 --- /dev/null +++ b/environments/egress/Dockerfile @@ -0,0 +1,14 @@ +# Use scripts/build_egress_gateway.py; it stages and verifies pinned APKs. +ARG COMPONENT_IMAGE=searchswe-egress-component:build-required +FROM ${COMPONENT_IMAGE} +ARG GATEWAY_SOURCE_SHA +ARG RUNTIME_LOCK_SHA +COPY packages/ /tmp/searchswe-apks/ +RUN apk add --no-cache --no-network --repositories-file /dev/null /tmp/searchswe-apks/*.apk && rm -rf /tmp/searchswe-apks +COPY runtime.lock.json /usr/share/searchswe-egress/runtime.json +COPY gateway.py /opt/searchswe/gateway.py +COPY --chmod=755 entrypoint.sh /opt/egress-sidecar/entrypoint.sh +COPY --chmod=755 network-policy /usr/local/bin/network-policy +LABEL org.search-swe.egress.gateway="1" +LABEL org.search-swe.egress.source-sha256=${GATEWAY_SOURCE_SHA} +LABEL org.search-swe.egress.runtime-lock-sha256=${RUNTIME_LOCK_SHA} diff --git a/environments/egress/Dockerfile.component b/environments/egress/Dockerfile.component new file mode 100644 index 0000000..717a10a --- /dev/null +++ b/environments/egress/Dockerfile.component @@ -0,0 +1,19 @@ +# Build from the audited helper image, then publish only the merged filesystem. +# This excludes replaced GOST and OpenSSL binaries from all final OCI layers. +ARG BASE_IMAGE=searchswe-harbor-base:build-required +FROM ${BASE_IMAGE} AS assembled +COPY packages/ /tmp/searchswe-apks/ +RUN apk add --no-cache --no-network --repositories-file /dev/null /tmp/searchswe-apks/*.apk && rm -rf /tmp/searchswe-apks +COPY --chmod=755 gost-searchswe /bin/gost +COPY LICENSE.* /usr/share/licenses/searchswe-egress/ +COPY third-party/ /usr/share/licenses/searchswe-egress/third-party/ +COPY component-manifest.json /usr/share/searchswe-egress/component.json + +FROM scratch +COPY --from=assembled / / +ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +WORKDIR /bin/ +ENTRYPOINT ["/bin/gost"] +ARG PATCH_SHA +LABEL org.search-swe.egress.component="gost-x-0.10.9-searchswe-1" +LABEL org.search-swe.egress.patch-sha256=${PATCH_SHA} diff --git a/environments/egress/README.md b/environments/egress/README.md new file mode 100644 index 0000000..10ed67f --- /dev/null +++ b/environments/egress/README.md @@ -0,0 +1,88 @@ +# Gateway builds and verification + +For normal task runs, the launcher pulls the published gateway if it is missing +locally. Follow the +[direct/proxy configuration guide](../../docs/network-policy.md). + +The image is shared by CPU/GPU tasks; it contains the network controller and +patched GOST, not the task runtime. The following instructions are for maintainers +changing or rebuilding it. Builds require native Linux amd64, local rootful +Docker, Harbor 0.22.0, Go **1.26.8** and curl. Verification also uses OpenSSL. + +## Build from source + +First build the helper base from the pinned Harbor installation. Its Dockerfile +pins the upstream image by digest; Docker may need registry access for this step. + +```bash +SEARCHSWE_EGRESS_CONTEXT="$(python -c 'from importlib.metadata import version; assert version("harbor") == "0.22.0"; from harbor.environments.docker.docker import DockerEnvironment; print(DockerEnvironment._EGRESS_CONTROL_SIDECAR_CONTEXT_PATH)')" +docker build --platform linux/amd64 \ + --tag searchswe-egress-harbor:0.22.0 "$SEARCHSWE_EGRESS_CONTEXT" +``` + +Build the patched component, then the gateway. Use fresh output directories and +new tags for each build; the builders reject overwriting an existing image. + +```bash +python scripts/build_egress_component.py \ + --go /path/to/go1.26.8/bin/go \ + --base-image searchswe-egress-harbor:0.22.0 \ + --work-dir jobs/egress-build/component-local1 \ + --cache-dir jobs/egress-build/cache \ + --tag searchswe-egress-component:local1 + +python scripts/build_egress_gateway.py \ + --component-image searchswe-egress-component:local1 \ + --tag searchswe-egress:local1 \ + --work-dir jobs/egress-build/gateway-local1 \ + --cache-dir jobs/egress-build/runtime-apks --download +``` + +Select the resulting gateway using `EGRESS_IMAGE=searchswe-egress:local1` for +direct mode, or the explicit config file's `image` field. The component image is +only a build input; task operators need the final gateway image alone. + +The component builder applies `patch_gost.py` and runs its protocol regressions. +Source archives, module dependencies and runtime APKs are checksum-verified; +the gateway's source and package locks must match the checkout. Retain the build +manifests, source/license materials and caches with each distributed version. +For offline builds, use the component builder's `--offline` and omit the gateway +builder's `--download`. Missing or changed inputs fail the build. + +## Verification + +Host regressions include launcher selection, configuration, Harbor phase hooks, +Compose restrictions, credential handling, build inputs and ownership recovery: + +```bash +python -B -m unittest discover -s scripts/tests -v +python -B -m unittest scripts.test_pi_trajectory -v +``` + +For a pulled published image, a bounded direct TLS check uses a local HTTPS/DNS +fixture and verifies both successful allowed requests and receiving-end denials: + +```bash +docker pull --platform linux/amd64 python:3.13-slim +python scripts/tests/egress_direct_transport.py \ + --gateway-image hanhainebula/search-swe-egress:1.0.0 \ + --output jobs/egress-checks/direct-tls-01 +``` + +For changes to network enforcement, run the relevant Docker regressions below. +Each command requires a fresh `--output jobs/egress-checks/NAME` directory and +`--gateway-image hanhainebula/search-swe-egress:1.0.0` (or your matching build): + +| Script under `scripts/tests/` | Coverage / additional arguments | +| --- | --- | +| `egress_lifecycle/run.py` | Filtering, phase changes, old connections, frozen-controller leases; run with `--transport direct` and `--transport proxy` | +| `egress_isolation/run.py` | Raw sockets, DNS and IPv6 bypass attempts; run both transports | +| `egress_transport/run.py` | HTTP/HTTPS proxies, authentication, TLS, DoH and H2; needs `--go /path/to/go` | +| `egress_full_trial.py` | Concurrent four-step Harbor trials with independent verifiers; run both transports | +| `egress_adapter_smoke.py` | Startup, cancellation and credential boundaries; add `--credential-probe` | +| `egress_orphan_smoke.py` | Host-driver death and owned-resource recovery; add `--credential-probe` | + +`egress_s0/run.py` tests the patched component separately using +`--gost-image YOUR_COMPONENT_IMAGE --output jobs/egress-checks/component-01`. +Fixtures use local receivers and synthetic credentials, with no paid API calls. +Host tests alone do not establish network enforcement. diff --git a/environments/egress/component-apks.lock.json b/environments/egress/component-apks.lock.json new file mode 100644 index 0000000..2db370e --- /dev/null +++ b/environments/egress/component-apks.lock.json @@ -0,0 +1,12 @@ +{ + "version": 1, + "repository": "https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64/", + "archives": { + "libcrypto3-3.5.8-r0.apk": "a5592b5cf276bc7a30ac7b161d46446085490b640faa1f839ea98eb15aacad31", + "libssl3-3.5.8-r0.apk": "f80b76cb5e5a52cfc1ced08f8dc3022adc0ce0a3d6e7741976731b22c71fe310" + }, + "installed": { + "libcrypto3": "3.5.8-r0", + "libssl3": "3.5.8-r0" + } +} diff --git a/environments/egress/entrypoint.sh b/environments/egress/entrypoint.sh new file mode 100644 index 0000000..a12c4b9 --- /dev/null +++ b/environments/egress/entrypoint.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +exec python3 /opt/searchswe/gateway.py serve diff --git a/environments/egress/gateway.py b/environments/egress/gateway.py new file mode 100644 index 0000000..4973ce3 --- /dev/null +++ b/environments/egress/gateway.py @@ -0,0 +1,423 @@ +#!/usr/bin/env python3 +"""Trusted per-environment gateway controller; no proxy/DNS protocol parsing. + +Only this container receives NET_ADMIN/NET_RAW and the private input mount. +Task containers share its network namespace, not its PID or mount namespace. +The control socket is filesystem-only and never mounted into task containers. +""" + +import hashlib +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import re +import signal +import socket +import socketserver +import subprocess +import sys +import threading +import time + + +RUN = Path("/run/searchswe") +READY = Path("/tmp/harbor-docker-egress-control-sidecar.ready") +CONTROL = RUN / "control.sock" +MARK = "114514" +LEASE_SECONDS = 30 +DOMAIN = re.compile(r"(?=.{1,253}\Z)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z](?:[a-z0-9-]{0,61}[a-z0-9])?\Z") + + +def policy(mode, hosts, *, allow_public=False): + modes = {"allowlist", "no-network", "public"} if allow_public else {"allowlist", "no-network"} + if mode not in modes or not isinstance(hosts, list): + raise ValueError("unsupported network policy") + if any(not isinstance(h, str) or not DOMAIN.fullmatch(h) for h in hosts): + raise ValueError("only lowercase exact DNS hosts are supported") + if mode == "no-network" and hosts: + raise ValueError("no-network must not contain hosts") + return {"mode": mode, "hosts": sorted(set(hosts))} + + +def gost_config(settings, hosts): + if settings.get("transport") == "direct": + # Resolve the approved HTTP authority/SNI independently of the task's + # original IP. Resolution goes through the same filtered DNS handler. + services = [{"name": "approved-api", "addr": "127.0.0.1:12345", "bypass": "phase", + "resolver": "phase-dns", "metadata": {"so_mark": MARK}, + "handler": {"type": "red", "metadata": { + "sniffing": True, "sniffing.timeout": "5s", "sniffing.fallback": False}}, + "listener": {"type": "red"}}] + for protocol in ("udp", "tcp"): + services.append({"name": "approved-dns-" + protocol, "addr": "127.0.0.1:1053", "bypass": "phase", + "metadata": {"so_mark": MARK}, + "handler": {"type": "dns", "metadata": { + "dns": ",".join("udp://" + address + ("" if ":" in address else ":53") + for address in settings["dns_servers"]), + "timeout": "10s", "readTimeout": "12s"}}, + "listener": {"type": "dns", "metadata": {"mode": protocol}}}) + return {"log": {"level": "error"}, "services": services, + "bypasses": [{"name": "phase", "whitelist": True, "matchers": hosts}], + "resolvers": [{"name": "phase-dns", "nameservers": [ + {"addr": "udp://127.0.0.1:1053", "only": "ipv4", "timeout": "12s"}]}]} + node = {"name": "operator-upstream", "addr": settings["upstream_addr"], + "metadata": {"so_mark": MARK}, "connector": {"type": "http"}, + "dialer": {"type": "tls" if settings["upstream_tls"] else "tcp"}} + if settings.get("auth"): + node["connector"]["auth"] = settings["auth"] + if settings["upstream_tls"]: + node["dialer"]["tls"] = {"serverName": settings["upstream_host"], "secure": True} + # Do not enable recorders, sniffing fallback, raw TCP listeners or a + # management API. All outbound sockets must travel via the sole chain. + services = [{"name": "approved-api", "addr": "127.0.0.1:12345", "bypass": "phase", + "metadata": {"so_mark": MARK}, + "handler": {"type": "red", "chain": "operator", "metadata": { + "sniffing": True, "sniffing.timeout": "5s", "sniffing.fallback": False}}, + "listener": {"type": "red"}}] + for protocol in ("udp", "tcp"): + services.append({"name": "approved-dns-" + protocol, "addr": "127.0.0.1:1053", "bypass": "phase", + "metadata": {"so_mark": MARK}, + "handler": {"type": "dns", "chain": "operator", "metadata": { + "dns": settings["doh_url"], "timeout": "10s", "readTimeout": "12s"}}, + "listener": {"type": "dns", "metadata": {"mode": protocol}}}) + return {"log": {"level": "error"}, "services": services, + "bypasses": [{"name": "phase", "whitelist": True, "matchers": hosts}], + "chains": [{"name": "operator", "hops": [{"name": "transport", "nodes": [node]}]}]} + + +def nft(script): + subprocess.run(["nft", "-f", "-"], input=script, text=True, check=True, + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, timeout=5, + env={**os.environ, "TZ": "UTC"}) + + +def close_gate(): + READY.unlink(missing_ok=True) + # Separate, higher-priority filter: existing marked upstream sessions are + # blocked too. Never flush/delete this gate as part of failure cleanup. + nft("""add table inet searchswe_guard +flush table inet searchswe_guard +table inet searchswe_guard { + chain output { type filter hook output priority -300; policy drop; oifname "lo" accept; } + chain input { type filter hook input priority -300; policy drop; iifname "lo" accept; } +} +""") + + +def install_rules(settings, *, public=False): + # upstream_ip and upstream_port are validated by the host adapter. Recheck + # locally before interpolating nft syntax; no user-supplied shell code. + import ipaddress + direct = settings.get("transport") == "direct" + if public and not direct: + raise ValueError("public proxy egress is unsupported") + if direct: + # Only the trusted worker can set MARK. Test it before local acceptance + # so even Docker's loopback DNS cannot escape the kernel lease. + outgoing = f"meta mark {MARK} meta nfproto ipv4 jump lease\n meta mark {MARK} drop" + else: + ip = str(ipaddress.IPv4Address(settings["upstream_ip"])) + port = int(settings["upstream_port"]) + if not 1 <= port <= 65535: + raise ValueError("invalid upstream port") + outgoing = f"meta mark {MARK} ip daddr {ip} tcp dport {port} jump lease\n meta mark {MARK} drop" + redirect = f"""meta mark {MARK} return + udp dport 53 redirect to :1053 + tcp dport 53 redirect to :1053 + fib daddr type local return + meta l4proto tcp redirect to :12345""" + if public: + redirect = "" + outgoing = "jump lease" + nft(f"""add table inet searchswe_egress +flush table inet searchswe_egress +table inet searchswe_egress {{ + set leased_marks {{ type mark; flags timeout; timeout {LEASE_SECONDS}s; }} + counter proxy_packets {{ }} + chain lease {{ }} + chain output {{ type nat hook output priority -110; policy accept; + {redirect} + }} + chain egress {{ type filter hook output priority 0; policy drop; + {outgoing} + ip daddr 127.0.0.11 drop comment "embedded_dns_requires_worker" + fib daddr type local counter accept comment "local_packets" + counter drop comment "dropped_packets" + }} + chain input {{ type filter hook input priority 0; policy drop; + iifname "lo" accept + ct state established,related counter accept + counter drop + }} +}} +""") + + +def lease_remaining(deadline_ns): + if type(deadline_ns) is not int: + raise ValueError("invalid host lease") + remaining = (deadline_ns - time.monotonic_ns()) // 1_000_000 + if not 0 < remaining <= LEASE_SECONDS * 1000: + raise ValueError("expired or invalid host lease") + return remaining + + +def packet_counters(): + """Packet counts, not HTTP requests or hostname authorization decisions.""" + try: + output = subprocess.run(["nft", "-j", "list", "table", "inet", "searchswe_egress"], + capture_output=True, text=True, check=True, timeout=2).stdout + counts = {} + for item in json.loads(output)["nftables"]: + counter = item.get("counter", {}) + if counter.get("name") == "proxy_packets": + counts["proxy_packets"] = {key: counter[key] for key in ("packets", "bytes")} + rule = item.get("rule", {}) + if rule.get("comment") in {"local_packets", "proxy_packets", "dropped_packets"}: + for expression in rule["expr"]: + if "counter" in expression: + counts[rule["comment"]] = expression["counter"] + return counts + except (OSError, ValueError, KeyError, subprocess.SubprocessError): + return None + + +def owns_listeners(pid): + """Readiness must not accept a port squatted by an untrusted task.""" + try: + inodes = {os.readlink(fd) for fd in Path(f"/proc/{pid}/fd").iterdir()} + found = set() + for protocol in ("tcp", "udp"): + for line in Path(f"/proc/net/{protocol}").read_text().splitlines()[1:]: + fields = line.split() + if f"socket:[{fields[9]}]" not in inodes: + continue + address, port = fields[1].split(":") + if address == "0100007F" and (protocol != "tcp" or fields[3] == "0A"): + found.add((protocol, int(port, 16))) + return {("tcp", 12345), ("tcp", 1053), ("udp", 1053)} <= found + except (OSError, IndexError): + return False + + +def validate_upstream_namespace(settings): + # The task shares the gateway's network namespace. It must not impersonate + # either the proxy (credentials) or the direct resolver (destination IPs). + import ipaddress + direct = settings.get("transport") == "direct" + addresses = settings["dns_servers"] if direct else [settings["upstream_ip"]] + if not addresses: + raise ValueError("upstream addresses are required") + for value in addresses: + host, separator, port = value.partition(":") + address = ipaddress.IPv4Address(host) + if direct and str(address) == "127.0.0.11": + if separator and int(port) != 53: + raise ValueError("Docker DNS requires port 53") + continue + if (address.is_loopback or address.is_unspecified or address.is_multicast + or address.is_link_local or address.is_reserved): + raise ValueError("upstream cannot be local/link-local/multicast") + route = subprocess.run(["ip", "-4", "route", "get", str(address)], capture_output=True, + text=True, check=True, timeout=5).stdout.split() + if not route or route[0] == "local": + raise ValueError("upstream must be outside the shared task network namespace") + + +class Controller: + def __init__(self, settings): + self.settings = settings + self.lock = threading.RLock() + self.worker = None + self.generation = 0 + self.deadline_ns = 0 + self.public = False + self.state = {"ready": False, "mode": "no-network"} + + def audit(self, event): + print(json.dumps({"event": event, "instance": os.environ.get("SEARCHSWE_EGRESS_INSTANCE", "standalone"), + "generation": self.generation, "policy_sha256": self.state.get("policy_sha256"), + "packet_counters": packet_counters()}), flush=True) + + def refresh_lease(self, deadline_ns): + remaining = lease_remaining(deadline_ns) + # Read wall time BEFORE monotonic time: a pause between the reads can + # only shorten this cutoff. The absolute cutoff prevents a queued nft + # command from granting a fresh relative lease after a controller pause. + cutoff = (time.time_ns() + deadline_ns - time.monotonic_ns()) // 10**9 + cutoff = datetime.fromtimestamp(cutoff, timezone.utc).strftime("%Y-%m-%d %H:%M:%S") + # Expiry is enforced by nftables even if the controller is SIGSTOPed. + # One atomic transaction: never a permanently accepted mark. + nft(f"flush set inet searchswe_egress leased_marks\n" + f"add element inet searchswe_egress leased_marks {{ {MARK} timeout {remaining}ms" + + (f", 0 timeout {remaining}ms" if self.public else "") + " }\n" + "flush chain inet searchswe_egress lease\n" + f'add rule inet searchswe_egress lease meta time < "{cutoff}" ' + "meta mark @leased_marks counter name proxy_packets accept\n") + self.deadline_ns = deadline_ns + + def renew(self, deadline_ns): + with self.lock: + if not self.state.get("ready") or time.monotonic_ns() >= self.deadline_ns: + self.fail("lease_expired") + raise RuntimeError("an expired lease requires an explicit policy replacement") + self.refresh_lease(deadline_ns) + return self.state + + def stop_worker(self): + worker, self.worker = self.worker, None + if worker is not None and worker.poll() is None: + worker.terminate() + try: + worker.wait(timeout=3) + except subprocess.TimeoutExpired: + worker.kill() + worker.wait(timeout=3) + (RUN / "gost.json").unlink(missing_ok=True) + + def fail(self, reason="policy_failure"): + try: + close_gate() + finally: + try: + self.stop_worker() + finally: + self.audit(reason) + self.deadline_ns = 0 + self.state = {"ready": False, "error": reason, "generation": self.generation} + + def apply(self, requested): + with self.lock: + try: + close_gate() + self.audit("generation_closed") + self.stop_worker() + current = policy(requested["mode"], requested["hosts"], + allow_public=self.settings.get("transport") == "direct") + if self.settings.get("upstream_host") in current["hosts"]: + raise ValueError("a general proxy is not an API destination") + self.public = current["mode"] == "public" + install_rules(self.settings, public=self.public) + deadline_ns = requested.get("deadline_ns", 0) + lease_remaining(deadline_ns) + self.generation += 1 + if current["mode"] == "allowlist" and current["hosts"]: + config = RUN / "gost.json" + config.write_text(json.dumps(gost_config(self.settings, current["hosts"]))) + config.chmod(0o600) + self.worker = subprocess.Popen(["/bin/gost", "-C", str(config)], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + deadline = time.monotonic() + 10 + while True: + if self.worker.poll() is not None or time.monotonic() > deadline: + raise RuntimeError("gateway worker failed readiness") + if owns_listeners(self.worker.pid): + break + time.sleep(.05) + self.refresh_lease(deadline_ns) + if self.worker is not None or self.public: + nft("delete table inet searchswe_guard\n") + # Empty allowlist and no-network retain the closed guard and + # have no proxy or DNS worker. Local application IPC remains. + self.state = {"ready": True, **current, "generation": self.generation, + "policy_sha256": hashlib.sha256(json.dumps(current, sort_keys=True).encode()).hexdigest()} + READY.touch() + self.audit("generation_ready") + return self.state + except Exception: + self.fail() + raise + + def watch(self): + while True: + time.sleep(.1) + with self.lock: + if self.state.get("ready") and time.monotonic_ns() >= self.deadline_ns: + self.fail("lease_expired") + elif self.worker is not None and self.worker.poll() is not None: + self.fail("worker_exit") + + +def serve(): + os.umask(0o077) + RUN.mkdir(mode=0o700, exist_ok=True) + close_gate() + settings = json.loads(Path("/opt/searchswe/input.json").read_text()) + validate_upstream_namespace(settings) + controller = Controller(settings) + + def shutdown(*_): + with controller.lock: + controller.fail() + raise SystemExit(0) + + signal.signal(signal.SIGTERM, shutdown) + signal.signal(signal.SIGINT, shutdown) + + class Handler(socketserver.StreamRequestHandler): + def handle(self): + try: + self.connection.settimeout(45) + request = json.loads(self.rfile.readline(65537)) + if request == {"action": "show"}: + with controller.lock: + response = dict(controller.state) + elif request.get("action") == "lease": + response = controller.renew(request["deadline_ns"]) + else: + response = controller.apply(request) + except Exception: + with controller.lock: + controller.fail() + response = {"ready": False, "error": "gateway policy request failed"} + try: + self.wfile.write(json.dumps(response).encode() + b"\n") + except OSError: + with controller.lock: + controller.fail() + + initial = os.environ.get("EGRESS_CONTROL_INITIAL_NETWORK_MODE", "no-network") + hosts = os.environ.get("EGRESS_CONTROL_INITIAL_ALLOWED_HOSTS", "").split() + deadline = int(os.environ.get("SEARCHSWE_EGRESS_INITIAL_DEADLINE_NS", time.monotonic_ns() + LEASE_SECONDS * 10**9)) + controller.apply({"mode": initial, "hosts": hosts, "deadline_ns": deadline}) + CONTROL.unlink(missing_ok=True) + server = socketserver.UnixStreamServer(str(CONTROL), Handler) + threading.Thread(target=controller.watch, daemon=True).start() + print("Search-SWE restricted gateway ready", flush=True) + server.serve_forever() + + +def client(arguments): + deadline_ns = time.monotonic_ns() + LEASE_SECONDS * 10**9 + if arguments[:1] == ["--deadline-ns"]: + deadline_ns = int(arguments[1]) + arguments = arguments[2:] + action = arguments[0] if arguments else "show" + if action == "show": + request = {"action": "show"} + elif action == "allow": + request = {"mode": "allowlist", "hosts": arguments[1:], "deadline_ns": deadline_ns} + elif action == "deny-all": + request = {"mode": "no-network", "hosts": [], "deadline_ns": deadline_ns} + elif action == "allow-all": + request = {"mode": "public", "hosts": [], "deadline_ns": deadline_ns} + elif action == "lease": + request = {"action": "lease", "deadline_ns": deadline_ns} + else: + raise ValueError("unsupported gateway command") + with socket.socket(socket.AF_UNIX) as sock: + sock.settimeout(60) + sock.connect(str(CONTROL)) + sock.sendall(json.dumps(request).encode() + b"\n") + response = json.loads(sock.makefile("rb").readline(65537)) + print(json.dumps(response)) + if not response.get("ready"): + raise SystemExit(1) + + +if __name__ == "__main__": + if sys.argv[1:] == ["serve"]: + serve() + else: + client(sys.argv[1:]) diff --git a/environments/egress/gost_tests/searchswe_dns_test.go b/environments/egress/gost_tests/searchswe_dns_test.go new file mode 100644 index 0000000..71631d4 --- /dev/null +++ b/environments/egress/gost_tests/searchswe_dns_test.go @@ -0,0 +1,69 @@ +package dns + +import ( + "context" + "testing" + + "github.com/go-gost/core/chain" + "github.com/go-gost/core/handler" + "github.com/go-gost/core/hop" + "github.com/go-gost/x/resolver/exchanger" + "github.com/miekg/dns" +) + +func TestSearchSWEDNSUnsupportedClassRefused(t *testing.T) { + h := newInitdHandler(handler.BypassOption(&mockBypass{whitelist: true})) + for _, class := range []uint16{dns.ClassCHAOS, dns.ClassHESIOD, dns.ClassANY} { + q := new(dns.Msg).SetQuestion("blocked.example.", dns.TypeA) + q.Question[0].Qclass = class + wire, _ := q.Pack() + response, err := h.request(context.Background(), wire, newRecObj(), nopLog()) + if err != nil { t.Fatal(err) } + r := new(dns.Msg) + if err := r.Unpack(response); err != nil { t.Fatal(err) } + if r.Rcode != dns.RcodeRefused { t.Fatalf("class %d: rcode %d", class, r.Rcode) } + } +} + +func TestSearchSWEDNSNonAddressPayloadRefused(t *testing.T) { + for _, kind := range []string{"TXT", "answer", "authority", "extra", "edns-payload"} { + t.Run(kind, func(t *testing.T) { + reached := 0 + h := newInitdHandler(handler.BypassOption(&mockBypass{whitelist: true})) + h.hop = &mockHop{selectFn: func(context.Context, ...hop.SelectOption) *chain.Node { return &chain.Node{Name: "fixture"} }} + h.exchangers = map[string]exchanger.Exchanger{"fixture": &mockExchanger{exchangeFn: func(ctx context.Context, wire []byte) ([]byte, error) { + reached++ + q := new(dns.Msg) + if err := q.Unpack(wire); err != nil { return nil, err } + return new(dns.Msg).SetReply(q).Pack() + }}} + // A real exchanger positive, not just an expected client error. + for _, typ := range []uint16{dns.TypeA, dns.TypeAAAA} { + if _, err := h.request(context.Background(), packDNSQuery("control.example.", typ), newRecObj(), nopLog()); err != nil { t.Fatal(err) } + } + if reached != 2 { t.Fatal("missing A/AAAA positive controls") } + plainEDNS := new(dns.Msg).SetQuestion("edns.example.", dns.TypeA).SetEdns0(1232, false) + ednsWire, _ := plainEDNS.Pack() + if _, err := h.request(context.Background(), ednsWire, newRecObj(), nopLog()); err != nil { t.Fatal(err) } + if reached != 3 { t.Fatal("empty EDNS positive control failed") } + q := new(dns.Msg).SetQuestion("allowed.example.", dns.TypeA) + rr := mustNewRR("blocked.example. 60 IN TXT \"must-not-reach-resolver\"") + switch kind { + case "TXT": q.Question[0].Qtype = dns.TypeTXT + case "answer": q.Answer = []dns.RR{rr} + case "authority": q.Ns = []dns.RR{rr} + case "extra": q.Extra = []dns.RR{rr} + case "edns-payload": + q.SetEdns0(1232, false) + q.IsEdns0().Option = append(q.IsEdns0().Option, &dns.EDNS0_LOCAL{Code: 65001, Data: []byte("must-not-reach-resolver")}) + } + wire, _ := q.Pack() + response, err := h.request(context.Background(), wire, newRecObj(), nopLog()) + if err != nil { t.Fatal(err) } + r := new(dns.Msg) + if err := r.Unpack(response); err != nil { t.Fatal(err) } + if reached != 3 { t.Fatalf("unsupported payload reached resolver (%d exchanges)", reached) } + if r.Rcode != dns.RcodeRefused { t.Fatalf("rcode %d", r.Rcode) } + }) + } +} diff --git a/environments/egress/gost_tests/searchswe_exchanger_test.go b/environments/egress/gost_tests/searchswe_exchanger_test.go new file mode 100644 index 0000000..d292fba --- /dev/null +++ b/environments/egress/gost_tests/searchswe_exchanger_test.go @@ -0,0 +1,59 @@ +package exchanger + +import ( + "context" + "crypto/tls" + "crypto/x509" + "io" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + xlogger "github.com/go-gost/x/logger" +) + +func searchSWETLS(server *httptest.Server) *tls.Config { + pool := x509.NewCertPool() + pool.AddCert(server.Certificate()) + return &tls.Config{RootCAs: pool} +} + +func TestSearchSWEDoHCertificateAndPositiveControl(t *testing.T) { + var queries atomic.Int32 + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + queries.Add(1) + io.Copy(w, r.Body) + })) + defer server.Close() + good, err := NewExchanger(server.URL, TLSConfigOption(searchSWETLS(server)), LoggerOption(xlogger.Nop())) + if err != nil { t.Fatal(err) } + if reply, err := good.Exchange(context.Background(), []byte("fixture")); err != nil || string(reply) != "fixture" { + t.Fatalf("positive control: %q %v", reply, err) + } + bad, err := NewExchanger(server.URL, LoggerOption(xlogger.Nop())) + if err != nil { t.Fatal(err) } + if _, err := bad.Exchange(context.Background(), []byte("must-not-arrive")); err == nil { + t.Fatal("untrusted DoH certificate accepted") + } + if queries.Load() != 1 { t.Fatal("untrusted TLS origin received a DNS payload") } +} + +func TestSearchSWEDoHRedirectNeverReachesOtherOrigin(t *testing.T) { + var reached atomic.Int32 + forbidden := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + reached.Add(1) + w.WriteHeader(200) + })) + defer forbidden.Close() + redirect := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, forbidden.URL, http.StatusTemporaryRedirect) + })) + defer redirect.Close() + ex, err := NewExchanger(redirect.URL, TLSConfigOption(searchSWETLS(redirect)), LoggerOption(xlogger.Nop())) + if err != nil { t.Fatal(err) } + if _, err := ex.Exchange(context.Background(), []byte("must-not-be-redirected")); err == nil { + t.Fatal("DoH redirect accepted") + } + if reached.Load() != 0 { t.Fatal("redirect destination received the query") } +} diff --git a/environments/egress/gost_tests/searchswe_sniffing_test.go b/environments/egress/gost_tests/searchswe_sniffing_test.go new file mode 100644 index 0000000..88c176f --- /dev/null +++ b/environments/egress/gost_tests/searchswe_sniffing_test.go @@ -0,0 +1,73 @@ +package sniffing + +import ( + "bufio" + "context" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/go-gost/core/bypass" + xlogger "github.com/go-gost/x/logger" + xrecorder "github.com/go-gost/x/recorder" +) + +type searchSWEPolicy struct{} +func (*searchSWEPolicy) IsWhitelist() bool { return true } +func (*searchSWEPolicy) Contains(ctx context.Context, network, addr string, opts ...bypass.Option) bool { + return addr != "allowed.example:443" && addr != "allowed.example:80" +} + +type searchSWETransport struct { hosts []string } +func (tr *searchSWETransport) RoundTrip(r *http.Request) (*http.Response, error) { + tr.hosts = append(tr.hosts, r.Host) + return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader("OK"))}, nil +} + +func TestSearchSWEH2PolicyOnEveryStream(t *testing.T) { + tr := &searchSWETransport{} + h := &h2Handler{transport: tr, bypass: &searchSWEPolicy{}, service: "test", network: "tcp", + recorderObject: &xrecorder.HandlerRecorderObject{}, log: xlogger.Nop()} + for _, host := range []string{"allowed.example", "blocked.example", "allowed.example", "127.0.0.1"} { + r := httptest.NewRequest("GET", "https://" + host + "/", nil) + w := httptest.NewRecorder() + h.ServeHTTP(w, r) + want := 403 + if host == "allowed.example" { want = 200 } + if w.Code != want { t.Fatalf("host %s: %d, want %d", host, w.Code, want) } + } + if strings.Join(tr.hosts, ",") != "allowed.example,allowed.example" { t.Fatal(tr.hosts) } +} + +func TestSearchSWEHTTPKeepAliveHostChange(t *testing.T) { + seen := make(chan string, 4) + origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + seen <- r.Host + w.Header().Set("Content-Length", "2") + io.WriteString(w, "OK") + })) + defer origin.Close() + client, server := net.Pipe() + defer client.Close() + defer server.Close() + client.SetDeadline(time.Now().Add(5 * time.Second)) + errCh := make(chan error, 1) + go func() { + defer server.Close() + errCh <- (&Sniffer{}).HandleHTTP(context.Background(), "tcp", server, + WithBypass(&searchSWEPolicy{}), WithLog(xlogger.Nop()), + WithRecorderObject(&xrecorder.HandlerRecorderObject{}), + WithDial(func(context.Context, string, string) (net.Conn, error) { return net.Dial("tcp", origin.Listener.Addr().String()) })) + }() + io.WriteString(client, "GET / HTTP/1.1\r\nHost: allowed.example\r\n\r\n") + r, err := http.ReadResponse(bufio.NewReader(client), nil) + if err != nil { t.Fatal(err) } + io.Copy(io.Discard, r.Body); r.Body.Close() + io.WriteString(client, "GET / HTTP/1.1\r\nHost: blocked.example\r\n\r\n") + if err := <-errCh; err == nil { t.Fatal("host change was accepted") } + if len(seen) != 1 || <-seen != "allowed.example" { t.Fatal("unexpected upstream request") } +} diff --git a/environments/egress/network-policy b/environments/egress/network-policy new file mode 100644 index 0000000..0825882 --- /dev/null +++ b/environments/egress/network-policy @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +exec python3 /opt/searchswe/gateway.py "$@" diff --git a/environments/egress/patch_gost.py b/environments/egress/patch_gost.py new file mode 100644 index 0000000..eed8ed8 --- /dev/null +++ b/environments/egress/patch_gost.py @@ -0,0 +1,76 @@ +"""Exact, fail-closed source transformations for go-gost/x v0.10.9 only. + +Invoked by the reproducible build, never applied to system-installed Harbor. +Each replacement must match once; upstream drift aborts the build. +""" + +from pathlib import Path +import sys + + +def replace(root, name, before, after): + path = root / name + text = path.read_text() + if text.count(before) != 1: + raise RuntimeError(f"Pinned GOST source mismatch: {name}") + path.write_text(text.replace(before, after)) + + +def apply(root): + # Check every H2 stream, including pooled-connection requests, before + # constructing or forwarding an upstream request. + name = "internal/util/sniffing/sniffer_h2.go" + replace(root, name, '\t"github.com/go-gost/core/logger"', + '\t"github.com/go-gost/core/bypass"\n\t"github.com/go-gost/core/logger"') + replace(root, name, "\t\t\ttransport: tr,", "\t\t\ttransport: tr,\n\t\t\tbypass: ho.bypass,\n\t\t\tservice: ho.service,\n\t\t\tnetwork: network,") + replace(root, name, "type h2Handler struct {", "type h2Handler struct {\n\tbypass bypass.Bypass\n\tservice string\n\tnetwork string") + replace(root, name, "func (h *h2Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {", """func (h *h2Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if h.bypass != nil && h.bypass.Contains(r.Context(), h.network, normalizeHost(r.Host, "443"), bypass.WithService(h.service)) { + w.WriteHeader(http.StatusForbidden) + return + }""") + + # Defense in depth: every red-handler dial checks the FINAL hostname:port, + # not just whichever protocol-specific sniffer dispatched to it. + replace(root, "handler/redirect/tcp/handler.go", """ ro.Host = address + + var buf bytes.Buffer""", """ ro.Host = address + if h.options.Bypass != nil && h.options.Bypass.Contains(ctx, network, address, bypass.WithService(h.options.Service)) { + return nil, xbypass.ErrBypass + } + + var buf bytes.Buffer""") + + # Transparent HTTP/1 keep-alive is bound to the initial origin. Do not + # forward a changed Host on its socket (even when both names are allowed). + replace(root, "internal/util/sniffing/sniffer_http.go", """ if shouldClose, err := h.httpRoundTrip(ctx, xio.NewReadWriteCloser(br, conn, conn), cc, req, readTimeout, ro, &pStats, log); err != nil || shouldClose {""", """ nextHost := normalizeHost(req.Host, "80") + if nextHost != host || (ho.bypass != nil && ho.bypass.Contains(ctx, network, nextHost, bypass.WithService(ho.service))) { + return xbypass.ErrBypass + } + if shouldClose, err := h.httpRoundTrip(ctx, xio.NewReadWriteCloser(br, conn, conn), cc, req, readTimeout, ro, &pStats, log); err != nil || shouldClose {""") + + # A configured DNS policy must not be skipped for non-IN questions. Reject + # unsupported messages outright rather than handing arbitrary DNS payloads + # to a trusted DoH resolver. Normal unfiltered GOST DNS use is unchanged. + replace(root, "handler/dns/handler.go", """ if h.options.Bypass != nil && mq.Question[0].Qclass == dns.ClassINET {""", """ if h.options.Bypass != nil { + unsupported := len(mq.Question) != 1 || mq.Response || mq.Opcode != dns.OpcodeQuery || mq.Question[0].Qclass != dns.ClassINET + unsupported = unsupported || (mq.Question[0].Qtype != dns.TypeA && mq.Question[0].Qtype != dns.TypeAAAA) || len(mq.Answer) != 0 || len(mq.Ns) != 0 || len(mq.Extra) > 1 + if !unsupported && len(mq.Extra) == 1 { + opt, ok := mq.Extra[0].(*dns.OPT) + unsupported = !ok || opt.Hdr.Name != "." || opt.Version() != 0 || len(opt.Option) != 0 + } + if unsupported { + mr = (&dns.Msg{}).SetRcode(&mq, dns.RcodeRefused) + return h.packResponse(mr) + }""") + + # The resolver is privileged infrastructure, not an arbitrary HTTP grant. + # A redirect must not forward its DNS payload to another destination. + replace(root, "resolver/exchanger/exchanger.go", """ ex.client = &http.Client{ + Timeout: options.timeout,""", """ ex.client = &http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }, + Timeout: options.timeout,""") + + +if __name__ == "__main__": + apply(Path(sys.argv[1])) diff --git a/environments/egress/runtime.lock.json b/environments/egress/runtime.lock.json new file mode 100644 index 0000000..ce2a464 --- /dev/null +++ b/environments/egress/runtime.lock.json @@ -0,0 +1,65 @@ +{ + "version": 1, + "alpine_release": "3.23.4", + "repository": "https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64/", + "installed": { + "alpine-baselayout": "3.7.2-r0", + "alpine-baselayout-data": "3.7.2-r0", + "alpine-keys": "2.6-r0", + "alpine-release": "3.23.4-r0", + "apk-tools": "3.0.6-r0", + "busybox": "1.37.0-r30", + "busybox-binsh": "1.37.0-r30", + "ca-certificates-bundle": "20260413-r0", + "gdbm": "1.26-r0", + "gmp": "6.3.0-r4", + "iptables": "1.8.11-r1", + "jansson": "2.14.1-r0", + "libapk": "3.0.6-r0", + "libbz2": "1.0.8-r6", + "libcrypto3": "3.5.8-r0", + "libexpat": "2.8.5-r0", + "libffi": "3.5.2-r0", + "libgcc": "15.2.0-r2", + "libmnl": "1.0.5-r2", + "libncursesw": "6.5_p20251123-r0", + "libnftnl": "1.3.0-r0", + "libpanelw": "6.5_p20251123-r0", + "libssl3": "3.5.8-r0", + "libstdc++": "15.2.0-r2", + "libxtables": "1.8.11-r1", + "mpdecimal": "4.0.1-r0", + "musl": "1.2.5-r23", + "musl-utils": "1.2.5-r23", + "ncurses-terminfo-base": "6.5_p20251123-r0", + "nftables": "1.1.5-r2", + "pyc": "3.12.14-r0", + "python3": "3.12.14-r0", + "python3-pyc": "3.12.14-r0", + "python3-pycache-pyc0": "3.12.14-r0", + "readline": "8.3.1-r0", + "scanelf": "1.3.8-r2", + "sqlite-libs": "3.53.4-r0", + "ssl_client": "1.37.0-r30", + "xz-libs": "5.8.4-r0", + "zlib": "1.3.2-r0" + }, + "archives": { + "libbz2-1.0.8-r6.apk": "9995840a76ac97ec006259391b7a81b73b73c5cf2f462bf12bb2250afbf665b9", + "libffi-3.5.2-r0.apk": "0b3473a1468bed5fbb913a1ffa121e253080b21b013fe47e0300bd7e85224e76", + "gdbm-1.26-r0.apk": "1a99c3ca2fbd96030a93325d50d04390e353f12584999276bdd3abcc6ab997da", + "xz-libs-5.8.4-r0.apk": "3c8cc415e88c2a097543d9ee828081ec1905e7d0f57a319c8896269b0e00a672", + "libgcc-15.2.0-r2.apk": "4279d14cbf43311312d3a7d43619f4c15ede2e46d5608ba421ed157ded57c700", + "libstdc++-15.2.0-r2.apk": "d4be2967a06d78b499f5e136323f4ebe51ad367b381933b3369539499c4f5752", + "mpdecimal-4.0.1-r0.apk": "51554fc6064cc7e595c9f331e6ea4ca87b62aad0d08b9bb9b441655a994a92e1", + "libpanelw-6.5_p20251123-r0.apk": "43b7346857c51ef2426fc4953177d4fa0ef4703d412215744fcd4bc96e6facfd", + "sqlite-libs-3.53.4-r0.apk": "414be12c879052f4614a42a10fc12af67a88ba49e8e1e33f78cd9ddbbdb13fee", + "python3-3.12.14-r0.apk": "60514ca149492e85090e8a8e1ec053bcb2355962f0967bbe91da5bebccb01d16", + "python3-pycache-pyc0-3.12.14-r0.apk": "984da228cd297ae3b3a85e465a44bdadbddb8c66342b0e1ef7e25bb9e247233e", + "pyc-3.12.14-r0.apk": "d808689e7660241a248b8c22576a126a46a12c476caaa58c9f459c43ea42b25b", + "python3-pyc-3.12.14-r0.apk": "16bbe19b383b0916ea0c5b6fa2fdb2b2687d493714cecf58aba1dde6d4e0406d", + "libcrypto3-3.5.8-r0.apk": "a5592b5cf276bc7a30ac7b161d46446085490b640faa1f839ea98eb15aacad31", + "libssl3-3.5.8-r0.apk": "f80b76cb5e5a52cfc1ced08f8dc3022adc0ce0a3d6e7741976731b22c71fe310", + "libexpat-2.8.5-r0.apk": "5216046085b92da88a6b107dbbb558f69d49c347a6335291fcf199ebd1670a8f" + } +} diff --git a/scripts/build_egress_component.py b/scripts/build_egress_component.py new file mode 100644 index 0000000..9cb6768 --- /dev/null +++ b/scripts/build_egress_component.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""Build the patched GOST base used by build_egress_gateway.py. + +Requires Go 1.26.8, curl, Docker and an already-built Harbor sidecar image. +Downloads are checksum pinned; work/output files never enter task containers. +""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import tarfile +import sys +from importlib.metadata import distribution + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from scripts.egress.build_audit import collect_notices, json_stream +from scripts.build_egress_gateway import stage_archives +SOURCES = { + "gost-3.2.6": ("https://codeload.github.com/go-gost/gost/tar.gz/v3.2.6", + "79874354530b899576dd4866d3b1400651d0b17c1e7a90ad30c44686a0642600"), + "x-0.10.9": ("https://codeload.github.com/go-gost/x/tar.gz/refs/tags/v0.10.9", + "5b3f2a97c047e4b90b24c5a6b3d147d1d1c0a38b900979daa5f94350db6bb6fb"), +} +LOCKS = { + "go.mod": "baa2a2e9c290f98d0b2c8165f9ca13fad456b349c17989191da53150b33adef1", + "go.sum": "023ebf43011ac2d3945e81b029b2b5cb11aa2fcbf913dbc08794b6eecb685806", +} + +SECURITY_MINIMA = {'github.com/pion/dtls/v3': 'v3.1.4', 'github.com/quic-go/quic-go': 'v0.59.1', 'github.com/quic-go/webtransport-go': 'v0.11.1', 'golang.org/x/crypto': 'v0.56.0', 'golang.org/x/net': 'v0.56.0', 'golang.org/x/text': 'v0.39.0', 'google.golang.org/grpc': 'v1.83.2'} + +def sha(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def command(argv, **kwargs): + return subprocess.run(argv, check=True, text=True, **kwargs) + + +def inspect(image): + return json.loads(command(["docker", "image", "inspect", image], capture_output=True).stdout)[0] + + +def build(args): + work = args.work_dir.resolve() + work.mkdir(parents=True, exist_ok=False) + cache = args.cache_dir.resolve() + cache.mkdir(parents=True, exist_ok=True) + go = str(Path(shutil.which(args.go) or args.go).resolve()) + manifest = {"sources": SOURCES, "status": "building"} + try: + if command([go, "version"], capture_output=True).stdout.strip() != "go version go1.26.8 linux/amd64": + raise ValueError("This component build currently requires Go 1.26.8 on Linux amd64") + base = inspect(args.base_image) + manifest["base_image_id"] = base["Id"] + recipe = ROOT / "environments/egress/Dockerfile.component" + recipe_bytes = recipe.read_bytes() + apk_lock_path = ROOT / "environments/egress/component-apks.lock.json" + apk_lock = json.loads(apk_lock_path.read_text()) + inputs = {str(p.relative_to(ROOT)): sha(p) for p in (Path(__file__).resolve(), ROOT / "scripts/egress/build_audit.py", ROOT / "scripts/build_egress_gateway.py", recipe, apk_lock_path)} + manifest["build_source_sha256"] = inputs + if base.get("Os") != "linux" or base.get("Architecture") != "amd64": + raise ValueError("The component base must be Linux amd64") + if args.base_image.startswith("sha256:"): + raise ValueError("BuildKit requires a preloaded local base name, not a bare image ID") + existing = subprocess.run(["docker", "image", "inspect", args.tag], capture_output=True, text=True, timeout=15) + if existing.returncode == 0 or "No such image" not in existing.stderr: + raise ValueError("Use a verified new component image tag; do not overwrite a previous build") + manifest.update(base_rootfs=base["RootFS"], base_repo_digests=base.get("RepoDigests", []), + go_executable_sha256=sha(Path(go)), offline=args.offline) + for name, (url, digest) in SOURCES.items(): + archive = cache / f"{name}.tar.gz" + if not archive.exists(): + if args.offline: + raise ValueError(f"Missing offline source cache: {name}") + temporary = work / f"{name}.download" + command(["curl", "--fail", "--location", "--retry", "2", "--max-time", "180", + "--output", str(temporary), url], timeout=600) + if sha(temporary) != digest: + raise ValueError(f"Source checksum mismatch: {name}") + shutil.copyfile(temporary, archive) + if sha(archive) != digest: + raise ValueError(f"Cached source checksum mismatch: {name}") + with tarfile.open(archive) as tar: + # Pinned archives only; reject path traversal/special files too. + for entry in tar.getmembers(): + path = Path(entry.name) + if path.is_absolute() or ".." in path.parts or not (entry.isdir() or entry.isfile()): + raise ValueError("Unsupported source archive entry") + tar.extractall(work, filter="data") + patcher = ROOT / "environments/egress/patch_gost.py" + patch_bytes = patcher.read_bytes() + patch_digest = hashlib.sha256(patch_bytes).hexdigest() + namespace = {"__name__": "egress_patch", "__file__": str(patcher)} + exec(compile(patch_bytes, str(patcher), "exec"), namespace) + namespace["apply"](work / "x-0.10.9") + manifest["patch_sha256"] = patch_digest + tests = ROOT / "environments/egress/gost_tests" + test_hashes = {} + for source, target in [("searchswe_sniffing_test.go", "internal/util/sniffing"), + ("searchswe_dns_test.go", "handler/dns"), + ("searchswe_exchanger_test.go", "resolver/exchanger")]: + data = (tests / source).read_bytes() + (work / "x-0.10.9" / target / source).write_bytes(data) + test_hashes[source] = hashlib.sha256(data).hexdigest() + env = {**os.environ, "GOTOOLCHAIN": "local", "GOMAXPROCS": "4", "CGO_ENABLED": "0", + "GOPATH": str(cache / "gopath"), "GOCACHE": str(cache / "gocache"), + "GOENV": "off", "GOWORK": "off", "GOFLAGS": "", "GOOS": "linux", "GOARCH": "amd64", + "GOAMD64": "v1", "GOEXPERIMENT": "", "GO111MODULE": "on", + "GOPRIVATE": "", "GONOPROXY": "", "GONOSUMDB": ""} + if args.offline: + env.update(GOPROXY="off", GOSUMDB="off") + manifest["regression_test_sha256"] = test_hashes + app = work / "gost-3.2.6" + with (work / "build.log").open("w") as log: + def run_go(*args, cwd=app): + command([go, *args], cwd=cwd, env=env, stdout=log, stderr=subprocess.STDOUT, timeout=900) + for module_dir in (app, work / "x-0.10.9"): + run_go("mod", "edit", "-go=1.26.8", "-toolchain=none", + *[f"-require={name}@{version}" for name, version in SECURITY_MINIMA.items()], cwd=module_dir) + run_go("mod", "tidy", cwd=work / "x-0.10.9") + run_go("mod", "edit", "-go=1.26.8", "-toolchain=none", + "-require=github.com/go-gost/x@v0.10.9", "-require=github.com/go-gost/core@v0.4.1", + "-replace=github.com/go-gost/x=../x-0.10.9") + run_go("mod", "tidy") + for filename, digest in LOCKS.items(): + if sha(app / filename) != digest: + raise ValueError(f"Dependency lock drift: {filename}") + run_go("mod", "verify") + run_go("test", "-p", "4", "./internal/util/sniffing", "./handler/dns", "./handler/redirect/tcp", "./resolver/exchanger", + cwd=work / "x-0.10.9") + context = work / "image" + context.mkdir() + stage_archives(apk_lock, cache / "component-apks", context / "packages", download=not args.offline) + manifest["component_apks"] = apk_lock + (context / "Dockerfile").write_bytes(recipe_bytes) + for name in SOURCES: + shutil.copyfile(work / name / "LICENSE", context / f"LICENSE.{name}") + harbor = distribution("harbor") + if harbor.version != "0.22.0": + raise ValueError("Base helper notice collection requires Harbor 0.22.0") + harbor_licenses = [harbor.locate_file(file) for file in harbor.files + if str(file).endswith(".dist-info/licenses/LICENSE")] + if len(harbor_licenses) != 1: + raise ValueError("Harbor's packaged license is unavailable") + for source, name in ((ROOT / "LICENSE", "LICENSE.Search-SWE"), + (Path(harbor_licenses[0]), "LICENSE.Harbor-0.22.0")): + shutil.copyfile(source, context / name) + manifest["top_level_license_sha256"] = {p.name: sha(p) for p in context.glob("LICENSE.*")} + binary = context / "gost-searchswe" + run_go("build", "-mod=readonly", "-p", "4", "-trimpath", "-buildvcs=false", "-o", str(binary), "./cmd/gost") + run_go("mod", "verify") + build_info = json.loads(command([go, "version", "-m", "-json", str(binary)], env=env, capture_output=True).stdout) + linked_paths = [build_info["Main"]["Path"], *[module["Path"] for module in build_info["Deps"]]] + modules = list(json_stream(command([go, "list", "-m", "-json", *linked_paths], cwd=app, env=env, capture_output=True).stdout)) + goroot = command([go, "env", "GOROOT"], env=env, capture_output=True).stdout.strip() + notices = collect_notices(build_info, modules, goroot, context / "third-party") + manifest.update(build_info=build_info, notice_modules=len(notices["modules"]), + notice_index_sha256=sha(context / "third-party/index.json")) + (context / "component-manifest.json").write_text(json.dumps({ + "binary_sha256": sha(binary), "patch_sha256": patch_digest, "sources": SOURCES, "locks": LOCKS, + "notice_index_sha256": manifest["notice_index_sha256"], + "top_level_license_sha256": manifest["top_level_license_sha256"], + })) + # BuildKit FROM does not accept Docker image IDs. Use the caller's + # local name and verify the immutable base/layers before and after. + if inspect(args.base_image)["Id"] != base["Id"]: + raise ValueError("Base image changed during build") + if sha(patcher) != patch_digest: + raise ValueError("GOST patch source changed during build") + if any(sha(tests / name) != digest for name, digest in test_hashes.items()): + raise ValueError("GOST regression source changed during build") + command(["docker", "build", "--pull=false", "--network=none", "--build-arg", + f"BASE_IMAGE={args.base_image}", "--build-arg", f"PATCH_SHA={patch_digest}", + "-t", args.tag, str(context)], stdout=log, stderr=subprocess.STDOUT, timeout=300) + result = inspect(args.tag) + if inspect(args.base_image)["Id"] != base["Id"] or any(sha(ROOT / p) != digest for p, digest in inputs.items()): + raise ValueError("Base or build source changed during component build") + final_layers = result["RootFS"]["Layers"] + if len(final_layers) not in (1, 2): + raise ValueError("Unexpected merged filesystem or WORKDIR layer count") + manifest["final_layer_policy"] = "merged filesystem plus optional WORKDIR metadata layer; audit final OCI contents before release" + manifest.update(status="built", image_id=result["Id"], binary_sha256=sha(binary), locks=LOCKS) + print(f"Built component: {args.tag}\nImage ID: {result['Id']}\nEvidence: {work}") + finally: + if manifest["status"] == "building": + manifest["status"] = "failed" + (work / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--go", default="go") + parser.add_argument("--base-image", required=True) + parser.add_argument("--tag", default="searchswe-egress-component:dev1") + parser.add_argument("--work-dir", type=Path, required=True, help="New build/evidence directory") + parser.add_argument("--cache-dir", type=Path, required=True, help="Reusable public source/module cache") + parser.add_argument("--offline", action="store_true", help="Require cached sources/modules; disable Go network resolution") + build(parser.parse_args()) diff --git a/scripts/build_egress_gateway.py b/scripts/build_egress_gateway.py new file mode 100644 index 0000000..ce6c62c --- /dev/null +++ b/scripts/build_egress_gateway.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Build the gateway from checksum-pinned APKs with an offline Docker build. + +An explicit --download permits fetching missing public APKs; standard host curl +proxy configuration is honored. Images are preloaded, never pulled or pushed. +""" + +import argparse +import hashlib +import json +from pathlib import Path +import shutil +import stat +import subprocess +import uuid +import sys + + +ROOT = Path(__file__).resolve().parents[1] +SOURCE = ROOT / "environments/egress" +sys.path.insert(0, str(ROOT)) +from scripts.egress.build_audit import runtime_inventory + + +def digest(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def command(*args, **kwargs): + return subprocess.run(list(args), check=True, text=True, timeout=kwargs.pop("timeout", 60), **kwargs) + + +def inspect(image): + return json.loads(command("docker", "image", "inspect", image, capture_output=True).stdout)[0] + + +def stage_archives(lock, cache, context, download=False): + cache.mkdir(parents=True, exist_ok=True) + context.mkdir() + for name, expected in lock["archives"].items(): + if Path(name).name != name or not name.endswith(".apk"): + raise ValueError("Invalid pinned APK filename") + cached = cache / name + target = context / name + if not cached.exists(): + if not download: + raise ValueError(f"Missing cached APK: {name}; use --download explicitly") + command("curl", "--fail", "--silent", "--show-error", "--proto", "=https", "--max-time", "120", + "--output", str(target), lock["repository"] + name, timeout=125) + if digest(target) != expected: + raise ValueError(f"Downloaded APK checksum mismatch: {name}") + shutil.copyfile(target, cached) + shutil.copyfile(cached, target) + if digest(target) != expected: + raise ValueError(f"Cached APK checksum mismatch: {name}") + + +def build(args): + work = args.work_dir.resolve() + work.mkdir(parents=True, exist_ok=False) + manifest = {"status": "building", "requested_tag": args.tag} + probe = "searchswe-runtime-build-" + uuid.uuid4().hex[:12] + try: + base = inspect(args.component_image) + if base.get("Os") != "linux" or base.get("Architecture") != "amd64": + raise ValueError("Only Linux amd64 component images are supported") + labels = base["Config"].get("Labels") or {} + if (labels.get("org.search-swe.egress.component") != "gost-x-0.10.9-searchswe-1" + or labels.get("org.search-swe.egress.patch-sha256") != digest(SOURCE / "patch_gost.py")): + raise ValueError("Component image is not from the pinned current patch") + if args.component_image.startswith("sha256:"): + raise ValueError("BuildKit needs a preloaded local image name, not a bare image ID") + if subprocess.run(["docker", "image", "inspect", args.tag], capture_output=True, timeout=15).returncode == 0: + raise ValueError("Use a new gateway image tag; do not overwrite an existing build") + context = work / "image" + context.mkdir() + sources = {} + for name in ("Dockerfile", "gateway.py", "entrypoint.sh", "network-policy", "runtime.lock.json"): + data = (SOURCE / name).read_bytes() + (context / name).write_bytes(data) + sources[name] = hashlib.sha256(data).hexdigest() + lock = json.loads((context / "runtime.lock.json").read_text()) + manifest.update(base_image_id=base["Id"], sources=sources, archives=lock["archives"]) + stage_archives(lock, args.cache_dir.resolve(), context / "packages", args.download) + with (work / "build.log").open("w") as log: + command("docker", "build", "--pull=false", "--network=none", "--build-arg", "COMPONENT_IMAGE=" + args.component_image, + "--build-arg", "GATEWAY_SOURCE_SHA=" + sources["gateway.py"], + "--build-arg", "RUNTIME_LOCK_SHA=" + sources["runtime.lock.json"], "-t", args.tag, str(context), + stdout=log, stderr=subprocess.STDOUT, timeout=600) + image = inspect(args.tag) + manifest["image_id"] = image["Id"] + if (inspect(args.component_image)["Id"] != base["Id"] + or image["RootFS"]["Layers"][:len(base["RootFS"]["Layers"])] != base["RootFS"]["Layers"]): + raise ValueError("Component image changed during runtime build") + # Inspect an unstarted container. Package verification needs no image + # process, namespace startup, credentials, capabilities or network. + command("docker", "create", "--pull", "never", "--name", probe, "--label", "searchswe.runtime-build=" + probe, + "--network", "none", "--cap-drop", "ALL", "--read-only", "--security-opt", "no-new-privileges:true", + "--entrypoint", "/bin/true", image["Id"], capture_output=True) + for source, name in (("/lib/apk/db/installed", "installed"), ("/etc/alpine-release", "alpine-release"), + ("/usr/share/searchswe-egress/runtime.json", "runtime.json")): + target = work / name + command("docker", "cp", probe + ":" + source, str(target), capture_output=True) + info = target.lstat() + if not stat.S_ISREG(info.st_mode) or info.st_size > 2_000_000: + raise ValueError("Unexpected runtime metadata file") + records = [dict(line.split(":", 1) for line in record.splitlines() if ":" in line) + for record in (work / "installed").read_text().split("\n\n")] + actual = {v["P"]: v["V"] for v in records if "P" in v} + if (actual != lock["installed"] or (work / "alpine-release").read_text().strip() != lock["alpine_release"] + or digest(work / "runtime.json") != sources["runtime.lock.json"]): + raise ValueError("Installed runtime package/lock drift") + manifest["runtime"] = {"packages": actual, "alpine_release": lock["alpine_release"]} + (work / "runtime-packages.json").write_text(json.dumps(runtime_inventory((work / "installed").read_text()), indent=2) + "\n") + manifest["runtime_inventory_sha256"] = digest(work / "runtime-packages.json") + for name, expected in sources.items(): + if digest(SOURCE / name) != expected: + raise ValueError("Runtime source changed during build") + manifest.update(status="passed", image_id=image["Id"]) + except Exception as error: + manifest.update(status="failed", error_class=type(error).__name__) + raise + finally: + try: + response = subprocess.run(["docker", "container", "inspect", probe], capture_output=True, text=True, timeout=15) + if response.returncode == 0: + info = json.loads(response.stdout)[0] + if info["Config"].get("Labels", {}).get("searchswe.runtime-build") != probe: + raise RuntimeError("Runtime probe ownership mismatch") + command("docker", "rm", "-f", probe, capture_output=True) + elif "No such" not in response.stderr: + raise RuntimeError("Could not verify runtime probe cleanup") + except Exception: + manifest["status"] = "cleanup_error" + raise + finally: + (work / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--component-image", required=True) + parser.add_argument("--tag", required=True) + parser.add_argument("--work-dir", required=True, type=Path) + parser.add_argument("--cache-dir", required=True, type=Path) + parser.add_argument("--download", action="store_true") + build(parser.parse_args()) diff --git a/scripts/docker_dns.py b/scripts/docker_dns.py index f46e4d0..912e307 100644 --- a/scripts/docker_dns.py +++ b/scripts/docker_dns.py @@ -1,14 +1,6 @@ -"""Opt-in upstream DNS for Harbor's isolated Docker network namespace. - -Docker's embedded resolver otherwise forwards to the host's systemd-resolved -stub. Explicit upstreams execute in the container namespace, where Harbor's -UDP deny rule also blocks DNS. Exempt only those servers on DNS port 53. -""" +"""Parse the optional direct-gateway DNS override.""" import ipaddress -import json -from pathlib import Path -import tempfile def parse_servers(value): @@ -20,31 +12,3 @@ def parse_servers(value): if str(address) not in servers: servers.append(str(address)) return servers - - -def create_overlay(servers): - from harbor.environments.docker import docker - - original = Path(docker.__file__).parent / 'harbor-docker-egress-control-sidecar/bin/network-policy' - if not original.is_file(): - raise RuntimeError('Installed Harbor does not provide the expected network-policy helper') - wrapper = Path(__file__).with_name('docker_dns_policy.sh').resolve() - directory = Path(tempfile.mkdtemp(prefix='searchswe-dns-')) - # Keep the installed Harbor policy implementation; no duplicated firewall. - snapshot = directory / 'network-policy-original' - snapshot.write_bytes(original.read_bytes()) - overlay = { - 'services': {'harbor-docker-egress-control-sidecar': { - 'dns': servers, - 'environment': {'SEARCH_SWE_DNS_SERVERS': ' '.join(servers)}, - 'volumes': [ - {'type': 'bind', 'source': str(snapshot), - 'target': '/opt/searchswe/network-policy-original', 'read_only': True}, - {'type': 'bind', 'source': str(wrapper), - 'target': '/usr/local/bin/network-policy', 'read_only': True}, - ], - }}, - } - path = directory / 'compose.json' - path.write_text(json.dumps(overlay, indent=2) + '\n') - return path diff --git a/scripts/docker_dns_policy.sh b/scripts/docker_dns_policy.sh deleted file mode 100755 index abb7e28..0000000 --- a/scripts/docker_dns_policy.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -# Preserve Harbor's dynamic policies; exceptions exist only in allowlist mode. -set -eu -/bin/sh /opt/searchswe/network-policy-original "$@" -if [ "${1:-}" = allow ] && [ "$#" -gt 1 ]; then - for server in ${SEARCH_SWE_DNS_SERVERS:?}; do - # Server values are IPv4-validated by the trusted host launcher. - nft insert rule inet gost_egress output ip daddr "$server" tcp dport 53 return - nft insert rule inet gost_egress egress ip daddr "$server" udp dport 53 accept - done -fi diff --git a/scripts/egress/__init__.py b/scripts/egress/__init__.py new file mode 100644 index 0000000..880b328 --- /dev/null +++ b/scripts/egress/__init__.py @@ -0,0 +1 @@ +"""Host-side configuration and orchestration for per-task network isolation.""" diff --git a/scripts/egress/build_audit.py b/scripts/egress/build_audit.py new file mode 100644 index 0000000..d1aedbe --- /dev/null +++ b/scripts/egress/build_audit.py @@ -0,0 +1,75 @@ +"""Collect linked dependency notices and runtime package metadata for image builds.""" + +import hashlib +import json +from pathlib import Path +import re + + +LEGAL = re.compile(r"^(licen[cs]e|copying|notice|patents|copyright|authors)([._-].*)?$", re.I) + + +def json_stream(text): + decoder = json.JSONDecoder() + position = 0 + while position < len(text): + if text[position].isspace(): + position += 1 + continue + value, position = decoder.raw_decode(text, position) + yield value + + +def collect_notices(build_info, modules, goroot, output): + """Preserve notices for every module actually linked into the Go binary.""" + output.mkdir(parents=True, exist_ok=False) + available = {entry["Path"]: entry for entry in modules} + rows = [] + linked = [build_info["Main"], *build_info.get("Deps", [])] + linked.append({"Path": "go-standard-library", "Version": build_info["GoVersion"]}) + for index, dependency in enumerate(sorted(linked, key=lambda value: value["Path"])): + path = dependency["Path"] + if path == "go-standard-library": + root = Path(goroot) + else: + entry = available[path] + root = Path(entry.get("Replace", entry)["Dir"]) + if not root.is_dir(): + raise ValueError("Missing linked-module source directory") + row = {"module": path, "version": dependency.get("Version"), "sum": dependency.get("Sum"), + "replacement": dependency.get("Replace"), "notices": []} + licensed = False + for source in sorted(root.rglob("*")): + if not LEGAL.fullmatch(source.name) or source.is_dir(): + continue + if source.is_symlink() or not source.is_file() or source.stat().st_size > 2_000_000: + raise ValueError("Unexpected module notice file") + relative = source.relative_to(root) + target = output / str(index) / relative + target.parent.mkdir(parents=True, exist_ok=True) + data = source.read_bytes() + target.write_bytes(data) + row["notices"].append({"source": relative.as_posix(), "file": target.relative_to(output).as_posix(), + "sha256": hashlib.sha256(data).hexdigest()}) + licensed |= source.name.lower().startswith(("license", "licence", "copying")) + if not licensed: + raise ValueError("Linked module has no collected license: " + path) + rows.append(row) + document = {"version": 1, "scope": "linked Go module and standard-library notice inventory; not legal clearance", + "modules": rows} + (output / "index.json").write_text(json.dumps(document, indent=2) + "\n") + return document + + +def runtime_inventory(installed): + rows = [] + for record in installed.split("\n\n"): + fields = dict(line.split(":", 1) for line in record.splitlines() if ":" in line) + if "P" not in fields: + continue + rows.append({"name": fields["P"], "version": fields["V"], "license_declared": fields.get("L"), + "origin": fields.get("o"), "aports_commit": fields.get("c"), "upstream_url": fields.get("U")}) + if not rows or any(not row["license_declared"] or not row["origin"] or not row["aports_commit"] for row in rows): + raise ValueError("Incomplete runtime package license/source metadata") + return {"version": 1, "scope": "APK-declared metadata; does not supply corresponding source or legal clearance", + "packages": sorted(rows, key=lambda row: row["name"])} diff --git a/scripts/egress/compose.py b/scripts/egress/compose.py new file mode 100644 index 0000000..fbfcab4 --- /dev/null +++ b/scripts/egress/compose.py @@ -0,0 +1,108 @@ +"""Reject unsupported topology before launch; verify the final merged Compose.""" + +import os +from pathlib import Path +import stat + + +SERVICE = "harbor-docker-egress-control-sidecar" +NETWORK = f"service:{SERVICE}" + + +def declared_services(documents): + names = {"main"} + for document in documents: + if not isinstance(document, dict): + raise ValueError("Invalid Compose document") + if document.get("networks"): + raise ValueError("Explicit Compose networks are not supported with restricted egress") + for name, service in document.get("services", {}).items(): + if name == SERVICE: + raise ValueError("Task Compose cannot define the trusted egress service") + if not isinstance(service, dict) or "network_mode" in service or "networks" in service: + raise ValueError("Explicit service networking is not supported with restricted egress") + names.add(name) + return sorted(names) + + +def validate_final(document, *, private_directory, image, engine_paths=(), protected_paths=()): + if document.get("configs") or document.get("secrets"): + raise ValueError("Compose config/secret objects are unsupported with restricted egress") + services = document.get("services", {}) + if SERVICE not in services or "main" not in services: + raise ValueError("Restricted-egress Compose is missing required services") + gateway = services[SERVICE] + if gateway.get("image") != image or gateway.get("entrypoint") != ["/opt/egress-sidecar/entrypoint.sh"]: + raise ValueError("Trusted gateway image/entrypoint was overridden") + if gateway.get("network_mode") or set(gateway.get("networks", {})) != {"default"}: + raise ValueError("Gateway must use its own default Compose network") + if gateway.get("ports") or gateway.get("privileged") or gateway.get("pid") or gateway.get("ipc") == "host": + raise ValueError("Unsupported trusted gateway topology") + if not gateway.get("read_only") or "/run/searchswe:mode=0700" not in gateway.get("tmpfs", []): + raise ValueError("Gateway runtime credentials must remain on private tmpfs") + private = Path(private_directory).resolve() + + def check_private_source(source): + source = Path(source).resolve() + if source == private or source in private.parents or private in source.parents: + raise ValueError("Untrusted service can read the private gateway directory") + for protected in protected_paths: + protected = Path(protected).resolve() + if source == protected or source in protected.parents or protected in source.parents: + raise ValueError("Untrusted service can access gateway configuration or proxy credentials") + controls = {Path(p).resolve() for p in ("/proc", "/sys", "/dev", "/run", "/var/lib/docker", "/var/lib/containerd", *engine_paths)} + if any(source == control or source in control.parents or control in source.parents for control in controls): + raise ValueError("Untrusted service has a host control mount/build context") + return source + + for name, service in services.items(): + if name == SERVICE: + continue + build = service.get("build") or {} + if build: + if (build.get("additional_contexts") or build.get("secrets") or build.get("privileged") + or build.get("entitlements") or build.get("network") not in {None, "default", "none"}): + raise ValueError("Unsupported privileged/extra-context task image build") + context = build.get("context", "") + if context and "://" not in context: + check_private_source(context) + if service.get("network_mode") != NETWORK or service.get("networks"): + raise ValueError(f"Service {name} is not in its own trial's gateway namespace") + dropped = {str(cap).removeprefix("CAP_").upper() for cap in service.get("cap_drop", [])} + if "ALL" not in dropped and not {"NET_RAW", "NET_ADMIN"} <= dropped: + raise ValueError(f"Service {name} retains network-mark capabilities") + if service.get("cap_add") or service.get("privileged"): + raise ValueError(f"Service {name} requests unsupported capabilities") + if service.get("pid") or service.get("ipc") == "host" or service.get("userns_mode"): + raise ValueError(f"Service {name} requests unsupported namespaces") + if service.get("ports") or service.get("devices") or service.get("device_cgroup_rules"): + raise ValueError(f"Service {name} exposes ports or raw devices") + if service.get("configs") or service.get("secrets") or service.get("container_name"): + raise ValueError(f"Service {name} uses unsupported configs/secrets/container names") + if any(key.startswith("net.") for key in service.get("sysctls", {})): + raise ValueError(f"Service {name} requests network sysctls") + security = service.get("security_opt", []) + if not any(s in {"no-new-privileges:true", "no-new-privileges"} for s in security): + raise ValueError(f"Service {name} is missing no-new-privileges") + if any("unconfined" in s for s in security): + raise ValueError(f"Service {name} disables a security profile") + if service.get("depends_on", {}).get(SERVICE, {}).get("condition") != "service_healthy": + raise ValueError(f"Service {name} can start before gateway readiness") + for mount in service.get("volumes", []): + if mount.get("type") != "bind": + continue + source = check_private_source(mount["source"]) + # Sockets and host kernel control mounts permit bypassing network + # capabilities. Task assets/log bind directories remain supported. + if source.exists() and stat.S_ISSOCK(os.stat(source).st_mode): + raise ValueError("Untrusted service has a host socket mount") + if source.name in {"docker.sock", "containerd.sock", "podman.sock"}: + raise ValueError("Untrusted service has a container-engine socket mount") + for network in document.get("networks", {}).values(): + if network.get("external") or network.get("enable_ipv6"): + raise ValueError("External/IPv6 Compose networks are unsupported") + for volume in document.get("volumes", {}).values(): + if volume.get("external") or not volume.get("name", "").startswith(document.get("name", "") + "_"): + raise ValueError("Shared/external named volumes are unsupported") + if volume.get("driver") not in {None, "local"} or volume.get("driver_opts"): + raise ValueError("Named volume drivers/options may bypass host mount validation") diff --git a/scripts/egress/config.py b/scripts/egress/config.py new file mode 100644 index 0000000..2b9bc8c --- /dev/null +++ b/scripts/egress/config.py @@ -0,0 +1,203 @@ +"""Strict public configuration; credentials are loaded only by the host adapter.""" + +from dataclasses import dataclass, field +from importlib.metadata import version +import ipaddress +import json +import os +from pathlib import Path +import re +import stat +from urllib.parse import urlsplit + + +HOST = re.compile(r"(?=.{1,253}\Z)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z](?:[a-z0-9-]{0,61}[a-z0-9])?\Z") +DEFAULT_IMAGE = "hanhainebula/search-swe-egress:1.0.0" + + +def require_supported_harbor(): + if version("harbor") != "0.22.0": + raise ValueError("Restricted egress is currently validated only against Harbor 0.22.0") + + +def exact_hosts(hosts): + if not isinstance(hosts, (list, tuple)) or any(not isinstance(h, str) or not HOST.fullmatch(h) for h in hosts): + raise ValueError("Restricted egress supports only lowercase exact DNS hosts (no IP/CIDR/wildcards)") + return sorted(set(hosts)) + + +def check_keys(value, required, optional=()): + if not isinstance(value, dict) or not set(required) <= set(value) or set(value) - set(required) - set(optional): + raise ValueError("Invalid restricted-egress configuration keys") + + +def endpoint(value, *, doh=False): + try: + if not isinstance(value, str) or any(ord(c) <= 32 for c in value): + raise ValueError() + parsed = urlsplit(value) + port = parsed.port or (443 if parsed.scheme == "https" else 80) + if (parsed.scheme not in ({"https"} if doh else {"http", "https"}) + or not parsed.hostname or parsed.username is not None or parsed.password is not None + or parsed.query or parsed.fragment or not 1 <= port <= 65535 + or (not doh and parsed.path not in {"", "/"})): + raise ValueError() + if not HOST.fullmatch(parsed.hostname): + ipaddress.IPv4Address(parsed.hostname) + return parsed, port + except (ValueError, TypeError): + # Never echo the supplied URL: rejected input may contain a password. + raise ValueError("Invalid egress endpoint: use HTTP(S), no userinfo/query/fragment; DoH requires HTTPS") from None + + +def read_auth(path): + parent = path.parent.stat() + if parent.st_uid != os.getuid() or stat.S_IMODE(parent.st_mode) != 0o700: + raise ValueError("Proxy credential directory must be owned by the current user and mode 0700") + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + with os.fdopen(fd) as stream: + info = os.fstat(stream.fileno()) + if (not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() + or stat.S_IMODE(info.st_mode) != 0o600 or info.st_nlink != 1): + raise ValueError("Proxy credential file must be owned by the current user and mode 0600") + try: + content = stream.read(16385) + if len(content) > 16384: + raise ValueError() + auth = json.loads(content) + check_keys(auth, {"username", "password"}) + if any(not isinstance(auth[k], str) or not auth[k] or len(auth[k]) > 4096 for k in auth): + raise ValueError() + except (ValueError, TypeError): + raise ValueError("Proxy credential file must contain a bounded username/password JSON object") from None + return auth + + +@dataclass(frozen=True) +class EgressConfig: + path: Path | None + image: str + settings: dict = field(repr=False) + auth_path: Path | None = field(default=None, repr=False) + + def private_settings(self): + settings = dict(self.settings) + if self.auth_path is not None: + settings["auth"] = read_auth(self.auth_path) + return settings + + +def direct_config(image=DEFAULT_IMAGE, dns_servers=None, path=None): + """Docker's embedded resolver is used only by the trusted DNS worker.""" + validate_image(image) + servers = ["127.0.0.11"] if dns_servers is None else dns_servers + if not isinstance(servers, list) or not servers: + raise ValueError("Direct DNS requires a nonempty list of IPv4 servers") + for value in servers: + try: + if not isinstance(value, str): + raise ValueError() + host, separator, port = value.partition(":") + if separator and (not port.isascii() or not port.isdecimal() or not 1 <= int(port) <= 65535): + raise ValueError() + address = ipaddress.IPv4Address(host) + if (address.is_unspecified or address.is_multicast + or address.is_link_local or address.is_reserved + or (address.is_loopback and (host != "127.0.0.11" or (separator and int(port) != 53)))): + raise ValueError() + except (ValueError, TypeError): + raise ValueError("Direct DNS requires IPv4 servers reachable from Docker; only Docker's 127.0.0.11 loopback resolver is supported") from None + return EgressConfig(path, image, {"transport": "direct", "dns_servers": list(dict.fromkeys(servers))}) + + +def validate_image(image): + if (not isinstance(image, str) or not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9._:/@-]*", image) + or "://" in image + or ("@" in image and not re.search(r"@sha256:[0-9a-f]{64}\Z", image))): + raise ValueError("Gateway image must be a valid image name or sha256 digest reference") + + +def load_config(path): + path = Path(path).expanduser().resolve(strict=True) + if not path.is_file() or path.stat().st_nlink != 1: + raise ValueError("EGRESS_CONFIG must be a regular, single-link file") + try: + document = json.loads(path.read_text()) + except (ValueError, UnicodeError): + raise ValueError("EGRESS_CONFIG must be a JSON object") from None + check_keys(document, {"version", "image", "dns"}, {"upstream", "mode"}) + if type(document["version"]) is not int or document["version"] != 1: + raise ValueError("Unsupported EGRESS_CONFIG version") + validate_image(document["image"]) + mode = document.get("mode", "proxy") + if mode == "direct": + if "upstream" in document: + raise ValueError("Direct egress must not specify an upstream proxy") + check_keys(document["dns"], set(), {"servers"}) + return direct_config(document["image"], document["dns"].get("servers"), path) + if mode != "proxy" or "upstream" not in document: + raise ValueError("Egress mode must be direct or proxy; proxy requires upstream") + upstream, dns = document["upstream"], document["dns"] + check_keys(upstream, {"url"}, {"address", "auth_file"}) + check_keys(dns, {"doh_url"}) + parsed, port = endpoint(upstream["url"]) + endpoint(dns["doh_url"], doh=True) + # Bootstrap the operator's proxy independently of potentially polluted + # container DNS. HTTPS still verifies the URL hostname, not this address. + try: + address = upstream.get("address", parsed.hostname) + if not isinstance(address, str): + raise ValueError() + parsed_address = ipaddress.IPv4Address(address) + if (parsed_address.is_loopback or parsed_address.is_unspecified or parsed_address.is_multicast + or parsed_address.is_link_local or parsed_address.is_reserved): + raise ValueError() + address = str(parsed_address) + except (ValueError, TypeError): + raise ValueError("Upstream requires an explicit IPv4 address outside the task namespace; loopback/link-local are unsupported") from None + settings = {"upstream_addr": f"{address}:{port}", "upstream_ip": address, "upstream_port": port, + "upstream_host": parsed.hostname, "upstream_tls": parsed.scheme == "https", + "doh_url": dns["doh_url"]} + auth_path = None + if "auth_file" in upstream: + if not isinstance(upstream["auth_file"], str) or not upstream["auth_file"]: + raise ValueError("upstream.auth_file must be a nonempty path") + # Do not resolve the final component: read_auth rejects symlinks. + auth_path = path.parent / Path(upstream["auth_file"]).expanduser() + return EgressConfig(path, document["image"], settings, auth_path) + + +def validate_task_networks(task_path, model_host, proxy_host=None, *, allow_public=False): + """Use Harbor's real per-step resolver, never a union or parallel TOML model.""" + require_supported_harbor() + from harbor.models.task.config import TaskConfig, TaskOS + from harbor.models.task.verifier_mode import ( + resolve_effective_verifier_env_config, resolve_step_verifier_mode, resolve_task_verifier_mode, + ) + from harbor.models.trial.config import AgentConfig, EnvironmentConfig + from harbor.trial.network_policy import resolve_trial_network_plan + + task = TaskConfig.model_validate_toml((Path(task_path) / "task.toml").read_text()) + if task.environment.os != TaskOS.LINUX: + raise ValueError("Restricted egress currently requires Linux task environments") + restricted = False + for step in task.steps or [None]: + verifier = resolve_effective_verifier_env_config(task, step) + if verifier is not None and verifier.os != TaskOS.LINUX: + raise ValueError("Restricted egress currently requires Linux verifier environments") + mode = resolve_task_verifier_mode(task) if step is None else resolve_step_verifier_mode(task, step) + plan = resolve_trial_network_plan( + task, AgentConfig(extra_allowed_hosts=[model_host] if model_host else []), + EnvironmentConfig(), step, verifier_mode=mode, env_config=verifier) + for policy in (plan.agent_env_baseline, plan.agent_phase, plan.verifier_env_baseline, plan.verifier_phase): + if policy is None: + continue + if policy.network_mode.value not in {"allowlist", "no-network"}: + if not allow_public: + raise ValueError("Restricted proxy egress does not support tasks with any public phase") + continue + restricted = True + exact_hosts(policy.allowed_hosts) + if proxy_host in policy.allowed_hosts: + raise ValueError("A general upstream proxy cannot also be an allowed task/API destination") + return restricted diff --git a/scripts/egress/ownership.py b/scripts/egress/ownership.py new file mode 100644 index 0000000..a56c0da --- /dev/null +++ b/scripts/egress/ownership.py @@ -0,0 +1,83 @@ +"""Host-owned lifetime locks and recoverable private instance directories.""" + +import fcntl +import hashlib +import json +import os +from pathlib import Path +import shutil +import stat +import tempfile + + +LABEL = "searchswe.egress.instance" + + +def owner_root(): + root = Path(tempfile.gettempdir()) / f"searchswe-egress-owner-{os.getuid()}" + try: + root.mkdir(mode=0o700) + except FileExistsError: + pass + info = root.lstat() + if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) != 0o700: + raise ValueError("Unsafe restricted-egress ownership directory") + return root + + +class ProjectLock: + def __init__(self, project): + self.fd = None + name = "lock-" + hashlib.sha256(project.encode()).hexdigest() + fd = os.open(owner_root() / name, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600) + try: + info = os.fstat(fd) + if info.st_uid != os.getuid() or info.st_nlink != 1 or not stat.S_ISREG(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o600: + raise ValueError("Unsafe egress project lock") + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BaseException: + os.close(fd) + raise + self.fd = fd + + def close(self): + if self.fd is not None: + os.close(self.fd) + self.fd = None + # Never unlink a lock file: another owner may already hold its inode. + + +class PrivateDirectory: + """No GC cleanup: after host failure, preserve its orphan recovery manifest.""" + + def __init__(self, instance, project, image, *, daemon_id=None): + self.name = tempfile.mkdtemp(prefix="instance-" + instance + "-", dir=owner_root()) + self.manifest = {"version": 1, "instance": instance, "project": project, + "image": image, "private_directory": self.name, "daemon_id": daemon_id} + path = Path(self.name) / "owner.json" + with os.fdopen(os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "w") as stream: + json.dump(self.manifest, stream) + + def cleanup(self): + root = Path(self.name) + if read_manifest(root) != self.manifest: + raise RuntimeError("Private instance ownership changed; refusing removal") + shutil.rmtree(root) + + +def read_manifest(directory): + directory = Path(directory) + if directory.parent != owner_root() or not directory.name.startswith("instance-"): + raise ValueError("Not a managed egress instance directory") + info = directory.lstat() + if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) != 0o700: + raise ValueError("Unsafe private instance directory") + fd = os.open(directory / "owner.json", os.O_RDONLY | os.O_NOFOLLOW) + with os.fdopen(fd) as stream: + info = os.fstat(stream.fileno()) + if info.st_uid != os.getuid() or info.st_nlink != 1 or stat.S_IMODE(info.st_mode) != 0o600: + raise ValueError("Unsafe instance manifest") + data = json.load(stream) + if data.get("version") != 1 or data.get("private_directory") != str(directory): + raise ValueError("Instance manifest mismatch") + return data diff --git a/scripts/egress_cleanup.py b/scripts/egress_cleanup.py new file mode 100644 index 0000000..2650e73 --- /dev/null +++ b/scripts/egress_cleanup.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""List owned orphan manifests, or recover one after its host owner has exited. + +Default is read-only. --remove stops/removes only resources with matching +instance AND Compose project labels, then removes that instance's private files. +It never removes images, host datasets, unrelated containers, or unowned volumes. +""" + +import argparse +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from scripts.egress.ownership import LABEL, ProjectLock, owner_root, read_manifest + + +def docker(*args): + return subprocess.run(["docker", *args], text=True, capture_output=True, check=True, timeout=30).stdout + + +def inventory(manifest): + context = json.loads(docker("context", "inspect"))[0] + endpoint = (None if os.environ.get("DOCKER_CONTEXT") else os.environ.get("DOCKER_HOST")) or context["Endpoints"]["docker"]["Host"] + if not endpoint.startswith("unix://"): + raise ValueError("Orphan cleanup supports only the local Unix-socket Docker daemon") + daemon_id = json.loads(docker("info", "--format", "{{json .ID}}")) + if not daemon_id or daemon_id != manifest.get("daemon_id"): + raise ValueError("Docker daemon identity mismatch or legacy manifest; refusing recovery") + resources = [] + for kind in ("container", "network", "volume"): + flags = ["-aq"] if kind == "container" else ["-q"] + ids = docker(kind, "ls", *flags, "--filter", f"label={LABEL}={manifest['instance']}").split() + for identity in ids: + info = json.loads(docker(kind, "inspect", identity))[0] + labels = info["Config"].get("Labels", {}) if kind == "container" else info.get("Labels", {}) + if labels.get(LABEL) != manifest["instance"] or labels.get("com.docker.compose.project") != manifest["project"]: + raise ValueError("Resource ownership mismatch; nothing may be removed") + gateway = kind == "container" and labels.get("com.docker.compose.service") == "harbor-docker-egress-control-sidecar" + if gateway and info["Config"]["Image"] != manifest["image"]: + raise ValueError("Gateway image ownership mismatch") + resources.append({"kind": kind, "id": identity, "gateway": gateway}) + container_ids = {r["id"] for r in resources if r["kind"] == "container"} + for resource in resources: + if resource["kind"] == "network": + info = json.loads(docker("network", "inspect", resource["id"]))[0] + if not set(info.get("Containers", {})) <= container_ids: + # docker ls emits short IDs; compare exact inspect IDs below. + full_ids = {json.loads(docker("container", "inspect", identity))[0]["Id"] for identity in container_ids} + if not set(info.get("Containers", {})) <= full_ids: + raise ValueError("Owned network includes an unrelated container") + return resources + + +def recover(directory, remove=False): + directory = Path(directory).absolute() + manifest = read_manifest(directory) + lock = ProjectLock(manifest["project"]) + try: + resources = inventory(manifest) + audit_paths = [] + if remove: + # Fence by killing the trusted gateway before removing other actors. + for resource in sorted(resources, key=lambda r: (not r["gateway"], r["kind"] != "container")): + kind, identity = resource["kind"], resource["id"] + # Recheck immediately before mutation, including project labels. + if resource not in inventory(manifest): + raise ValueError("Resource ownership changed during recovery") + if resource["gateway"]: + # Fence first, then preserve the controller's sanitized + # audit before deleting its Docker log. An audit I/O error + # leaves the stopped resource and manifest recoverable. + state = json.loads(docker("container", "inspect", identity))[0] + if state["State"]["Running"]: + docker("container", "kill", identity) + audit = docker("logs", identity) + fd, path = tempfile.mkstemp(prefix="recovery-audit-", suffix=".log", dir=directory.parent) + with os.fdopen(fd, "w") as stream: + stream.write(audit) + stream.flush() + os.fsync(stream.fileno()) + parent_fd = os.open(directory.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(parent_fd) + finally: + os.close(parent_fd) + audit_paths.append(path) + docker(kind, "rm", *(["-f"] if kind == "container" else []), identity) + if inventory(manifest): + raise RuntimeError("Owned resources remain after recovery") + if read_manifest(directory) != manifest: + raise RuntimeError("Private manifest changed during recovery") + shutil.rmtree(directory) + return {"instance": manifest["instance"], "project": manifest["project"], + "resources": resources, "removed": remove, "audit_paths": audit_paths} + finally: + lock.close() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--directory", type=Path) + parser.add_argument("--remove", action="store_true") + args = parser.parse_args() + if args.remove and args.directory is None: + parser.error("--remove requires one explicit --directory") + if args.directory is not None: + print(json.dumps(recover(args.directory, args.remove), indent=2)) + else: + rows = [] + for directory in sorted(owner_root().glob("instance-*")): + manifest = read_manifest(directory) + rows.append({"directory": str(directory), "instance": manifest["instance"], "project": manifest["project"]}) + print(json.dumps(rows, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/harbor_environments.py b/scripts/harbor_environments.py new file mode 100644 index 0000000..5d63aec --- /dev/null +++ b/scripts/harbor_environments.py @@ -0,0 +1,532 @@ +"""Phase-scoped direct/proxy Docker extension; never patches installed Harbor.""" + +import asyncio +import hashlib +import json +import logging +import os +from pathlib import Path +import platform +import subprocess +import time +import uuid + +import yaml +from harbor.environments.capabilities import EnvironmentCapabilities +from harbor.environments.docker.docker import DockerEnvironment, _sanitize_docker_compose_project_name +from harbor.models.task.config import NetworkMode, TaskOS + +from scripts.egress.compose import SERVICE, declared_services, validate_final +from scripts.egress.config import DEFAULT_IMAGE, direct_config, exact_hosts, load_config, require_supported_harbor +from scripts.egress.ownership import ProjectLock, PrivateDirectory, owner_root +from environments.egress.gateway import LEASE_SECONDS + + +def docker_json(*arguments, timeout=15): + response = subprocess.run(["docker", *arguments], check=True, capture_output=True, text=True, timeout=timeout) + return json.loads(response.stdout) + + +def gateway_image_info(reference): + try: + return docker_json("image", "inspect", reference)[0] + except subprocess.CalledProcessError as error: + # Only a missing image permits a pull; daemon/permission errors must + # retain their original cause rather than becoming registry requests. + if "no such image:" not in (error.stderr or "").lower(): + raise + logging.getLogger(__name__).info("Pulling missing egress gateway image %s", reference) + try: + subprocess.run(["docker", "pull", "--platform", "linux/amd64", reference], + check=True, capture_output=True, text=True, timeout=300) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as error: + raise RuntimeError( + f"Could not pull egress gateway image {reference}; check Docker registry " + "access/authentication and the Docker daemon's proxy settings, or preload " + "the image with docker pull. Task EGRESS_CONFIG does not configure image downloads." + ) from error + return docker_json("image", "inspect", reference)[0] + + +class PhaseScopedDocker(DockerEnvironment): + """Linux/rootful/local, exact-host, phase-scoped direct or proxy egress.""" + + def __init__(self, *args, egress_config: str | None = None, egress_image: str = DEFAULT_IMAGE, + egress_dns: str | None = None, **kwargs): + require_supported_harbor() + if egress_config is not None and (egress_dns is not None or egress_image != DEFAULT_IMAGE): + raise ValueError("egress_config cannot be combined with egress_image/egress_dns overrides") + self.egress = (load_config(egress_config) if egress_config is not None else + direct_config(egress_image, egress_dns.split(",") if egress_dns is not None else None)) + self._private = None + self._gateway_overlay = None + self._gateway_image = None + self._engine_paths = () + self._policy_lock = asyncio.Lock() + self._heartbeat = None + self._heartbeat_error = None + self._starting = True + self._instance = uuid.uuid4().hex + self._owner_lock = None + self._compose_started = False + self._rendered_document = None + self._daemon_id = None + super().__init__(*args, **kwargs) + if self.os != TaskOS.LINUX or not self._enable_egress_control or self._keep_containers: + raise ValueError("Restricted egress requires Linux isolation and does not support keep_containers") + for policy in (self.network_policy, *self._phase_network_policies): + self._check_policy(policy) + + @staticmethod + def _requires_egress_control(*, startup_network_policy, phase_network_policies): + # Explicit selection also supports a direct public phase. Keep the + # controller present so a later restriction never needs a new namespace. + return True + + @staticmethod + def _egress_control_kernel_support(): + # The trusted controller executes real nft transactions before becoming + # healthy; no main service may start before that. Avoid Harbor's separate + # probe image/pull, but never bypass the actual kernel enforcement gate. + return True + + @property + def capabilities(self): + return EnvironmentCapabilities(disable_internet=True, network_allowlist=True, + network_allowlist_hostnames=True, dynamic_network_policy=True, + mounted=True, docker_compose=True) + + def _check_policy(self, policy): + if (policy.network_mode == NetworkMode.PUBLIC + and self.egress.settings.get("transport") == "direct"): + return + if policy.network_mode not in (NetworkMode.ALLOWLIST, NetworkMode.NO_NETWORK): + raise ValueError("Restricted egress does not support any public phase") + exact_hosts(policy.allowed_hosts) + if self.egress.settings.get("upstream_host") in policy.allowed_hosts: + raise ValueError("An upstream proxy cannot also be an allowed task/API destination") + if policy.network_mode == NetworkMode.NO_NETWORK and policy.allowed_hosts: + raise ValueError("no-network policy cannot include allowed_hosts") + + def _preflight_gateway(self): + if platform.system() != "Linux" or platform.machine() not in {"x86_64", "amd64"}: + raise ValueError("Restricted egress currently supports local Linux amd64 only") + context = docker_json("context", "inspect")[0] + endpoint = (None if os.environ.get("DOCKER_CONTEXT") else os.environ.get("DOCKER_HOST")) or context["Endpoints"]["docker"]["Host"] + if not endpoint.startswith("unix://"): + raise ValueError("Restricted egress requires a local Unix-socket Docker daemon") + info = docker_json("info", "--format", "{{json .}}") + security = " ".join(info.get("SecurityOptions", [])) + if (info.get("OSType") != "linux" or "rootless" in security or "userns" in security + or "desktop" in info.get("OperatingSystem", "").lower()): + raise ValueError("Restricted egress requires native Linux rootful Docker without userns remapping") + if info.get("Architecture") not in {"x86_64", "amd64"} or info.get("KernelVersion") != platform.release(): + raise ValueError("Host and Docker daemon must share the same native Linux amd64 kernel") + if not isinstance(info.get("ID"), str) or not info["ID"]: + raise ValueError("Docker daemon identity is unavailable") + self._daemon_id = info["ID"] + self._engine_paths = (endpoint.removeprefix("unix://"), info["DockerRootDir"]) + project = _sanitize_docker_compose_project_name(self.session_id) + self._owner_lock = ProjectLock(project) + existing = subprocess.run(["docker", "ps", "-aq", "--filter", f"label=com.docker.compose.project={project}"], + check=True, capture_output=True, text=True, timeout=15) + if existing.stdout.strip(): + raise ValueError("Existing Compose project; recover its owned orphan before starting a new instance") + for resource in ("network", "volume"): + existing = subprocess.run(["docker", resource, "ls", "-q", "--filter", f"label=com.docker.compose.project={project}"], + check=True, capture_output=True, text=True, timeout=15) + if existing.stdout.strip(): + raise ValueError("Existing Compose resources; recover the owned orphan before starting") + self._engine_paths = (*self._engine_paths, str(owner_root())) + image = gateway_image_info(self.egress.image) + labels = image["Config"].get("Labels", {}) + if (image.get("Os") != "linux" or image.get("Architecture") != "amd64" + or labels.get("org.search-swe.egress.gateway") != "1" + or labels.get("org.search-swe.egress.component") != "gost-x-0.10.9-searchswe-1"): + raise ValueError("Expected a supported Search-SWE gateway image (linux/amd64)") + sources = Path(__file__).resolve().parents[1] / "environments/egress" + gateway_sha = hashlib.sha256((sources / "gateway.py").read_bytes()).hexdigest() + patch_sha = hashlib.sha256((sources / "patch_gost.py").read_bytes()).hexdigest() + runtime_sha = hashlib.sha256((sources / "runtime.lock.json").read_bytes()).hexdigest() + if (labels.get("org.search-swe.egress.source-sha256") != gateway_sha + or labels.get("org.search-swe.egress.patch-sha256") != patch_sha + or labels.get("org.search-swe.egress.runtime-lock-sha256") != runtime_sha): + raise ValueError("Gateway image is stale or unverified; rebuild it from this checkout") + # Verify contents as well as labels. This unprivileged, networkless + # preflight has no configuration/credential mounts, and is always removed. + name = "searchswe-egress-preflight-" + uuid.uuid4().hex[:12] + script = """import hashlib,json +from pathlib import Path +m=json.loads(Path('/usr/share/searchswe-egress/component.json').read_text()) +assert hashlib.sha256(Path('/bin/gost').read_bytes()).hexdigest()==m['binary_sha256'] +runtime=Path('/usr/share/searchswe-egress/runtime.json').read_bytes() +lock=json.loads(runtime) +records=[dict(line.split(':',1) for line in record.splitlines() if ':' in line) + for record in Path('/lib/apk/db/installed').read_text().split('\\n\\n')] +assert {v['P']:v['V'] for v in records if 'P' in v}==lock['installed'] +assert Path('/etc/alpine-release').read_text().strip()==lock['alpine_release'] +print(json.dumps({'gateway':hashlib.sha256(Path('/opt/searchswe/gateway.py').read_bytes()).hexdigest(), + 'patch':m['patch_sha256'],'runtime':hashlib.sha256(runtime).hexdigest()})) +""" + try: + contents = docker_json("run", "--pull", "never", "--name", name, + "--label", f"searchswe.preflight={name}", "--network", "none", "--read-only", + "--cap-drop", "ALL", "--security-opt", "no-new-privileges:true", + "--memory", "256m", "--cpus", "1", "--entrypoint", "python3", image["Id"], "-c", script, + timeout=60) + if contents != {"gateway": gateway_sha, "patch": patch_sha, "runtime": runtime_sha}: + raise ValueError("Gateway image contents do not match this checkout") + finally: + inspected = subprocess.run(["docker", "container", "inspect", name], capture_output=True, text=True, timeout=15) + if inspected.returncode and "No such" not in inspected.stderr: + raise RuntimeError("Could not verify gateway preflight cleanup") + if inspected.returncode == 0: + owned = json.loads(inspected.stdout)[0]["Config"].get("Labels", {}) + if owned.get("searchswe.preflight") != name: + raise RuntimeError("Gateway preflight container ownership mismatch") + # This probe has no network, capabilities or secrets. Allow a + # bounded slow containerd teardown; this is not an egress lease. + removed = subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=60) + if removed.returncode: + remains = subprocess.run(["docker", "container", "inspect", name], capture_output=True, text=True, timeout=15) + if remains.returncode == 0 or "No such" not in remains.stderr: + raise RuntimeError("Gateway preflight container cleanup failed") + self._gateway_image = image["Id"] + + def _declared_services(self): + paths = list(self.extra_docker_compose_paths) + if self._environment_docker_compose_path.exists(): + paths.insert(0, self._environment_docker_compose_path) + return declared_services([yaml.safe_load(path.read_text()) for path in paths]) + + @property + def _docker_compose_paths(self): + paths = super()._docker_compose_paths + return [*paths, self._gateway_overlay] if self._gateway_overlay else paths + + async def _ensure_egress_control_sidecar_image_built(self): + # Immutable ID is resolved from the validated image; never rebuild + # Harbor's original, unpatched sidecar or fall back to it. + self._env_vars.egress_control_sidecar_image_name = self._gateway_image + + def _prepare_private_overlay(self): + names = self._declared_services() + settings = self.egress.private_settings() + self._private = PrivateDirectory(self._instance, _sanitize_docker_compose_project_name(self.session_id), self._gateway_image, + daemon_id=self._daemon_id) + root = Path(self._private.name) + root.chmod(0o700) + config = root / "input.json" + with os.fdopen(os.open(config, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "w") as stream: + json.dump(settings, stream) + services = {name: {"cap_drop": ["NET_RAW", "NET_ADMIN"], + "security_opt": ["no-new-privileges:true"], + "labels": {"searchswe.egress.instance": self._instance}} for name in names} + services[SERVICE] = { + "cap_drop": ["ALL"], "cap_add": ["NET_ADMIN", "NET_RAW", "DAC_OVERRIDE"], + "security_opt": ["no-new-privileges:true"], + "read_only": True, + "labels": {"searchswe.egress.instance": self._instance}, + # Late Docker creates may bootstrap only a closed namespace. The + # host applies Harbor's real baseline before starting task services. + "environment": {"SEARCHSWE_EGRESS_INSTANCE": self._instance, + "EGRESS_CONTROL_INITIAL_NETWORK_MODE": "no-network", + "EGRESS_CONTROL_INITIAL_ALLOWED_HOSTS": ""}, + "tmpfs": ["/run/searchswe:mode=0700", "/tmp:mode=1777"], + "sysctls": {"net.ipv6.conf.all.disable_ipv6": "1", "net.ipv6.conf.default.disable_ipv6": "1"}, + "volumes": [{"type": "bind", "source": str(config), "target": "/opt/searchswe/input.json", "read_only": True}], + } + self._gateway_overlay = root / "compose.json" + self._gateway_overlay.write_text(json.dumps({"services": services, "networks": { + "default": {"labels": {"searchswe.egress.instance": self._instance}}}})) + self._gateway_overlay.chmod(0o600) + + async def _run_docker_compose_command(self, command, *args, **kwargs): + if command and command[0] in {"up", "build"}: + rendered = await super()._run_docker_compose_command(["config", "--format", "json"], timeout_sec=30) + document = json.loads(rendered.stdout) + if any(v.get("labels", {}).get("searchswe.egress.instance") != self._instance for v in document.get("volumes", {}).values()): + overlay = json.loads(self._gateway_overlay.read_text()) + overlay["volumes"] = {name: {"labels": {"searchswe.egress.instance": self._instance}} + for name in document["volumes"]} + self._gateway_overlay.write_text(json.dumps(overlay)) + rendered = await super()._run_docker_compose_command(["config", "--format", "json"], timeout_sec=30) + protected = [self.trial_paths.trial_dir / "egress"] + if self.egress.path is not None: + protected.append(self.egress.path) + if self.egress.auth_path is not None: + protected.append(self.egress.auth_path) + document = json.loads(rendered.stdout) + validate_final(document, private_directory=self._private.name, + image=self._gateway_image, engine_paths=self._engine_paths, protected_paths=protected) + await asyncio.to_thread(self._check_resource_ownership, document) + self._rendered_document = document + if command[0] == "up" and self._heartbeat is None: + self._heartbeat = asyncio.create_task(self._keep_lease()) + if command[0] == "up": + self._compose_started = True + # Docker create may be slow, but cannot be allowed to mint a + # fresh permissive lease after cancellation. Bootstrap ONLY the + # trusted service in no-network; authorize with a deadline once + # its controller is alive, before any task entrypoint can run. + await super()._run_docker_compose_command( + ["up", "--detach", "--wait", "--no-deps", SERVICE], *args, **kwargs) + await self.set_network_policy(self.network_policy) + return await super()._run_docker_compose_command(command, *args, **kwargs) + + def _check_resource_ownership(self, document): + """Compose must never adopt/delete an existing foreign named resource.""" + if docker_json("info", "--format", "{{json .ID}}") != self._daemon_id: + raise ValueError("Docker daemon identity changed; preserve the instance for recovery") + project = document["name"] + def check_labels(labels): + if (labels.get("searchswe.egress.instance") != self._instance + or labels.get("com.docker.compose.project") != project): + raise ValueError("Existing Compose resource is not owned by this egress instance") + for kind, entries in (("volume", document.get("volumes", {})), ("network", document.get("networks", {}))): + for resource in entries.values(): + name = resource["name"] + inspected = subprocess.run(["docker", kind, "inspect", name], capture_output=True, text=True, timeout=15) + if inspected.returncode: + error = inspected.stderr.lower() + if (f"no such {kind}:" in error or f"{kind} {name} not found" in error + or f"get {name}: no such volume" in error): + continue + raise RuntimeError("Cannot verify Compose resource ownership") + info = json.loads(inspected.stdout)[0] + check_labels(info.get("Labels") or {}) + if kind == "network": + for identity in info.get("Containers", {}): + check_labels(docker_json("container", "inspect", identity)[0]["Config"].get("Labels") or {}) + containers = subprocess.run(["docker", "ps", "-aq", "--filter", f"label=com.docker.compose.project={project}"], + capture_output=True, text=True, check=True, timeout=15).stdout.split() + for identity in containers: + check_labels(docker_json("container", "inspect", identity)[0]["Config"].get("Labels") or {}) + + @staticmethod + def _deadline(): + return time.monotonic_ns() + LEASE_SECONDS * 10**9 + + async def _require_daemon(self): + identity = await asyncio.to_thread(docker_json, "info", "--format", "{{json .ID}}", timeout=8) + if identity != self._daemon_id: + raise RuntimeError("Docker daemon identity changed; original instance requires recovery") + + async def _renew_lease(self): + await self._require_daemon() + result = await self._run_docker_compose_command( + ["exec", "--no-TTY", SERVICE, "network-policy", "--deadline-ns", str(self._deadline()), "lease"], + timeout_sec=8) + if not json.loads(result.stdout).get("ready"): + raise RuntimeError("Gateway lease was not acknowledged") + + async def _keep_lease(self): + while True: + try: + async with self._policy_lock: + await self._renew_lease() + except asyncio.CancelledError: + raise + except Exception: + if not self._starting: + self._heartbeat_error = RuntimeError("Gateway control lease failed; environment must stop") + return + # Only during startup, before Docker has created the service. + await asyncio.sleep(1 if self._starting else 5) + + async def _stop_heartbeat(self): + if self._heartbeat is not None: + self._heartbeat.cancel() + try: + await self._heartbeat + except asyncio.CancelledError: + pass + self._heartbeat = None + + @staticmethod + async def _collect_buffered_output(process, **kwargs): + try: + return await DockerEnvironment._collect_buffered_output(process, **kwargs) + except asyncio.CancelledError: + await DockerEnvironment._terminate_process(process) + raise + + async def _force_closed(self): + await self._stop_heartbeat() + self._heartbeat_error = RuntimeError("Gateway control failed or stopped; create a new environment") + try: + await self._require_daemon() + except Exception: + # Never send a deny/kill to another daemon's similarly named + # Compose service. Fence the original through its bounded lease. + await asyncio.sleep(LEASE_SECONDS + 1) + raise RuntimeError("Original Docker daemon unavailable; lease expired, cleanup unverified") from None + try: + await self._run_docker_compose_command( + ["exec", "--no-TTY", SERVICE, "network-policy", "deny-all"], timeout_sec=75) + except Exception: + pass + # Even a successful deny-all cannot fence a delayed Docker exec from a + # cancelled setter: terminating the CLI does not terminate its remote + # exec. Kill the container (including every pending exec) before return. + # Failure is terminal; a later trial must create a new environment. + try: + await self._run_docker_compose_command(["kill", SERVICE], timeout_sec=30) + except Exception: + # No further renewals. Already-issued requests retain their original + # deadlines and the kernel expires even with a paused controller. + await asyncio.sleep(LEASE_SECONDS + 1) + raise RuntimeError("Docker control unavailable; lease expired, cleanup remains unverified") from None + + async def set_network_policy(self, policy): + async with self._policy_lock: + try: + if policy == self.network_policy: + # Reapplying the same effective policy must also recover a + # crashed/closed worker rather than trust a cached value. + self.validate_network_policy_support(policy) + await self._apply_network_policy(policy) + else: + await super().set_network_policy(policy) + except BaseException: + cleanup = asyncio.create_task(self._force_closed()) + # Cancellation is not permission to leave an in-flight policy + # replacement running after the environment reports failure. + while not cleanup.done(): + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError: + continue + cleanup.result() + raise + + async def _apply_network_policy(self, policy): + if self._heartbeat_error is not None: + raise self._heartbeat_error + await self._require_daemon() + self._check_policy(policy) + arguments = ({NetworkMode.NO_NETWORK: ["deny-all"], NetworkMode.PUBLIC: ["allow-all"]} + .get(policy.network_mode, ["allow", *policy.allowed_hosts])) + response = await self._run_docker_compose_command( + ["exec", "--no-TTY", SERVICE, "network-policy", "--deadline-ns", str(self._deadline()), *arguments], timeout_sec=75) + state = json.loads(response.stdout) + if (not state.get("ready") or state.get("mode") != policy.network_mode.value + or state.get("hosts") != exact_hosts(policy.allowed_hosts)): + raise RuntimeError("Gateway did not acknowledge the requested phase policy") + + async def start(self, force_build): + preflight = asyncio.create_task(asyncio.to_thread(self._preflight_gateway)) + try: + await asyncio.shield(preflight) + self._prepare_private_overlay() + await super().start(force_build) + await self._renew_lease() + self._starting = False + except BaseException: + # A cancelled to_thread must not create a late probe/lock after the + # caller believes cleanup is finished. Drain it, then release. + while not preflight.done(): + try: + await asyncio.shield(preflight) + except asyncio.CancelledError: + continue + except Exception: + break + try: + preflight.result() + except Exception: + pass + try: + cleanup = asyncio.create_task(self.stop(delete=True)) + while not cleanup.done(): + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError: + continue + cleanup.result() + finally: + self._release_owner() + raise + + def _release_owner(self): + if self._owner_lock is not None: + self._owner_lock.close() + self._owner_lock = None + + async def stop(self, delete): + cleanup = asyncio.create_task(self._stop_owned(delete)) + cancelled = False + try: + while not cleanup.done(): + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError: + cancelled = True + cleanup.result() + if cancelled: + raise asyncio.CancelledError + finally: + self._release_owner() + + async def _stop_owned(self, delete): + if self._private is None: + self._release_owner() + return + if not self._compose_started: + # Validation/build failed before any Compose up. In particular, + # never run down against a foreign named-volume collision. + self._cleanup_mounts_compose_file() + self._cleanup_resources_compose_file() + self._cleanup_env_compose_file() + self._cleanup_egress_control_services_compose_file() + self._gateway_overlay = None + self._private.cleanup() + self._private = None + return + await self._stop_heartbeat() + self._heartbeat_error = RuntimeError("Environment teardown requested; create a new environment") + await asyncio.to_thread(self._check_resource_ownership, self._rendered_document) + try: + await self._force_closed() + except Exception: + pass # down below must succeed, not merely log a cleanup failure. + try: + await self.prepare_logs_for_host() + except Exception: + pass + # Host-owned audit output is outside every task mount. It contains only + # controller events/hash/packet counts, never proxy/DNS payloads. + audit_dir = self.trial_paths.trial_dir / "egress" + audit_error = False + try: + audit_dir.mkdir(mode=0o700, parents=True, exist_ok=True) + logs = await self._run_docker_compose_command(["logs", "--no-color", SERVICE], timeout_sec=15) + (audit_dir / f"{self._instance}.log").write_text(logs.stdout) + except Exception: + audit_error = True + command = ["down", "--remove-orphans"] + if delete: + command.append("--volumes") + try: + await self._run_docker_compose_command(command, timeout_sec=60) + except BaseException: + # Preserve credentials/manifest in the owner-only directory for + # verified recovery; never pretend that Docker down succeeded. + self._release_owner() + raise + self._cleanup_mounts_compose_file() + self._cleanup_resources_compose_file() + self._cleanup_env_compose_file() + self._cleanup_egress_control_services_compose_file() + self._gateway_overlay = None + # Harbor's delete=False retains named volumes. Keep their recoverable + # owner manifest too; otherwise the next start cannot safely adopt them. + if delete or not self._rendered_document.get("volumes"): + self._private.cleanup() + else: + (Path(self._private.name) / "input.json").unlink(missing_ok=True) + self._private = None + self._compose_started = False + self._release_owner() + if audit_error: + raise RuntimeError("Gateway removed, but policy audit collection failed") diff --git a/scripts/run_task.py b/scripts/run_task.py index f9e8f6a..c8de82d 100755 --- a/scripts/run_task.py +++ b/scripts/run_task.py @@ -6,17 +6,16 @@ from pathlib import Path import shlex import shutil -import subprocess import sys import tomllib from urllib.parse import urlsplit if __package__: - from .docker_dns import create_overlay, parse_servers + from .docker_dns import parse_servers from .task_paths import select_task, task_key from .download_assets import destination_path, read_manifest, relative_path else: - from docker_dns import create_overlay, parse_servers + from docker_dns import parse_servers from task_paths import select_task, task_key from download_assets import destination_path, read_manifest, relative_path @@ -93,29 +92,6 @@ def host_is_allowed(host, allowed_hosts): ) -def prepare_network_probe(): - """Prepare Harbor's probe image outside its short kernel-test deadline.""" - from harbor.environments.docker.docker import DockerEnvironment - - image = DockerEnvironment._EGRESS_CONTROL_KERNEL_PROBE_IMAGE - inspected = subprocess.run( - ["docker", "image", "inspect", image], capture_output=True, timeout=30 - ) - if inspected.returncode: - print("Preparing Harbor network-isolation probe image...", flush=True) - subprocess.run(["docker", "pull", image], check=True, timeout=300) - result = subprocess.run( - ["docker", "run", "--rm", image, "sh", "-c", - DockerEnvironment._EGRESS_CONTROL_KERNEL_PROBE_SCRIPT], - capture_output=True, text=True, timeout=30, - ) - if result.returncode: - raise RuntimeError( - "Docker network-isolation probe failed: " - + (result.stderr.strip() or "kernel lacks CONFIG_NFT_FIB_INET") - ) - - def main(): parser = argparse.ArgumentParser(description=__doc__) selection = parser.add_mutually_exclusive_group(required=True) @@ -131,6 +107,8 @@ def main(): parser.add_argument("--codex-config", type=Path, help="Optional native Codex TOML configuration") parser.add_argument("--thinking", choices=PI_THINKING_LEVELS, help="Pi thinking level; overrides PI_THINKING") parser.add_argument("--container-dns", help="Comma-separated upstream IPv4 DNS servers; overrides CONTAINER_DNS. Keeps Harbor's API allowlist.") + parser.add_argument("--egress-config", type=Path, help="Direct/proxy gateway JSON; overrides EGRESS_CONFIG. Without it, restricted tasks use the direct gateway.") + parser.add_argument("--egress-image", help="Direct gateway image (pulled if missing); overrides EGRESS_IMAGE. Cannot be combined with EGRESS_CONFIG.") parser.add_argument("--output", type=Path, help="Job output directory; relative to the current directory") parser.add_argument("--dry-run", action="store_true", help="Print the command with variable references; do not launch") args = parser.parse_args() @@ -147,6 +125,7 @@ def main(): elif args.env_file is not None: parser.error(f"Environment file does not exist: {env_file}") env = {**file_env, **os.environ} + egress_path = args.egress_config or (REPO / env["EGRESS_CONFIG"] if env.get("EGRESS_CONFIG") else None) dns_servers = None if dns_value := (args.container_dns or env.get("CONTAINER_DNS")): try: @@ -257,7 +236,9 @@ def main(): verifier_network_mode, verifier_allowed_hosts = effective_network_policy( task_config, "verifier" ) - if agent_network_mode != "public": + agent_requires_host = any(step.get("agent", {}).get("network_mode", agent_network_mode) != "public" + for step in task_config.get("steps") or [{}]) + if agent_requires_host: if agent_host is None: parser.error( "Set AGENT_OPENAI_BASE_URL so Harbor can allow only the selected " @@ -299,8 +280,26 @@ def main(): "[verifier].allowed_hosts" ) + if __package__: + from .egress.config import DEFAULT_IMAGE, direct_config, load_config, validate_task_networks + else: + from egress.config import DEFAULT_IMAGE, direct_config, load_config, validate_task_networks + egress_image = args.egress_image or env.get("EGRESS_IMAGE") + try: + if egress_path is not None: + if dns_servers or env.get("CONTAINER_PROXY") or egress_image: + parser.error("EGRESS_CONFIG cannot be combined with CONTAINER_DNS/--container-dns, CONTAINER_PROXY, or EGRESS_IMAGE/--egress-image; clear them explicitly") + egress = load_config(egress_path) + else: + egress = direct_config(egress_image or DEFAULT_IMAGE, dns_servers) + restricted = validate_task_networks(task, agent_host, egress.settings.get("upstream_host"), + allow_public=egress.settings.get("transport") == "direct") + except (OSError, ValueError) as error: + parser.error(str(error)) + use_gateway = restricted or egress_path is not None + if env.get("CONTAINER_PROXY"): - if agent_network_mode != "public" or verifier_network_mode != "public": + if restricted: parser.error( "CONTAINER_PROXY is incompatible with this task's Harbor network " "allowlists because a general proxy can bypass destination filtering" @@ -312,6 +311,15 @@ def main(): for name in ("no_proxy", "NO_PROXY"): command.extend([flag, f"{name}=${{CONTAINER_NO_PROXY}}"]) + if use_gateway: + command[command.index("--env") + 1] = "scripts.harbor_environments:PhaseScopedDocker" + if egress_path is not None: + command.extend(["--environment-kwarg", f"egress_config={egress.path}"]) + else: + command.extend(["--environment-kwarg", f"egress_image={egress.image}"]) + if dns_servers: + command.extend(["--environment-kwarg", "egress_dns=" + ",".join(dns_servers)]) + # This launcher uses explicit API keys, without consulting a host auth.json. env.pop("CODEX_AUTH_JSON_PATH", None) env.pop("CODEX_FORCE_AUTH_JSON", None) @@ -321,10 +329,13 @@ def main(): if args.dry_run: print("Preview only; credentials, assets, Docker, GPU, and API access are not checked.") + if use_gateway: + mode = egress.settings.get("transport", "proxy") + print(f"Phase-scoped {mode} gateway; task and phase allowlists are enforced by the trusted sidecar.") + if dns_servers: + print("Trusted gateway DNS servers: " + ", ".join(dns_servers)) print("Environment values are passed to Harbor separately from the command:") print(shlex.join(command)) - if dns_servers: - print("Docker sidecar DNS override (allowlist preserved): " + ", ".join(dns_servers)) return 0 missing = [name for name in required if not env.get(name)] @@ -341,18 +352,8 @@ def main(): parser.error(f"Run python scripts/download_assets.py {selection_flag} to restore the missing or incomplete assets: " + ", ".join(unavailable)) if shutil.which("harbor", path=env.get("PATH")) is None: parser.error("harbor was not found; activate the supported Harbor environment") - if any(effective_network_policy(task_config, phase)[0] != "public" - for phase in ("environment", "agent", "verifier")): - try: - prepare_network_probe() - except (OSError, subprocess.SubprocessError, RuntimeError) as error: - parser.error(f"Docker network preflight failed (allowlist remains enforced): {error}") - if dns_servers: - if all(effective_network_policy(task_config, phase)[0] == "public" - for phase in ("environment", "agent", "verifier")): - parser.error("CONTAINER_DNS currently requires Harbor's network-isolation sidecar") - command.extend(["--extra-docker-compose", str(create_overlay(dns_servers))]) - print("Docker DNS upstreams (API allowlist unchanged): " + ", ".join(dns_servers), flush=True) + if dns_servers and not use_gateway: + parser.error("CONTAINER_DNS requires a restricted task or an explicit direct EGRESS_CONFIG") # Harbor is a console script: changing cwd alone does not put this checkout # on its interpreter's search path for scripts.harbor_agents. diff --git a/scripts/tests/egress_adapter_smoke.py b/scripts/tests/egress_adapter_smoke.py new file mode 100644 index 0000000..e4977d4 --- /dev/null +++ b/scripts/tests/egress_adapter_smoke.py @@ -0,0 +1,278 @@ +"""Real Docker adapter/factory smoke, without model calls or task downloads. + +Uses no-network except listener-readiness-only phase changes. No API/DNS probes +are sent, and the configured upstream is a reserved TEST-NET address. +""" + +import argparse +import asyncio +import json +import logging +from pathlib import Path +import shlex +import subprocess +import sys +from types import MethodType, SimpleNamespace +import uuid + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from harbor.environments.factory import EnvironmentFactory +from harbor.models.task.config import EnvironmentConfig as TaskEnvironment, NetworkPolicy, TaskConfig +from harbor.models.task.verifier_mode import resolve_effective_verifier_env_config, resolve_task_verifier_mode +from harbor.models.trial.config import AgentConfig, EnvironmentConfig +from harbor.models.trial.paths import TrialPaths +from harbor.trial.network_policy import resolve_trial_network_plan +from harbor.trial.trial import Trial +from scripts.tests.egress_credentials import CredentialProbe + + +async def cancel_policy(environment, phase, *, queued): + """Exercise real remote execs, including CLI cancellation before apply.""" + service = "harbor-docker-egress-control-sidecar" + original = environment._run_docker_compose_command + applied = asyncio.Event() + baseline = environment.network_policy + async def delayed_response(command, *arguments, **keywords): + if "network-policy" in command and "allow" in command: + if queued: + payload = "touch /run/searchswe/cancel-probe; sleep 10; exec " + shlex.join(command[3:]) + return await original(command[:3] + ["sh", "-c", payload], *arguments, **keywords) + response = await original(command, *arguments, **keywords) + applied.set() + await asyncio.Event().wait() + return response + return await original(command, *arguments, **keywords) + environment._run_docker_compose_command = delayed_response + setter = asyncio.create_task(environment.set_network_policy(phase)) + try: + if queued: + for _ in range(20): + check = await environment.service_exec("test -f /run/searchswe/cancel-probe", service=service) + if check.return_code == 0: + break + await asyncio.sleep(.1) + else: + raise AssertionError("delayed remote exec did not start") + else: + await asyncio.wait_for(applied.wait(), timeout=20) + finally: + setter.cancel() + try: + await setter + except asyncio.CancelledError: + pass + environment._run_docker_compose_command = original + state = await original(["ps", "--all", "--format", "json", service]) + rows = [json.loads(line) for line in state.stdout.splitlines() if line.strip()] + if not rows or any(row["State"] not in {"exited", "dead"} for row in rows): + raise AssertionError("cancelled setter left the gateway or queued exec running") + if environment.network_policy != baseline: + raise AssertionError("cancelled setter updated the cached policy") + try: + await environment.set_network_policy(phase) + except RuntimeError: + pass + else: + raise AssertionError("a terminal control failure was silently recovered") + + +async def run(args): + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=False) + session = "searchswe-adapter-" + uuid.uuid4().hex[:12] + environment_dir = output / "environment" + environment_dir.mkdir() + (environment_dir / "Dockerfile").write_text(f'FROM python:3.13-slim\nLABEL searchswe.fixture="{session}"\n') + (environment_dir / "docker-compose.yaml").write_text(json.dumps({"volumes": {"state": {}}, "services": { + "helper": {"image": args.python_image, "command": ["sleep", "infinity"], "volumes": ["state:/state"]}}})) + config = output / "egress.json" + credentials = CredentialProbe(args.auth_file) if args.auth_file or args.credential_probe else None + upstream = {"url": "http://192.0.2.1:8080"} + if credentials: + upstream["auth_file"] = str(credentials.path) + config.write_text(json.dumps({"version": 1, "image": args.gateway_image, + "upstream": upstream, + "dns": {"doh_url": "https://resolver.example/dns-query"}})) + agent_overlay = output / "agent-overlay.json" + agent_overlay.write_text(json.dumps({"services": {"main": {"environment": {"AGENT_ONLY_FIXTURE": "present"}}}})) + runtime_config = EnvironmentConfig(import_path="scripts.harbor_environments:PhaseScopedDocker", + kwargs={"egress_config": str(config)}, extra_docker_compose=[agent_overlay]) + if credentials: + credentials.scan(runtime_config.model_dump_json(), "Harbor-environment-job-config") + baseline = NetworkPolicy(network_mode="no-network") + phase = NetworkPolicy(network_mode="allowlist", allowed_hosts=["allowed.example"]) + environment = EnvironmentFactory.create_environment_from_config( + config=runtime_config, + environment_dir=environment_dir, environment_name=session, session_id=session, + trial_paths=TrialPaths(trial_dir=output / "trial"), + task_env_config=TaskEnvironment(docker_image=args.python_image), + network_policy=baseline, phase_network_policies=[phase]) + result = {"status": "infrastructure_error", "session": session, "checks": []} + (output / "driver-identity.json").write_text(json.dumps({"session": session, "instance": environment._instance})) + private = None + try: + if not args.orphan_child: + foreign_volume = session + "_state" + subprocess.run(["docker", "volume", "create", "--label", f"searchswe.fixture={session}", foreign_volume], + check=True, capture_output=True, timeout=30) + try: + await environment.start(force_build=False) + except ValueError as error: + if "not owned" not in str(error): + raise + else: + raise AssertionError("foreign named volume was adopted") + info = json.loads(subprocess.run(["docker", "volume", "inspect", foreign_volume], + check=True, capture_output=True, text=True, timeout=30).stdout)[0] + if info.get("Labels", {}).get("searchswe.fixture") != session: + raise AssertionError("foreign-volume positive control lost ownership") + subprocess.run(["docker", "volume", "rm", foreign_volume], check=True, capture_output=True, timeout=30) + result["checks"].append({"case": "foreign-volume-rejected-and-not-deleted", "passed": True}) + await environment.start(force_build=args.force_build) + private = environment._private.name + if args.orphan_child: + await environment.set_network_policy(phase) + if credentials: + result["checks"].append(await credentials.environment(environment, worker=True)) + (output / "owner-ready.json").write_text(json.dumps({"private": private, "instance": environment._instance, + "session": session, + "credential_boundary_checked": bool(credentials)})) + # The dedicated parent test SIGKILLs this process. If the parent + # disappears instead, normal finally cleanup runs after this bound. + await asyncio.sleep(180) + raise RuntimeError("orphan fixture was not interrupted") + rendered = await environment._run_docker_compose_command(["config", "--format", "json"]) + (output / "compose.json").write_text(rendered.stdout) + script = """import json,os,socket +from pathlib import Path +status=dict(line.split(':',1) for line in Path('/proc/self/status').read_text().splitlines() if ':' in line) +assert int(status['CapBnd'].strip(),16)&((1<<12)|(1<<13))==0 +assert not Path('/opt/searchswe/input.json').exists() +assert not Path('/run/searchswe/control.sock').exists() +sock=socket.socket() +try: + sock.setsockopt(socket.SOL_SOCKET,36,114514) +except PermissionError: + print(json.dumps({'uid':os.getuid(),'mark_denied':True,'agent_fixture':os.environ.get('AGENT_ONLY_FIXTURE')})) +else: + raise AssertionError('mark unexpectedly permitted') +""" + for service in ("main", "helper"): + response = await environment.service_exec("python -c " + shlex.quote(script), service=service) + if response.return_code: + raise RuntimeError(f"capability probe failed: {service}") + if service == "main" and json.loads(response.stdout)["agent_fixture"] != "present": + raise RuntimeError("agent-only overlay positive control failed") + result["checks"].append({"service": service, "result": json.loads(response.stdout)}) + try: + async with Trial._phase_network_policy(None, environment, baseline_policy=baseline, phase_policy=phase): + if environment.network_policy != phase: + raise AssertionError("phase not applied") + if credentials: + result["checks"].append(await credentials.environment(environment, worker=True)) + raise RuntimeError("intentional fixture exception") + except RuntimeError as error: + if str(error) != "intentional fixture exception": + raise + if environment.network_policy != baseline: + raise AssertionError("baseline not restored") + result["checks"].append({"case": "real-phase-exception-restores-baseline", "passed": True}) + repo = Path(__file__).resolve().parents[2] + task_config = TaskConfig.model_validate_toml((repo / "tasks/task-1-1/task.toml").read_text()) + plan = resolve_trial_network_plan(task_config, AgentConfig(extra_allowed_hosts=["model.example"]), + runtime_config, None, verifier_mode=resolve_task_verifier_mode(task_config), + env_config=resolve_effective_verifier_env_config(task_config, None)) + trial = SimpleNamespace(config=SimpleNamespace(environment=runtime_config), + task=SimpleNamespace(short_name=session), paths=environment.trial_paths, _id=session, + logger=logging.getLogger(session), _environment_build_timeout_sec=180, + _verifier_env_build_context=lambda step: environment_dir, + _separate_verifier_session_id=lambda key: session + "-verifier-" + key, + _verifier_env_mounts=lambda config: []) + trial._validate_dynamic_phase_switch = MethodType(Trial._validate_dynamic_phase_switch, trial) + trial._validate_separate_verifier_env_policies = MethodType(Trial._validate_separate_verifier_env_policies, trial) + private_paths = {private} + for key in ("first", "second"): + async with Trial._separate_verifier_env(trial, TaskEnvironment(docker_image=args.python_image), + key=key, plan=plan) as verifier: + if type(verifier) is not type(environment) or verifier.extra_docker_compose_paths: + raise AssertionError("independent verifier adapter/config was not preserved") + if verifier._private.name in private_paths: + raise AssertionError("private gateway instance was reused") + private_paths.add(verifier._private.name) + response = await verifier.exec("test -z \"$AGENT_ONLY_FIXTURE\"") + if response.return_code: + raise AssertionError("agent-only overlay leaked into verifier") + async with Trial._phase_network_policy(None, verifier, baseline_policy=plan.verifier_env_baseline, + phase_policy=plan.verifier_phase): + if "model.example" in verifier.network_policy.allowed_hosts: + raise AssertionError("agent model host leaked into verifier") + response = await verifier.exec("python -c " + shlex.quote(script)) + if response.return_code: + raise AssertionError("verifier capability/private mount probe failed") + if credentials: + result["checks"].append(await credentials.environment(verifier, worker=True)) + if key == "second": + await cancel_policy(verifier, phase, queued=True) + result["checks"].append({"case": "cancel-before-queued-apply-kills-remote-exec", "passed": True}) + if verifier._private is not None: + raise AssertionError("independent verifier cleanup did not complete") + result["checks"].append({"case": "independent-verifier-" + key, "passed": True}) + await asyncio.gather(environment.set_network_policy(phase), environment.set_network_policy(baseline)) + state = await environment._run_docker_compose_command( + ["exec", "--no-TTY", "harbor-docker-egress-control-sidecar", "network-policy", "show"]) + if environment.network_policy != baseline or json.loads(state.stdout).get("mode") != "no-network": + raise AssertionError("concurrent setters were not serialized") + result["checks"].append({"case": "concurrent-setters-serialized", "passed": True}) + + await cancel_policy(environment, phase, queued=False) + result["checks"].append({"case": "cancel-after-apply-kills-before-return", "passed": True}) + result["status"] = "passed" + except Exception as error: + result["error"] = str(error) + finally: + try: + await environment.stop(delete=True) + result["private_directory_removed"] = private is None or not Path(private).exists() + if args.force_build: + # Harbor image IDs are content-based, not session-based. The + # unique fixture label above makes this tag exclusively ours. + inspected = subprocess.run(["docker", "image", "inspect", environment._main_image_name], + capture_output=True, text=True, timeout=30) + if inspected.returncode == 0: + labels = json.loads(inspected.stdout)[0]["Config"].get("Labels", {}) + if labels.get("searchswe.fixture") != session: + raise RuntimeError("fixture image ownership mismatch") + subprocess.run(["docker", "image", "rm", environment._main_image_name], + capture_output=True, text=True, check=True, timeout=30) + elif "No such image" not in inspected.stderr: + raise RuntimeError("could not verify fixture image cleanup") + result["fixture_image_tag_removed"] = True + except Exception as error: + result["status"] = "cleanup_error" + result["cleanup_error"] = str(error) + if credentials: + try: + credentials.artifacts(output) + credentials.scan(json.dumps(result), "final-result") + result["checks"].append({"case": "credential-artifacts-and-audits-clean", "passed": True}) + except Exception: + result = {"status": "credential_evidence_error", "error": "Credential evidence validation failed; inspect withheld-file notices"} + finally: + credentials.close() + (output / "result.json").write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result)) + return 0 if result["status"] == "passed" else 2 + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--gateway-image", required=True) + parser.add_argument("--python-image", default="python:3.13-slim") + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--force-build", action="store_true", help="Also exercise the launcher's task image build path") + credentials = parser.add_mutually_exclusive_group() + credentials.add_argument("--credential-probe", action="store_true", help="Use synthetic credentials and inspect all service boundaries") + credentials.add_argument("--auth-file", type=Path, help=argparse.SUPPRESS) + parser.add_argument("--orphan-child", action="store_true", help=argparse.SUPPRESS) + raise SystemExit(asyncio.run(run(parser.parse_args()))) diff --git a/scripts/tests/egress_credentials.py b/scripts/tests/egress_credentials.py new file mode 100644 index 0000000..95ae28f --- /dev/null +++ b/scripts/tests/egress_credentials.py @@ -0,0 +1,135 @@ +"""Synthetic credential probes. Never print or persist the canary in evidence.""" + +import asyncio +import base64 +import hashlib +import json +import os +from pathlib import Path +import shlex +import subprocess +import tempfile +from urllib.parse import quote +import uuid + +from scripts.egress.config import read_auth +from scripts.egress.ownership import owner_root + + +class CredentialLeak(RuntimeError): + pass + + +class CredentialProbe: + def __init__(self, auth_file=None): + self.owned = None + if auth_file is None: + self.owned = tempfile.TemporaryDirectory(prefix="credential-probe-", dir=owner_root()) + auth_file = Path(self.owned.name) / "auth.json" + auth = {"username": "swe_user_" + uuid.uuid4().hex, + "password": "swe_password_" + uuid.uuid4().hex + '/+=\\"ü'} + with os.fdopen(os.open(auth_file, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "w") as stream: + json.dump(auth, stream) + self.path = Path(auth_file).absolute() + self.auth = read_auth(self.path) + self.patterns = {base64.b64encode((self.auth["username"] + ":" + self.auth["password"]).encode())} + for value in self.auth.values(): + self.patterns.update(v.encode() for v in (value, quote(value, safe=""), json.dumps(value)[1:-1])) + # A scanner that never matches must not pass the isolation gate. + for value in self.patterns: + try: + self.scan(b"prefix " + value + b" suffix", "scanner-positive-control") + except CredentialLeak: + continue + raise RuntimeError("credential scanner positive control failed") + + def scan(self, data, surface): + data = data.encode() if isinstance(data, str) else data + if any(pattern in data for pattern in self.patterns): + raise CredentialLeak("Synthetic credential exposed on " + surface) + + def artifacts(self, output): + found = [] + for path in sorted(Path(output).rglob("*")): + if path.is_symlink(): + raise RuntimeError("unexpected symlink in credential evidence") + if path.is_file(): + try: + self.scan(path.read_bytes(), "output-artifact") + except CredentialLeak: + # Keep the filename/failure, not the credential-bearing data. + path.write_text("Withheld: synthetic credential canary detected.\n") + found.append(str(path.relative_to(output))) + if found: + raise CredentialLeak("Synthetic credential artifacts withheld: " + ", ".join(found)) + + async def environment(self, environment, *, worker=False): + """Inspect real services without supplying expected secrets to them.""" + private = Path(environment._private.name) + loaded = json.loads((private / "input.json").read_text()).get("auth") + if loaded != self.auth: + raise RuntimeError("private credential positive control failed") + self.scan(repr(environment.egress), "public-config-repr") + rendered = await environment._run_docker_compose_command(["config", "--format", "json"]) + self.scan(rendered.stdout, "merged-compose") + services = json.loads(rendered.stdout)["services"] + service_gateway = "harbor-docker-egress-control-sidecar" + paths = [str(self.path), str(private / "input.json"), + "/opt/searchswe/input.json", "/run/searchswe/gost.json", "/run/searchswe/control.sock"] + script = """import base64,json,os +from pathlib import Path +data=[json.dumps(dict(os.environ)).encode()] +count=0 +for process in Path('/proc').iterdir(): + if not process.name.isdigit(): continue + for field in ('cmdline','environ'): + try: data.append((process/field).read_bytes());count+=1 + except (FileNotFoundError,PermissionError,ProcessLookupError): pass +print(json.dumps({'data':[base64.b64encode(v).decode() for v in data], + 'process_fields':count,'positive_file':Path('/etc/os-release').is_file(), + 'private_visible':any(Path(p).exists() or (Path('/proc/1/root')/p.lstrip('/')).exists() for p in PRIVATE_PATHS)})) +""" + task_services = [] + for service in services: + # The gateway's PRIVATE files legitimately contain credentials; + # its environment and process argv must still not contain them. + checked_paths = [] if service == service_gateway else paths + code = "PRIVATE_PATHS=" + repr(checked_paths) + "\n" + script + response = await environment.service_exec("python3 -c " + shlex.quote(code), service=service) + if response.return_code: + raise RuntimeError("credential process probe failed") + report = json.loads(response.stdout) + if not report["positive_file"] or not report["process_fields"] or report["private_visible"]: + raise RuntimeError("private-files/process-visibility probe failed") + for blob in report["data"]: + self.scan(base64.b64decode(blob), "service-environment-or-argv") + if service != service_gateway: + task_services.append(service) + ids = await asyncio.to_thread(subprocess.run, + ["docker", "ps", "-aq", "--filter", f"label=searchswe.egress.instance={environment._instance}"], + capture_output=True, text=True, check=True, timeout=30) + if not ids.stdout.split(): + raise RuntimeError("container metadata positive control failed") + metadata = await asyncio.to_thread(subprocess.run, ["docker", "inspect", *ids.stdout.split()], + capture_output=True, text=True, check=True, timeout=30) + self.scan(metadata.stdout, "container-metadata") + if worker: + code = """import hashlib,json +from pathlib import Path +config=json.loads(Path('/run/searchswe/gost.json').read_text()) +auth=config['chains'][0]['hops'][0]['nodes'][0]['connector']['auth'] +print(hashlib.sha256(json.dumps(auth,sort_keys=True).encode()).hexdigest()) +""" + loaded = await environment.service_exec("python3 -c " + shlex.quote(code), service=service_gateway) + expected = hashlib.sha256(json.dumps(self.auth, sort_keys=True).encode()).hexdigest() + if loaded.return_code or loaded.stdout.strip() != expected: + raise RuntimeError("live worker credential positive control failed") + logs = await environment._run_docker_compose_command(["logs", "--no-color"]) + self.scan((logs.stdout or "") + (logs.stderr or ""), "container-logs") + return {"case": "synthetic-credential-boundary", "passed": True, + "instance": environment._instance, "task_services": sorted(task_services), + "worker_config_checked": worker} + + def close(self): + if self.owned is not None: + self.owned.cleanup() diff --git a/scripts/tests/egress_direct_transport.py b/scripts/tests/egress_direct_transport.py new file mode 100644 index 0000000..8c88e38 --- /dev/null +++ b/scripts/tests/egress_direct_transport.py @@ -0,0 +1,139 @@ +"""Offline direct HTTPS gate with real certificates and receiving-side evidence.""" + +import argparse +import asyncio +import hashlib +import json +from pathlib import Path +import shlex +import ssl +import subprocess +import sys +import threading +import uuid + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) + +from harbor.models.task.config import EnvironmentConfig, NetworkPolicy +from harbor.models.trial.paths import TrialPaths +from scripts.harbor_environments import PhaseScopedDocker +from scripts.tests.egress_scenarios_fixture import DirectReceiver + + +CLIENT = '''import http.client,json,socket,ssl,sys,time +host,ip,port,sni=sys.argv[1:] +ctx=ssl.create_default_context(cafile='/tmp/origin.crt') +if sni=='none': ctx.check_hostname=False +try: + with socket.create_connection((ip,int(port)),timeout=4) as sock: + with ctx.wrap_socket(sock,server_hostname=None if sni=='none' else host) as tls: + tls.sendall(f'GET /client/tls/{time.monotonic_ns()} HTTP/1.1\\r\\nHost: {host}:{port}\\r\\nConnection: close\\r\\n\\r\\n'.encode()) + response=http.client.HTTPResponse(tls);response.begin() + print(json.dumps({'ok':response.status==200 and response.read()==b'OK','tls':tls.version()})) +except (OSError,http.client.HTTPException) as e: + print(json.dumps({'ok':False,'error':type(e).__name__})) +''' + + +def docker(*args): + return subprocess.run(['docker', *args], capture_output=True, text=True, check=True, timeout=60).stdout + + +async def run(args): + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=False) + identity = 'searchswe-direct-tls-' + uuid.uuid4().hex[:12] + result = {'status': 'running', 'identity': identity, 'checks': [], 'cleanup_errors': []} + env, receiver = None, None + try: + result['sources'] = {str(p.relative_to(ROOT)): hashlib.sha256(p.read_bytes()).hexdigest() for p in ( + Path(__file__), ROOT / 'scripts/tests/egress_scenarios_fixture.py', + ROOT / 'scripts/harbor_environments.py', ROOT / 'environments/egress/gateway.py')} + result['images'] = {role: json.loads(docker('image', 'inspect', name))[0]['Id'] + for role, name in [('gateway', args.gateway_image), ('python', args.python_image)]} + subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1', + '-subj', '/CN=allowed.example', '-addext', + 'subjectAltName=DNS:allowed.example,DNS:blocked.example', + '-keyout', str(output / 'origin.key'), '-out', str(output / 'origin.crt')], + check=True, capture_output=True, timeout=30) + (output / 'origin.key').chmod(0o600) + bridge = json.loads(docker('network', 'inspect', 'bridge'))[0]['IPAM']['Config'][0]['Gateway'] + receiver = DirectReceiver((bridge, 0), output / 'receiver.jsonl') + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(output / 'origin.crt', output / 'origin.key') + receiver.socket = context.wrap_socket(receiver.socket, server_side=True) + threading.Thread(target=receiver.serve_forever, daemon=True).start() + config = output / 'egress.json' + config.write_text(json.dumps(receiver.configuration(args.gateway_image))) + directory = output / 'environment' + directory.mkdir() + (directory / 'Dockerfile').write_text('FROM ' + args.python_image + '\n') + both = NetworkPolicy(network_mode='allowlist', allowed_hosts=['allowed.example', 'blocked.example']) + env = PhaseScopedDocker(environment_dir=directory, environment_name=identity, session_id=identity, + trial_paths=TrialPaths(trial_dir=output / 'trial'), + task_env_config=EnvironmentConfig(docker_image=args.python_image), + network_policy=both, egress_image=args.gateway_image, + egress_dns=receiver.configuration(args.gateway_image)['dns']['servers'][0]) + await env.start(force_build=False) + client = output / 'client.py' + client.write_text(CLIENT) + await env.upload_file(client, '/tmp/client.py') + await env.upload_file(output / 'origin.crt', '/tmp/origin.crt') + + async def probe(name, host, expected, *, destination='198.51.100.42', sni='hostname'): + before = len(receiver.snapshot()) + response = await env.exec('python /tmp/client.py ' + shlex.join( + [host, destination, str(receiver.server_address[1]), sni])) + if response.return_code: + raise RuntimeError('TLS client failed to report a result') + client_result = json.loads(response.stdout) + events = receiver.snapshot()[before:] + reached = any(e.get('kind') == 'http' for e in events) + if client_result['ok'] != expected or reached != expected: + raise AssertionError('TLS authorization mismatch: ' + name) + if not expected and events: + raise AssertionError('rejected TLS hostname reached DNS or origin: ' + name) + result['checks'].append({'case': name, 'passed': True, 'client': client_result, 'events': events}) + + await probe('allowed-certificate-positive', 'allowed.example', True) + await probe('forbidden-target-positive-control', 'blocked.example', True) + await env.set_network_policy(NetworkPolicy(network_mode='allowlist', allowed_hosts=['allowed.example'])) + await probe('allowed-tls-after-restriction', 'allowed.example', True) + await probe('blocked-sni-never-reaches-dns-or-origin', 'blocked.example', False) + await probe('no-sni-rejected', 'allowed.example', False, sni='none') + await env.set_network_policy(NetworkPolicy(network_mode='public')) + await probe('public-tls-direct-positive', 'blocked.example', True, destination=bridge) + await env.set_network_policy(NetworkPolicy(network_mode='no-network')) + await probe('no-network-allowed-host-denied', 'allowed.example', False, destination=bridge) + await probe('no-network-public-host-denied', 'blocked.example', False, destination=bridge) + await env.set_network_policy(both) + await probe('restored-policy-positive', 'allowed.example', True) + result['status'] = 'passed' + except BaseException as error: + result.update(status='failed', error=type(error).__name__ + ': ' + str(error)) + finally: + if env is not None: + try: + await env.stop(delete=True) + for command in (['ps', '-aq'], ['network', 'ls', '-q'], ['volume', 'ls', '-q']): + if docker(*command, '--filter', 'label=searchswe.egress.instance=' + env._instance).strip(): + raise RuntimeError('owned TLS gate resources remain') + except Exception as error: + result['cleanup_errors'].append(str(error)) + if receiver is not None: + receiver.shutdown() + receiver.server_close() + if result['cleanup_errors']: + result['status'] = 'cleanup_error' + (output / 'result.json').write_text(json.dumps(result, indent=2) + '\n') + print(json.dumps({'status': result['status'], 'checks': len(result['checks']), 'error': result.get('error')})) + return 0 if result['status'] == 'passed' else 2 + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--gateway-image', required=True) + parser.add_argument('--python-image', default='python:3.13-slim') + parser.add_argument('--output', required=True, type=Path) + raise SystemExit(asyncio.run(run(parser.parse_args()))) diff --git a/scripts/tests/egress_full_trial.py b/scripts/tests/egress_full_trial.py new file mode 100644 index 0000000..d564f35 --- /dev/null +++ b/scripts/tests/egress_full_trial.py @@ -0,0 +1,238 @@ +"""Full Harbor Trial.run gate: two simultaneous trials of one four-step task. + +Formal adapter/Oracle extension points only. No patched Harbor methods, API calls, +task-1-1 edits, proxy forwarding, or changes to the operator Docker daemon. +""" +import argparse +import asyncio +import hashlib +import json +from pathlib import Path +import shutil +import subprocess +import sys +import threading +import time +import uuid + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) +sys.modules['scripts.tests.egress_full_trial'] = sys.modules[__name__] + +from harbor.models.trial.config import TrialConfig +from harbor.models.task.config import TaskConfig +from harbor.trial.trial import Trial +from scripts.harbor_environments import PhaseScopedDocker +from scripts.tests.egress_scenarios_fixture import DirectReceiver, Receiver + +INSTANCES = [] +BARRIERS = {} + +class ObservedDocker(PhaseScopedDocker): + def __init__(self, *args, **kwargs): + self.observation = {'session': kwargs['session_id'], 'policies': []} + super().__init__(*args, **kwargs) + self.observation['instance'] = self._instance + INSTANCES.append(self) + + async def _apply_network_policy(self, policy): + start = time.monotonic_ns() + await super()._apply_network_policy(policy) + self.observation['policies'].append({'start_ns': start, 'ack_ns': time.monotonic_ns(), + 'policy': policy.model_dump(mode='json')}) + + async def start(self, force_build): + await super().start(force_build) + container = await self._run_docker_compose_command(['ps', '-q', 'harbor-docker-egress-control-sidecar']) + info = json.loads(await asyncio.to_thread(docker, 'inspect', container.stdout.strip()))[0] + addresses = [v['IPAddress'] for v in info['NetworkSettings']['Networks'].values()] + assert len(addresses) == 1 and addresses[0] + self.observation.update(started_ns=time.monotonic_ns(), source=addresses[0], private=self._private.name) + # Both real separate-verifier environments must coexist at every step. + if '__verifier__' in self.observation['session']: + step = self.observation['session'].split('__verifier__')[-1] + await asyncio.wait_for(BARRIERS.setdefault(step, asyncio.Barrier(2)).wait(), timeout=90) + + async def stop(self, delete): + await super().stop(delete) + self.observation.setdefault('stopped_ns', time.monotonic_ns()) + +def docker(*args): + return subprocess.run(['docker', *args], capture_output=True, text=True, check=True, timeout=90).stdout + +def allow(*hosts): + return {"network_mode": "allowlist", "allowed_hosts": list(hosts)} + + +def task_config(name, image): + return TaskConfig.model_validate({ + "environment": {"docker_image": image, **allow(f"base-{name}.example")}, + "agent": allow(f"task-{name}.example"), + "verifier": {**allow(f"judge-{name}.example"), + "environment": {"docker_image": image, **allow(f"vbase-{name}.example")}}, + "steps": [ + {"name": "inherited"}, + {"name": "override", "agent": allow(f"step-{name}.example"), + "verifier": {**allow(f"score-{name}.example"), + "environment": {"docker_image": image, "network_mode": "no-network"}}}, + {"name": "offline", "agent": {"network_mode": "no-network"}, + "verifier": {"network_mode": "no-network"}}, + {"name": "shared", "verifier": {"environment_mode": "shared", **allow(f"shared-{name}.example")}}, + ]}) + + +def create_task(output, image): + directory = output / 'task' + directory.mkdir() + (directory / 'environment').mkdir() + (directory / 'environment/Dockerfile').write_text('FROM ' + image + '\n') + cfg = task_config('alpha', image).model_dump(mode='json', exclude_none=True) + cfg['environment'].update(workdir='/app', cpus=1, memory_mb=512) + cfg['agent']['timeout_sec'] = 60 + cfg['verifier']['timeout_sec'] = 60 + cfg['artifacts'] = ['/app'] + for index, step in enumerate(cfg['steps']): + name = step['name'] + step['verifier']['env'] = {'FIXTURE_STEP_INDEX': str(index)} + step['min_reward'] = 1.0 + root = directory / 'steps' / name + for sub in ('workdir', 'solution', 'tests'): + (root / sub).mkdir(parents=True) + (root / 'instruction.md').write_text('Synthetic fixture: construct and independently score an arithmetic function.\n') + (root / 'workdir/setup.sh').write_text(f'#!/bin/sh\nset -eu\npython /opt/fixture/client.py setup {index}\n') + (root / 'solution/solve.sh').write_text(f'#!/bin/sh\nset -eu\npython /opt/fixture/client.py agent {index}\n') + (root / 'tests/test.sh').write_text(f'#!/bin/sh\nset -eu\npython /opt/fixture/client.py verifier {index}\n') + (root / 'tests/Dockerfile').write_text('FROM ' + image + '\n') + (directory / 'task.toml').write_text(TaskConfig.model_validate(cfg).model_dump_toml()) + return directory + +async def main(args): + output = args.output.resolve();output.mkdir(parents=True, exist_ok=False) + identity = 'searchswe-full-trial-' + uuid.uuid4().hex[:10] + result = {'status': 'running', 'identity': identity, 'checks': [], 'cleanup_errors': [], + 'transport': args.transport, + 'scope': 'synthetic Oracle coding and real independent scoring; no model API calls', + 'release_ready': False} + receiver, futures = None, [] + try: + sources = [Path(__file__), Path(__file__).with_name('egress_full_trial_client.py'), + ROOT / 'scripts/tests/egress_scenarios_fixture.py', + ROOT / 'scripts/harbor_environments.py'] + result['sources'] = {str(p.relative_to(ROOT)): hashlib.sha256(p.read_bytes()).hexdigest() for p in sources} + # Fail before build if either required input is not already local. + inputs = {role: json.loads(docker('image', 'inspect', name))[0]['Id'] + for role, name in [('python', args.python_image), ('gateway', args.gateway_image)]} + bridge = json.loads(docker('network', 'inspect', 'bridge'))[0]['IPAM']['Config'][0]['Gateway'] + receiver_type = DirectReceiver if args.transport == 'direct' else Receiver + receiver = receiver_type((bridge, 0), output / 'receiver.jsonl') + threading.Thread(target=receiver.serve_forever, daemon=True).start() + image = identity + ':fixture' + context = output / 'image';context.mkdir() + shutil.copyfile(Path(__file__).with_name('egress_full_trial_client.py'), context / 'client.py') + shutil.copyfile(ROOT / 'scripts/tests/egress_scenarios_fixture.py', context / 'receiver.py') + (context / 'origin-port').write_text(str(receiver.server_address[1] if args.transport == 'direct' else 80)) + (context / 'test.sh').write_text('#!/bin/sh\nset -eu\npython /opt/fixture/client.py verifier "$FIXTURE_STEP_INDEX"\n') + (context / 'Dockerfile').write_text('FROM ' + args.python_image + '\nRUN mkdir -p /app /tests /opt/fixture\n' + 'COPY client.py receiver.py /opt/fixture/\nCOPY origin-port /tmp/searchswe-fixture-port\n' + 'COPY test.sh /tests/test.sh\nWORKDIR /app\n') + with (output / 'build.log').open('w') as log: + p = await asyncio.create_subprocess_exec('docker', 'build', '--network=none', '--pull=false', '-t', image, str(context), + stdout=log, stderr=log) + if await asyncio.wait_for(p.wait(), 90): + raise RuntimeError('fixture image build failed') + result['images'] = {k: json.loads(docker('image', 'inspect', v))[0]['Id'] + for k,v in [('fixture', image), ('gateway', args.gateway_image), ('python', args.python_image)]} + if any(result['images'][role] != image_id for role, image_id in inputs.items()): + raise RuntimeError('fixture input image changed during build') + task = create_task(output, image) + result['fixture_sha256'] = {str(p.relative_to(output)): hashlib.sha256(p.read_bytes()).hexdigest() + for root in (task, context) for p in root.rglob('*') if p.is_file()} + config = output / 'egress.json' + document = {'version': 1, 'image': args.gateway_image, + 'upstream': {'url': f'http://{bridge}:{receiver.server_address[1]}'}, + 'dns': {'doh_url': 'https://resolver.example/dns-query'}} + if args.transport == 'direct': + document = receiver.configuration(args.gateway_image) + config.write_text(json.dumps(document)) + environment_kwargs = ({'egress_image': args.gateway_image, 'egress_dns': document['dns']['servers'][0]} + if args.transport == 'direct' else {'egress_config': str(config)}) + trials = [] + for actor in ('left', 'right'): + cfg = TrialConfig.model_validate({'task': {'path': str(task)}, 'trial_name': identity + '-' + actor, + 'trials_dir': str(output / 'trials'), + 'agent': {'name': 'oracle', 'extra_allowed_hosts': ['model-' + actor + '.example']}, + 'environment': {'import_path': 'scripts.tests.egress_full_trial:ObservedDocker', + 'kwargs': environment_kwargs, 'env': {'SCENARIO_ACTOR': actor}}, + 'verifier': {'env': {'SCENARIO_ACTOR': actor}}}) + trials.append(await Trial.create(cfg)) + futures = [asyncio.create_task(t.run()) for t in trials] + replies = await asyncio.wait_for(asyncio.gather(*futures), 900) + for actor, reply in zip(('left','right'), replies): + if reply.exception_info or len(reply.step_results or []) != 4: + raise AssertionError('full Trial.run did not finish four steps') + for step in reply.step_results: + if step.exception_info or not step.verifier_result or step.verifier_result.rewards != {'reward': 1.0}: + raise AssertionError('step coding/artifact/scoring failed') + result['checks'].append({'case': actor + '-' + step.step_name, 'passed': True, 'reward': 1.0}) + if reply.verifier_result.rewards != {'reward': 1.0}: + raise AssertionError('trial aggregation failed') + events = receiver.snapshot() + for event in events: + if event['kind'] == 'fixture_error': + raise AssertionError('receiving fixture failed') + if event['kind'] != ('http' if args.transport == 'direct' else 'connect'): + continue + t = event['received_ns'] + envs = [e.observation for e in INSTANCES if e.observation.get('source') == event['source'] + and e.observation.get('started_ns', t+1) <= t < e.observation.get('stopped_ns', t+1)] + if len(envs) != 1: + raise AssertionError('receiving connection lacks unique live instance attribution') + policies = [p for p in envs[0]['policies'] if p['ack_ns'] <= t] + host = event['host'] if args.transport == 'direct' else event['target'].removesuffix(':80') + if not policies or host not in policies[-1]['policy']['allowed_hosts']: + raise AssertionError('unauthorized request arrived at receiving oracle') + result['checks'].append({'case': 'receiving-authorizations', 'passed': True, 'events': len(events)}) + for step in ('inherited', 'override', 'offline'): + pair = [e.observation for e in INSTANCES if e.observation['session'].endswith('__verifier__'+step)] + if len(pair) != 2 or max(e['started_ns'] for e in pair) >= min(e['stopped_ns'] for e in pair): + raise AssertionError('same-task separate verifiers did not overlap') + if pair[0]['private'] == pair[1]['private'] or pair[0]['instance'] == pair[1]['instance']: + raise AssertionError('same-task verifiers reused private state') + result['checks'].append({'case': 'overlapping-verifiers-'+step, 'passed': True}) + result['status'] = 'passed' + except BaseException as e: + result.update(status='failed', error_class=type(e).__name__, error=str(e)) + finally: + for f in futures: + if not f.done(): + f.cancel() + if futures: + await asyncio.gather(*futures, return_exceptions=True) + for env in INSTANCES: + if env._private is not None: + try: + await env.stop(delete=True) + except Exception as e: + result['cleanup_errors'].append(type(e).__name__) + for cmd in (['ps', '-aq'], ['network','ls','-q'], ['volume','ls','-q']): + try: + if docker(*cmd, '--filter', 'label=searchswe.egress.instance='+env._instance).strip(): + result['cleanup_errors'].append('owned resource remains') + except Exception as e: + result['cleanup_errors'].append('inventory unverified: '+type(e).__name__) + if receiver: + receiver.shutdown();receiver.server_close() + result['instances'] = [e.observation for e in INSTANCES] + if result['cleanup_errors']: + result['status'] = 'cleanup_error' + (output / 'result.json').write_text(json.dumps(result, indent=2)+'\n') + print(json.dumps({'status': result['status'], 'checks': len(result['checks']), 'error': result.get('error')})) + return 0 if result['status'] == 'passed' else 2 + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--gateway-image', default='searchswe-egress:dev10') + parser.add_argument('--python-image', default='python:3.13-slim') + parser.add_argument('--transport', choices=('proxy', 'direct'), default='proxy') + raise SystemExit(asyncio.run(main(parser.parse_args()))) diff --git a/scripts/tests/egress_full_trial_client.py b/scripts/tests/egress_full_trial_client.py new file mode 100644 index 0000000..7c2a0b8 --- /dev/null +++ b/scripts/tests/egress_full_trial_client.py @@ -0,0 +1,61 @@ +"""Synthetic coding, phase probes and independent scoring; never uses the internet.""" +from concurrent.futures import ThreadPoolExecutor +import json +import os +from pathlib import Path +import socket +import subprocess +import sys + +from receiver import request + +STEPS = ['inherited', 'override', 'offline', 'shared'] +AGENT_HOSTS = [['task-alpha.example'], ['step-alpha.example'], [], ['task-alpha.example']] +VERIFIER_HOSTS = [['judge-alpha.example'], ['score-alpha.example'], [], ['shared-alpha.example']] +HOSTS = ['base-alpha.example', 'vbase-alpha.example', 'task-alpha.example', 'step-alpha.example', + 'judge-alpha.example', 'score-alpha.example', 'shared-alpha.example', + 'model-left.example', 'model-right.example', 'never.example'] + +def run(mode, index): + actor = os.environ['SCENARIO_ACTOR'] + step = STEPS[index] + phase = mode + '-' + step + if mode == 'setup': + allowed = ['base-alpha.example'] + elif mode == 'agent': + allowed = AGENT_HOSTS[index] + ['model-' + actor + '.example'] + else: + allowed = VERIFIER_HOSTS[index] + with ThreadPoolExecutor(max_workers=8) as pool: + attempts = list(pool.map(lambda host: request(host, actor, phase, timeout=2), HOSTS)) + assert all(row['ok'] == (row['host'] in allowed) for row in attempts), 'phase network mismatch' + app = Path('/app') + state_file = app / 'state.json' + report = {'actor': actor, 'phase': phase, 'allowed': allowed, 'attempts': attempts, + 'hostname': socket.gethostname()} + if mode == 'agent': + if index: + state = json.loads(state_file.read_text()) + assert state['actor'] == actor and state['steps'] == STEPS[:index], 'step state crossed trials' + else: + assert not state_file.exists(), 'new trial inherited another trial state' + state = {'actor': actor, 'steps': [], 'agent_hostname': socket.gethostname()} + # A real file is authored, executed, collected and transferred by Harbor. + (app / 'answer.py').write_text('def answer(values):\n return sum(values) + ' + str(index) + '\n') + subprocess.run([sys.executable, '-c', 'from answer import answer; assert answer([1,2,3]) == ' + str(6+index)], + cwd=app, check=True) + state['steps'].append(step);state_file.write_text(json.dumps(state)) + elif mode == 'verifier': + state = json.loads(state_file.read_text()) + assert state['actor'] == actor and state['steps'] == STEPS[:index+1], 'wrong transferred artifact' + assert (state['agent_hostname'] == socket.gethostname()) == (step == 'shared'), 'wrong verifier isolation' + subprocess.run([sys.executable, '-c', 'from answer import answer; assert answer([4,5,6]) == ' + str(15+index)], + cwd=app, check=True) + report['artifact_scored'] = True + Path('/logs/verifier/reward.txt').write_text('1.0\n') + dest = Path('/logs/verifier/probe.json') if mode == 'verifier' else app / (phase + '.json') + dest.write_text(json.dumps(report, indent=2) + '\n') + print(json.dumps({'actor': actor, 'phase': phase, 'passed': True})) + +if __name__ == '__main__': + run(sys.argv[1], int(sys.argv[2])) diff --git a/scripts/tests/egress_isolation/fixture.py b/scripts/tests/egress_isolation/fixture.py new file mode 100644 index 0000000..6e1b29a --- /dev/null +++ b/scripts/tests/egress_isolation/fixture.py @@ -0,0 +1,89 @@ +"""Socket-only isolation probes. Test payloads never contain real credentials.""" + +import json +import socket +import socketserver +import sys +import threading + +import lifecyclefixture +import s0fixture + + +class Receiver(socketserver.BaseRequestHandler): + def handle(self): + if isinstance(self.request, tuple): + packet, sock = self.request + else: + sock = self.request + sock.settimeout(3) + packet = sock.recv(100) + if packet: + print(json.dumps({"kind": self.server.kind}), flush=True) + if not isinstance(self.request, tuple): + sock.sendall(b"OK") + + +class V6(s0fixture.Server): + address_family = socket.AF_INET6 + + +def serve(target, direct=False): + if target: + servers = [(s0fixture.Server(("0.0.0.0", 18083), Receiver), "cross_inbound")] + else: + servers = [(s0fixture.Server(("0.0.0.0", 18080), lifecyclefixture.Proxy), "proxy"), + (socketserver.ThreadingUDPServer(("0.0.0.0", 18081), Receiver), "udp"), + (V6(("::", 18082), Receiver), "ipv6")] + if direct: + servers.extend([(s0fixture.Server(("0.0.0.0", 443), lifecyclefixture.DirectHTTP), "http"), + (socketserver.ThreadingUDPServer(("0.0.0.0", 53), lifecyclefixture.DirectDNS), "dns")]) + for server, kind in servers: + server.kind = kind + threading.Thread(target=server.serve_forever, daemon=True).start() + print(json.dumps({"kind": "ready"}), flush=True) + threading.Event().wait() + + +def probe(mode, address): + if mode == "capabilities": + for family, kind, protocol in [(socket.AF_PACKET, socket.SOCK_RAW, 0), + (socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_RAW)]: + try: + sock = socket.socket(family, kind, protocol) + except PermissionError: + continue + sock.close() + raise AssertionError("raw socket permitted") + print("raw sockets denied") + return + if mode == "udp": + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: + sock.sendto(b"fixture", (address, 18081)) + return + ports = {"ipv6": 18082, "cross": 18083, "proxy-allow": 18080, "proxy-deny": 18080, "metadata": 443, "api": 443} + with socket.create_connection((address, ports[mode]), timeout=3) as sock: + if mode.startswith("proxy-"): + host = "allowed.example" if mode.endswith("allow") else "blocked.example" + packet = f"CONNECT {host}:443 HTTP/1.1\r\nHost: {host}:443\r\n\r\n".encode() + elif mode == "metadata": + packet = b"GET / HTTP/1.1\r\nHost: 169.254.169.254\r\n\r\n" + elif mode == "api": + packet = b"GET / HTTP/1.1\r\nHost: allowed.example\r\nConnection: close\r\n\r\n" + else: + packet = b"fixture" + sock.sendall(packet) + print("received bytes:", len(sock.recv(100))) + + +if __name__ == "__main__": + if sys.argv[1] == "idle": + print(json.dumps({"kind": "ready"}), flush=True) + threading.Event().wait() + elif sys.argv[1] in {"serve", "serve-direct", "target"}: + serve(sys.argv[1] == "target", direct=sys.argv[1] == "serve-direct") + else: + try: + probe(sys.argv[1], sys.argv[2]) + except OSError as error: + print(type(error).__name__) diff --git a/scripts/tests/egress_isolation/run.py b/scripts/tests/egress_isolation/run.py new file mode 100644 index 0000000..d6f6561 --- /dev/null +++ b/scripts/tests/egress_isolation/run.py @@ -0,0 +1,152 @@ +"""Offline UDP/IPv6/raw-capability/direct-proxy/cross-namespace gate.""" + +import argparse +import hashlib +import json +from pathlib import Path +import runpy +import time +import uuid + +HERE = Path(__file__).resolve().parent +helpers = runpy.run_path(str(HERE.parent / "egress_s0/run.py")) +docker, events, wait_ready = (helpers[n] for n in ("docker", "events", "wait_ready")) + + +def run(args): + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=False) + identity = "searchswe-isolation-" + uuid.uuid4().hex[:12] + label = "searchswe.isolation=" + identity + network, mock, gateway, target, control = (identity + s for s in ("", "-mock", "-gateway", "-target", "-control")) + containers, networks = [], [] + result = {"status": "infrastructure_error", "identity": identity, "checks": [], + "transport": args.transport, + "runner_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest()} + try: + images = {key: json.loads(docker("image", "inspect", value).stdout)[0]["Id"] for key, value in + [("gateway", args.gateway_image), ("python", args.python_image)]} + result["images"] = images + ula = "fd" + uuid.uuid4().hex[:2] + ":" + uuid.uuid4().hex[:4] + ":" + uuid.uuid4().hex[:4] + "::/64" + docker("network", "create", "--internal", "--ipv6", "--subnet", ula, "--label", label, network) + networks.append(network) + mounts = [] + result["sources"] = {} + for source, destination in [(HERE / "fixture.py", "fixture.py"), (HERE.parent / "egress_lifecycle/fixture.py", "lifecyclefixture.py"), + (HERE.parent / "egress_s0/fixture.py", "s0fixture.py")]: + mounts.extend(["-v", f"{source}:/{destination}:ro"]) + result["sources"][destination] = hashlib.sha256(source.read_bytes()).hexdigest() + containers.append(mock) + docker("run", "-d", "--pull", "never", "--name", mock, "--label", label, "--network", network, + "--cap-drop", "ALL", *mounts, "--entrypoint", "python", images["python"], "-u", "/fixture.py", + "serve-direct" if args.transport == "direct" else "serve") + wait_ready(mock) + containers.append(control) + docker("run", "-d", "--pull", "never", "--name", control, "--label", label, "--network", network, + "--cap-drop", "NET_RAW", "--cap-drop", "NET_ADMIN", "--security-opt", "no-new-privileges:true", + *mounts, "--entrypoint", "python", images["python"], "-u", "/fixture.py", "idle") + wait_ready(control) + addresses = json.loads(docker("inspect", mock).stdout)[0]["NetworkSettings"]["Networks"][network] + ip, ip6 = addresses["IPAddress"], addresses["GlobalIPv6Address"] + if not ip6: + raise RuntimeError("no IPv6 positive-control address") + config = output / "input.json" + settings = {"upstream_ip": ip, "upstream_port": 18080, "upstream_addr": f"{ip}:18080", + "upstream_host": ip, "upstream_tls": False, "doh_url": "tcp://resolver.example:15353"} + if args.transport == "direct": + settings = {"transport": "direct", "dns_servers": [ip]} + config.write_text(json.dumps(settings)) + config.chmod(0o600) + containers.append(gateway) + docker("run", "-d", "--pull", "never", "--name", gateway, "--label", label, "--network", network, + "--cap-drop", "ALL", "--cap-add", "NET_ADMIN", "--cap-add", "NET_RAW", "--cap-add", "DAC_OVERRIDE", + "--sysctl", "net.ipv6.conf.all.disable_ipv6=1", "--sysctl", "net.ipv6.conf.default.disable_ipv6=1", + "-v", f"{config}:/opt/searchswe/input.json:ro", "--entrypoint", "/opt/egress-sidecar/entrypoint.sh", images["gateway"]) + for _ in range(30): + if docker("exec", gateway, "network-policy", "show", check=False).returncode == 0: + break + time.sleep(.1) + else: + raise RuntimeError("gateway failed startup") + docker("exec", gateway, "network-policy", "allow", "allowed.example") + containers.append(target) + docker("run", "-d", "--pull", "never", "--name", target, "--label", label, "--network", "container:" + gateway, + "--cap-drop", "NET_RAW", "--cap-drop", "NET_ADMIN", "--security-opt", "no-new-privileges:true", *mounts, + "--entrypoint", "python", images["python"], "-u", "/fixture.py", "target") + wait_ready(target) + gateway_ip = json.loads(docker("inspect", gateway).stdout)[0]["NetworkSettings"]["Networks"][network]["IPAddress"] + + def probe(case, address, expected, *, confined=True, observer=mock): + docker("exec", gateway, "network-policy", "lease") + before = len(events(observer)) + response = docker("exec", target if confined else control, "python", "/fixture.py", case, address, + check=False, timeout=20) + time.sleep(.2) + # An expired lease must not turn a failed positive infrastructure + # into an apparently successful security negative. + docker("exec", gateway, "network-policy", "lease") + observed = events(observer)[before:] + if response.returncode: + raise RuntimeError("isolation client fixture failed") + if any(e.get("kind") == "fixture_error" for e in observed): + raise RuntimeError("isolation upstream fixture failed") + passed = bool(observed) == expected + if case == "api" and expected: + passed = passed and any(e.get("kind") == "http" for e in observed) + if case == "capabilities": + passed = passed and "raw sockets denied" in response.stdout + result["checks"].append({"case": case, "confined": confined, "passed": passed, + "upstream_events": observed, "client": response.stdout.strip()}) + if not passed: + if expected: + raise RuntimeError("isolation positive control failed") + result["status"] = "security_gate_failed" + raise RuntimeError("forbidden isolation traffic reached target") + + probe("proxy-allow", ip, True, confined=False) + probe("api", ip, True) + probe("proxy-deny", ip, False) + probe("metadata", ip, False) + probe("udp", ip, True, confined=False) + probe("udp", ip, False) + probe("ipv6", ip6, True, confined=False) + probe("ipv6", ip6, False) + probe("capabilities", ip, False) + probe("cross", "127.0.0.1", True, observer=target) + probe("cross", gateway_ip, False, confined=False, observer=target) + probe("api", ip, True) + result["status"] = "passed" + except Exception as error: + result["error"] = str(error) + finally: + errors = [] + for name in reversed(containers): + inspected = docker("inspect", name, check=False) + if inspected.returncode: + continue + info = json.loads(inspected.stdout)[0] + if info["Config"].get("Labels", {}).get("searchswe.isolation") != identity: + errors.append("ownership mismatch") + continue + logs = docker("logs", name, check=False) + (output / (name.removeprefix(identity + "-") + ".log")).write_text(logs.stdout + logs.stderr) + if docker("rm", "-f", name, check=False).returncode: + errors.append("container cleanup failed") + for name in networks: + if docker("network", "rm", name, check=False).returncode: + errors.append("network cleanup failed") + result["cleanup_errors"] = errors + if errors: + result["status"] = "cleanup_error" + (output / "result.json").write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps({"status": result["status"], "error": result.get("error"), "output": str(output)})) + return 0 if result["status"] == "passed" else 1 if result["status"] == "security_gate_failed" else 2 + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--gateway-image", required=True) + parser.add_argument("--python-image", default="python:3.13-slim") + parser.add_argument("--transport", choices=("proxy", "direct"), default="proxy") + parser.add_argument("--output", required=True, type=Path) + raise SystemExit(run(parser.parse_args())) diff --git a/scripts/tests/egress_lifecycle/fixture.py b/scripts/tests/egress_lifecycle/fixture.py new file mode 100644 index 0000000..a7389e2 --- /dev/null +++ b/scripts/tests/egress_lifecycle/fixture.py @@ -0,0 +1,186 @@ +"""Offline upstream + held-connection fixture. Never dials requested targets.""" + +import http.client +import json +from pathlib import Path +import socket +import socketserver +import struct +import sys +import threading +import time + +import s0fixture + + +class Proxy(socketserver.StreamRequestHandler): + def handle(self): + # Exceed the 30-second lease and the long-idle positive control. A + # short fixture timeout would falsely "prove" old-socket revocation. + self.connection.settimeout(120) + try: + request = self.rfile.readline(4096) + if not request: + return + while self.rfile.readline(4096) not in (b"\r\n", b"\n", b""): + pass + print(json.dumps({"kind": "connect", "request": request.decode().strip()}), flush=True) + self.wfile.write(b"HTTP/1.1 200 Connection established\r\n\r\n") + self.wfile.flush() + if request == b"CONNECT resolver.example:15353 HTTP/1.1\r\n": + s0fixture.MockDNS.handle(self) + return + while True: + line = self.rfile.readline(4096) + if not line: + return + headers = {} + while True: + header = self.rfile.readline(4096) + if header in (b"\r\n", b"\n", b""): + break + key, value = header.decode().split(":", 1) + headers[key.lower()] = value.strip() + print(json.dumps({"kind": "http", "host": headers.get("host"), + "request": line.decode().strip()}), flush=True) + self.wfile.write(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nOK") + self.wfile.flush() + if headers.get("connection") == "close": + return + except (TimeoutError, ConnectionError, EOFError): + return + + +def serve(): + server = s0fixture.Server(("0.0.0.0", 18080), Proxy) + print(json.dumps({"kind": "ready"}), flush=True) + server.serve_forever() + + +class DirectHTTP(socketserver.StreamRequestHandler): + def handle(self): + self.connection.settimeout(120) + try: + while True: + line = self.rfile.readline(4096) + if not line: + return + headers = {} + while True: + header = self.rfile.readline(4096) + if header in (b"\r\n", b"\n", b""): + break + key, value = header.decode().split(":", 1) + headers[key.lower()] = value.strip() + print(json.dumps({"kind": "http", "host": headers.get("host"), + "request": line.decode().strip()}), flush=True) + self.wfile.write(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nOK") + self.wfile.flush() + if headers.get("connection") == "close": + return + except (TimeoutError, ConnectionError, EOFError): + return + + +class DirectDNS(socketserver.BaseRequestHandler): + def handle(self): + packet, sock = self.request + questions = s0fixture.questions(packet) + print(json.dumps({"kind": "dns", "questions": questions}), flush=True) + address = socket.inet_aton(socket.gethostbyname(socket.gethostname())) + # These test clients send a single uncompressed A question. Ignore any + # EDNS tail and provide an actual address for the direct router. + offset = 12 + while packet[offset]: + offset += packet[offset] + 1 + question = packet[12:offset + 5] + response = packet[:2] + struct.pack("!5H", 0x8180, 1, 1, 0, 0) + question + response += b"\xc0\x0c" + struct.pack("!HHIH", 1, 1, 0, 4) + address + sock.sendto(response, self.client_address) + + +def serve_direct(): + for port in (80, 443): + server = s0fixture.Server(("0.0.0.0", port), DirectHTTP) + threading.Thread(target=server.serve_forever, daemon=True).start() + dns = socketserver.ThreadingUDPServer(("0.0.0.0", 53), DirectDNS) + threading.Thread(target=dns.serve_forever, daemon=True).start() + print(json.dumps({"kind": "ready"}), flush=True) + threading.Event().wait() + + +def dns_probe(case): + host = "allowed.example" if case == "dns-allow" else "blocked.example" + packet = s0fixture.dns_packet([(host, 3 if case == "dns-chaos" else 1)]) + addresses = [("127.0.0.1", 1053)] + if case == "docker-dns-bypass": + addresses = [("127.0.0.11", int(line.split()[1].split(":")[1], 16)) + for line in Path("/proc/net/udp").read_text().splitlines()[1:] + if line.split()[1].startswith("0B00007F:")] + if not addresses: + raise RuntimeError("embedded Docker DNS positive target missing") + answers = [] + for address in addresses: + try: + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: + sock.settimeout(2) + sock.sendto(packet, address) + response = sock.recv(4096) + answers.append(struct.unpack("!6H", response[:12])[3]) + except (TimeoutError, ConnectionError, PermissionError): + answers.append(0) + print(json.dumps({"answers": sum(answers), "targets": len(addresses)}), flush=True) + + +def wait(squat=False): + sockets = [] + if squat: + for kind, port in ((socket.SOCK_STREAM, 12345), (socket.SOCK_STREAM, 1053), (socket.SOCK_DGRAM, 1053)): + sock = socket.socket(socket.AF_INET, kind) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(("127.0.0.1", port)) + if kind == socket.SOCK_STREAM: + sock.listen() + sockets.append(sock) + print(json.dumps({"kind": "ready"}), flush=True) + threading.Event().wait() + + +def held(destination): + with socket.create_connection((destination, 443), timeout=4) as sock: + request = b"GET /held HTTP/1.1\r\nHost: allowed.example\r\n\r\n" + sock.sendall(request) + response = http.client.HTTPResponse(sock) + response.begin() + assert response.status == 200 and response.read() == b"OK" + print(json.dumps({"kind": "ready"}), flush=True) + deadline = time.monotonic() + 60 + while not Path("/tmp/release").exists(): + if time.monotonic() > deadline: + raise TimeoutError("held fixture not released") + time.sleep(.05) + try: + sock.sendall(request) + print("post-transition response bytes:", len(sock.recv(1024)), flush=True) + except (TimeoutError, ConnectionError) as error: + print(type(error).__name__, flush=True) + + +if __name__ == "__main__": + if sys.argv[1] == "serve": + serve() + elif sys.argv[1] == "serve-direct": + serve_direct() + elif sys.argv[1] == "dns-probe": + dns_probe(sys.argv[2]) + elif sys.argv[1] in {"idle", "squat"}: + wait(sys.argv[1] == "squat") + else: + if sys.argv[1] == "wait-held": + print(json.dumps({"kind": "started"}), flush=True) + deadline = time.monotonic() + 60 + while not Path("/tmp/begin").exists(): + if time.monotonic() > deadline: + raise TimeoutError("fixture start timeout") + time.sleep(.05) + held(sys.argv[2]) diff --git a/scripts/tests/egress_lifecycle/run.py b/scripts/tests/egress_lifecycle/run.py new file mode 100644 index 0000000..86fa887 --- /dev/null +++ b/scripts/tests/egress_lifecycle/run.py @@ -0,0 +1,329 @@ +"""Offline S1 controller integration; no Harbor adapter or real API claims.""" + +import argparse +import hashlib +import ipaddress +import json +from pathlib import Path +import runpy +import time +import uuid + + +S0 = Path(__file__).resolve().parents[1] / "egress_s0" +FIXTURE = Path(__file__).with_name("fixture.py") +helpers = runpy.run_path(str(S0 / "run.py")) +docker, events, wait_ready = (helpers[name] for name in ("docker", "events", "wait_ready")) + + +class SecurityGateFailure(RuntimeError): + pass + + +def run(args): + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=False) + identity = "searchswe-lifecycle-" + uuid.uuid4().hex[:12] + network, mock, gateway = identity, identity + "-mock", identity + "-gateway" + containers = [] + created_network = False + result = {"status": "infrastructure_error", "checks": [], "identity": identity, + "transport": args.transport, "dns_source": args.dns_source} + try: + images = {role: json.loads(docker("image", "inspect", image).stdout)[0]["Id"] + for role, image in [("gateway", args.gateway_image), ("python", args.python_image)]} + result["images"] = images + result["fixture_sha256"] = hashlib.sha256(FIXTURE.read_bytes()).hexdigest() + docker("network", "create", "--internal", "--label", f"searchswe.lifecycle={identity}", network) + created_network = True + containers.append(mock) + common = ["--pull", "never", "--label", f"searchswe.lifecycle={identity}"] + mounts = ["-v", f"{FIXTURE}:/fixture.py:ro", "-v", f"{S0 / 'fixture.py'}:/s0fixture.py:ro"] + docker("run", "-d", "--name", mock, "--network", network, + "--network-alias", "allowed.example", "--network-alias", "blocked.example", "--cap-drop", "ALL", *common, + *mounts, "--entrypoint", "python", images["python"], "-u", "/fixture.py", + "serve-direct" if args.transport == "direct" else "serve") + wait_ready(mock) + ip = json.loads(docker("inspect", mock).stdout)[0]["NetworkSettings"]["Networks"][network]["IPAddress"] + if args.transport == "direct": + ipam = json.loads(docker("network", "inspect", network).stdout)[0]["IPAM"]["Config"] + subnet = next(ipaddress.IPv4Network(row["Subnet"]) for row in ipam if ":" not in row["Subnet"]) + local_ip = str(subnet.network_address + 10) + for suffix, address, error in (("self-dns", local_ip, "outside the shared task network namespace"), + ("fake-docker-dns", "127.0.0.11:5353", "Docker DNS requires port 53")): + bad_gateway = identity + "-" + suffix + bad_config = output / (suffix + ".json") + bad_config.write_text(json.dumps({"transport": "direct", "dns_servers": [address]})) + bad_config.chmod(0o600) + containers.append(bad_gateway) + docker("run", "-d", "--name", bad_gateway, "--network", network, + *(["--ip", local_ip] if suffix == "self-dns" else []), + "--cap-drop", "ALL", "--cap-add", "NET_ADMIN", "--cap-add", "NET_RAW", *common, + "-v", f"{bad_config}:/opt/searchswe/input.json:ro", + "--entrypoint", "/opt/egress-sidecar/entrypoint.sh", images["gateway"]) + status = docker("wait", bad_gateway, timeout=20).stdout.strip() + log = docker("logs", bad_gateway) + if status == "0" or error not in log.stdout + log.stderr: + raise SecurityGateFailure("task-owned DNS endpoint was not rejected at gateway startup") + result["checks"].append({"case": suffix + "-rejected-before-task-start", "passed": True}) + settings = {"upstream_ip": ip, "upstream_port": 18080, "upstream_addr": f"{ip}:18080", + "upstream_host": ip, "upstream_tls": False, "doh_url": "tcp://resolver.example:15353"} + if args.transport == "direct": + settings = {"transport": "direct", "dns_servers": ["127.0.0.11" if args.dns_source == "docker" else ip]} + config = output / "input.json" + config.write_text(json.dumps(settings)) + config.chmod(0o600) + containers.append(gateway) + docker("run", "-d", "--name", gateway, "--network", network, "--cap-drop", "ALL", + "--cap-add", "NET_ADMIN", "--cap-add", "NET_RAW", "--cap-add", "KILL", *common, + "-v", f"{config}:/opt/searchswe/input.json:ro", "--entrypoint", "/opt/egress-sidecar/entrypoint.sh", + images["gateway"]) + for _ in range(50): + status = docker("exec", gateway, "network-policy", "show", check=False) + if status.returncode == 0: + break + time.sleep(.1) + else: + raise RuntimeError("controller did not start") + client = identity + "-client" + containers.append(client) + docker("run", "-d", "--name", client, "--network", f"container:{gateway}", + "--cap-drop", "NET_RAW", "--cap-drop", "NET_ADMIN", "--security-opt", "no-new-privileges:true", + *common, *mounts, "--entrypoint", "python", images["python"], "-u", "/fixture.py", "idle") + wait_ready(client) + + def change(*args): + response = docker("exec", gateway, "network-policy", *args) + state = json.loads(response.stdout) + result.setdefault("transitions", []).append(state) + return state + + def probe(case, expected, renew=True): + if renew: + response = docker("exec", gateway, "network-policy", "lease", check=False) + if expected and response.returncode: + raise RuntimeError("positive-control lease is unavailable") + before = len(events(mock)) + if args.dns_source == "docker" and case.startswith("dns-"): + response = docker("exec", client, "python", "/fixture.py", "dns-probe", case, timeout=15) + evidence = json.loads(response.stdout) + if (evidence["answers"] > 0) != expected: + raise SecurityGateFailure("Docker DNS response violated phase policy") + result["checks"].append({"case": case, "passed": True, **evidence}) + return + response = docker("exec", client, "python", "/s0fixture.py", case, ip, timeout=15) + time.sleep(.2) + observed = events(mock)[before:] + reached = any(e.get("kind") in {"connect", "dns", "http"} for e in observed) + valid_positive = any(e.get("kind") == ("dns" if case.startswith("dns-") else "http") for e in observed) + passed = reached == expected and (not expected or valid_positive) + if any(e.get("kind") == "fixture_error" for e in observed): + raise RuntimeError("mock upstream fixture failed") + result["checks"].append({"case": case, "passed": passed, "upstream_events": observed, + "expected_upstream": expected, + "client_output": response.stdout.strip()}) + if not passed: + if expected: + raise RuntimeError(f"positive control failed: {case}") + raise SecurityGateFailure(f"forbidden request reached upstream: {case}") + + def hold(suffix, public=False): + name = identity + "-" + suffix + containers.append(name) + docker("run", "-d", "--name", name, "--network", f"container:{gateway}", "--cap-drop", "ALL", + *common, *mounts, "--entrypoint", "python", images["python"], "-u", "/fixture.py", "wait-held", ip) + for _ in range(50): + if any(e.get("kind") == "started" for e in events(name)): + break + time.sleep(.1) + else: + raise RuntimeError("held actor failed to start") + deadline = time.monotonic_ns() + 30 * 10**9 + change("--deadline-ns", str(deadline), *( ["allow-all"] if public else ["allow", "allowed.example"])) + docker("exec", name, "touch", "/tmp/begin") + wait_ready(name) + return name, deadline + + change("allow", "allowed.example") + capability = docker("exec", client, "python", "/s0fixture.py", "capability", ip) + if "SO_MARK denied" not in capability.stdout: + raise SecurityGateFailure("root task can mark its own egress sockets") + result["checks"].append({"case": "root-so-mark-denied", "passed": True}) + probe("http-allow", True) + probe("http-deny", False) + probe("dns-allow", True) + probe("dns-chaos", False) + if args.transport == "direct": + change("allow-all") + probe("http-deny", True) + def embedded_dns(expected): + response = docker("exec", client, "python", "/fixture.py", "dns-probe", "docker-dns-bypass") + evidence = json.loads(response.stdout) + if (evidence["answers"] > 0) != expected: + raise SecurityGateFailure("embedded DNS bypass control failed") + result["checks"].append({"case": "embedded-dns-public-positive" if expected else "embedded-dns-bypass-denied", + "passed": True, **evidence}) + embedded_dns(True) + public_held, _ = hold("public-held", public=True) + change("allow", "allowed.example") + before = len(events(mock)) + docker("exec", public_held, "touch", "/tmp/release") + if docker("wait", public_held, timeout=10).stdout.strip() != "0" or events(mock)[before:]: + raise SecurityGateFailure("old raw public connection survived restriction") + result["checks"].append({"case": "public-connection-revoked-by-allowlist", "passed": True}) + embedded_dns(False) + probe("http-deny", False) + probe("http-allow", True) + probe("dns-deny", False) + # Prove that the SAME idle connection can outlive the lease interval + # when the host renews. Otherwise an origin/proxy idle timeout could + # masquerade as the later frozen-controller old-socket revocation. + idle_control, _ = hold("idle-control") + time.sleep(16) + docker("exec", gateway, "network-policy", "lease") + time.sleep(17) + before = len(events(mock)) + docker("exec", idle_control, "touch", "/tmp/release") + if docker("wait", idle_control, timeout=15).stdout.strip() != "0": + raise RuntimeError("long-idle positive client failed") + observed = events(mock)[before:] + if not any(e.get("kind") == "http" and e.get("request") == "GET /held HTTP/1.1" for e in observed): + raise RuntimeError("long-idle connection positive control failed") + result["checks"].append({"case": "idle-connection-outlives-lease-with-renewal", "passed": True, + "upstream_events": observed}) + # Keep an acknowledged real upstream HTTP connection across revocation. + held, _ = hold("held") + change("deny-all") + before = len(events(mock)) + docker("exec", held, "touch", "/tmp/release") + exited = docker("wait", held, timeout=10) + if exited.stdout.strip() != "0": + raise RuntimeError("held client failed") + time.sleep(.2) + observed = events(mock)[before:] + passed = not observed + result["checks"].append({"case": "existing-connection-revoked", "passed": passed, + "upstream_events": observed}) + if not passed: + raise SecurityGateFailure("old connection remained authorized") + probe("http-allow", False) + change("allow", "allowed.example") + probe("http-allow", True) + # Invalid replacement must not leave the previous permissive worker. + response = docker("exec", gateway, "network-policy", "allow", "*.example", check=False) + if response.returncode == 0: + raise RuntimeError("invalid policy accepted") + probe("http-allow", False) + change("allow", "allowed.example") + probe("http-allow", True) + docker("exec", gateway, "sh", "-c", "kill -9 $(pidof gost)") + for _ in range(50): + if docker("exec", gateway, "network-policy", "show", check=False).returncode: + break + time.sleep(.1) + else: + raise RuntimeError("worker crash did not invalidate readiness") + probe("http-allow", False) + change("allow", "allowed.example") + probe("http-allow", True) + # Readiness must not mistake a task-owned listener for the new worker. + change("deny-all") + squatter = identity + "-squatter" + containers.append(squatter) + docker("run", "-d", "--name", squatter, "--network", f"container:{gateway}", "--cap-drop", "ALL", + *common, *mounts, "--entrypoint", "python", images["python"], "-u", "/fixture.py", "squat") + wait_ready(squatter) + response = docker("exec", gateway, "network-policy", "allow", "allowed.example", check=False) + if response.returncode == 0: + raise SecurityGateFailure("task-owned listener was accepted as gateway readiness") + probe("http-allow", False) + result["checks"][-1]["case"] = "task-listener-cannot-fake-readiness" + docker("rm", "-f", squatter) + change("allow", "allowed.example") + probe("http-allow", True) + # Freeze only the controller (PID 1), not the proxy worker. The kernel + # lease, independent of the Python watchdog, must revoke egress. + frozen_held, deadline = hold("frozen-held") + probe("http-allow", True, renew=False) + docker("kill", "--signal=STOP", gateway) + docker("exec", gateway, "python3", "-c", "from pathlib import Path; import subprocess; " + "pid=subprocess.check_output(['pidof','gost'],text=True).strip(); " + "assert Path('/proc/'+pid+'/stat').read_text().split()[2] not in {'T','t','Z'}") + time.sleep(max(0, (deadline - time.monotonic_ns()) / 10**9) + .3) + probe("http-allow", False, renew=False) + result["checks"][-1]["case"] = "kernel-lease-controller-frozen" + before = len(events(mock)) + docker("exec", frozen_held, "touch", "/tmp/release") + if docker("wait", frozen_held, timeout=15).stdout.strip() != "0": + raise RuntimeError("frozen held client failed") + observed = events(mock)[before:] + if observed: + raise SecurityGateFailure("old connection survived the kernel lease") + result["checks"].append({"case": "old-connection-kernel-lease-controller-frozen", "passed": True, + "upstream_events": observed}) + docker("kill", "--signal=CONT", gateway) + response = docker("exec", gateway, "network-policy", "--deadline-ns", str(deadline), + "allow", "allowed.example", check=False) + if response.returncode == 0: + raise SecurityGateFailure("stale host command reopened an expired generation") + probe("http-allow", False) + result["checks"][-1]["case"] = "expired-command-cannot-reopen" + change("allow", "allowed.example") + probe("http-allow", True) + if args.transport == "direct": + public_held, deadline = hold("public-frozen-held", public=True) + docker("kill", "--signal=STOP", gateway) + time.sleep(max(0, (deadline - time.monotonic_ns()) / 10**9) + .3) + probe("http-allow", False, renew=False) + result["checks"][-1]["case"] = "public-kernel-lease-controller-frozen" + before = len(events(mock)) + docker("exec", public_held, "touch", "/tmp/release") + if docker("wait", public_held, timeout=10).stdout.strip() != "0" or events(mock)[before:]: + raise SecurityGateFailure("old public connection survived expired kernel lease") + result["checks"].append({"case": "old-public-connection-kernel-lease", "passed": True}) + docker("kill", "--signal=CONT", gateway) + change("allow", "allowed.example") + probe("http-allow", True) + result["status"] = "passed" + return 0 + except SecurityGateFailure as error: + result["status"] = "security_gate_failed" + result["error"] = str(error) + return 1 + except Exception as error: + result["error"] = str(error) + return 2 + finally: + errors = [] + for name in reversed(containers): + inspected = docker("inspect", name, check=False) + if inspected.returncode: + continue + info = json.loads(inspected.stdout)[0] + if info["Config"].get("Labels", {}).get("searchswe.lifecycle") != identity: + errors.append(f"ownership mismatch: {name}") + continue + log = docker("logs", name, check=False) + (output / f"{name.removeprefix(identity + '-')}.log").write_text(log.stdout + log.stderr) + if name == gateway: + rules = docker("exec", gateway, "nft", "list", "ruleset", check=False) + (output / "rules.nft").write_text(rules.stdout + rules.stderr) + if docker("rm", "-f", name, check=False).returncode: + errors.append(f"container cleanup failed: {name}") + if created_network and docker("network", "rm", network, check=False).returncode: + errors.append("network cleanup failed") + result["cleanup_errors"] = errors + (output / "result.json").write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps({"status": result["status"], "error": result.get("error"), "output": str(output)})) + if errors: + raise RuntimeError("lifecycle cleanup incomplete") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--gateway-image", required=True) + parser.add_argument("--python-image", default="python:3.13-slim") + parser.add_argument("--transport", choices=("proxy", "direct"), default="proxy") + parser.add_argument("--dns-source", choices=("fixture", "docker"), default="fixture") + parser.add_argument("--output", required=True, type=Path) + raise SystemExit(run(parser.parse_args())) diff --git a/scripts/tests/egress_orphan_smoke.py b/scripts/tests/egress_orphan_smoke.py new file mode 100644 index 0000000..edb7a68 --- /dev/null +++ b/scripts/tests/egress_orphan_smoke.py @@ -0,0 +1,140 @@ +"""Kill one dedicated host driver, then verify its lease and orphan recovery.""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import shutil +import stat +import subprocess +import sys +import time + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from scripts.egress.ownership import owner_root, read_manifest +from scripts.egress_cleanup import recover, inventory +from scripts.tests.egress_credentials import CredentialProbe +from environments.egress.gateway import LEASE_SECONDS + + +def run(args): + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=False) + result = {"status": "infrastructure_error", "checks": []} + private, child = None, None + credentials = CredentialProbe() if args.credential_probe else None + try: + with (output / "driver.log").open("w") as log: + child = subprocess.Popen([sys.executable, str(Path(__file__).with_name("egress_adapter_smoke.py")), + "--gateway-image", args.gateway_image, "--output", str(output / "child"), + "--orphan-child", *(["--auth-file", str(credentials.path)] if credentials else [])], + stdout=log, stderr=subprocess.STDOUT) + ready = output / "child/owner-ready.json" + deadline = time.monotonic() + 180 + while not ready.exists(): + if child.poll() is not None or time.monotonic() > deadline: + raise RuntimeError("orphan driver failed startup") + time.sleep(.2) + owner = json.loads(ready.read_text()) + private = Path(owner["private"]) + if credentials: + if not owner.get("credential_boundary_checked"): + raise RuntimeError("child credential boundary probe did not run") + result["checks"].append({"case": "pre-crash-credential-boundary", "passed": True}) + try: + recover(private, remove=False) + except BlockingIOError: + result["checks"].append({"case": "live-owner-excluded", "passed": True}) + else: + raise RuntimeError("live owner lock was not enforced") + child.kill() + child.wait(timeout=15) + result["driver_returncode"] = child.returncode + manifest = read_manifest(private) + resources = inventory(manifest) + gateway = next(r["id"] for r in resources if r["gateway"]) + # A single bounded lease deadline, not polling for an external wake. + time.sleep(LEASE_SECONDS + 1) + closed = subprocess.run(["docker", "exec", gateway, "network-policy", "show"], + capture_output=True, text=True, timeout=15) + state = json.loads(closed.stdout) + if closed.returncode == 0 or state.get("ready") or state.get("error") != "lease_expired": + raise RuntimeError("host death did not expire the gateway lease") + result["checks"].append({"case": "host-sigkill-lease-expired", "passed": True}) + recovered = subprocess.run([sys.executable, str(Path(__file__).resolve().parents[1] / "egress_cleanup.py"), + "--directory", str(private), "--remove"], + check=True, capture_output=True, text=True, timeout=180) + result["recovery"] = json.loads(recovered.stdout) + audits = result["recovery"]["audit_paths"] + if len(audits) != 1: + raise RuntimeError("orphan recovery did not preserve one gateway audit") + audit = Path(audits[0]) + info = audit.lstat() + if (audit.parent != owner_root() or not stat.S_ISREG(info.st_mode) + or stat.S_IMODE(info.st_mode) != 0o600 or info.st_uid != os.getuid() or info.st_nlink != 1): + raise RuntimeError("unsafe recovery audit permissions") + data = audit.read_bytes() + if credentials: + credentials.scan(data, "recovered-audit") + # entrypoint.sh emits one fixed readiness banner in addition to JSON + # controller events. Do not silently accept any other unstructured log. + records = [json.loads(line) for line in data.splitlines() + if line and line != b"Search-SWE restricted gateway ready"] + if not records or any(row.get("instance") != owner["instance"] for row in records): + raise RuntimeError("recovered audit lost instance ownership") + if not {"generation_ready", "lease_expired"} <= {row["event"] for row in records}: + raise RuntimeError("recovered audit lost policy/expiry events") + ready = [row for row in records if row["event"] == "generation_ready"] + if any(not row.get("generation") or len(row.get("policy_sha256", "")) != 64 for row in ready): + raise RuntimeError("recovered audit lost policy generation/hash") + if not any(isinstance(row.get("packet_counters"), dict) and "proxy_packets" in row["packet_counters"] for row in ready): + raise RuntimeError("recovered audit lost kernel packet counters") + shutil.copyfile(audit, output / "recovery-audit.log") + result["checks"].append({"case": "durable-owner-only-policy-audit", "passed": True, + "sha256": hashlib.sha256(data).hexdigest(), "events": len(records)}) + if private.exists() or inventory(manifest): + raise RuntimeError("orphan resources/private directory remain") + result["checks"].append({"case": "owned-orphan-removed", "passed": True}) + result["status"] = "passed" + except Exception as error: + result["error"] = str(error) + finally: + if child is not None and child.poll() is None: + child.kill() + child.wait(timeout=15) + if private is None: + identity_file = output / "child/driver-identity.json" + if identity_file.exists(): + instance = json.loads(identity_file.read_text())["instance"] + candidates = list(owner_root().glob("instance-" + instance + "-*")) + if len(candidates) == 1: + private = candidates[0] + if private is not None and private.exists(): + try: + result["failure_recovery"] = recover(private, remove=True) + except Exception as error: + result["cleanup_error"] = str(error) + if credentials: + try: + credentials.artifacts(output) + credentials.scan(json.dumps(result), "orphan-result") + credentials.close() + if credentials.path.exists(): + raise RuntimeError("credential source was not removed") + result["checks"].append({"case": "orphan-credential-artifacts-clean", "passed": True}) + except Exception: + result = {"status": "credential_evidence_error", "error": "Orphan credential evidence validation failed"} + finally: + credentials.close() + (output / "result.json").write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result)) + return 0 if result["status"] == "passed" else 2 + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--gateway-image", required=True) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--credential-probe", action="store_true", help="Also test synthetic credentials across host death and recovery") + raise SystemExit(run(parser.parse_args())) diff --git a/scripts/tests/egress_s0/fixture.py b/scripts/tests/egress_s0/fixture.py new file mode 100644 index 0000000..c441e6e --- /dev/null +++ b/scripts/tests/egress_s0/fixture.py @@ -0,0 +1,185 @@ +"""Offline S0 fixtures, NOT a production proxy or resolver. + +The mock upstream records requests but never connects to their destinations. +All names are reserved .example names; no credentials or internet are needed. +""" + +import json +import socket +import socketserver +import ssl +import struct +import sys +import threading + + +def read_exact(stream, length): + data = b"" + while len(data) < length: + part = stream.read(length - len(data)) + if not part: + raise EOFError("short fixture message") + data += part + return data + + +def questions(packet): + """Decode the uncompressed questions generated by this fixture only.""" + count = struct.unpack("!H", packet[4:6])[0] + offset, result = 12, [] + for _ in range(count): + labels = [] + while packet[offset]: + length = packet[offset] + if length > 63: + raise ValueError("compressed fixture question not supported") + labels.append(packet[offset + 1:offset + 1 + length].decode("ascii")) + offset += length + 1 + offset += 1 + qtype, qclass = struct.unpack("!HH", packet[offset:offset + 4]) + offset += 4 + result.append({"name": ".".join(labels), "type": qtype, "class": qclass}) + return result + + +def dns_packet(items): + packet = struct.pack("!HHHHHH", 1234, 0x100, len(items), 0, 0, 0) + for name, qclass in items: + for label in name.split("."): + packet += bytes([len(label)]) + label.encode("ascii") + packet += b"\0" + struct.pack("!HH", 1, qclass) + return packet + + +class MockProxy(socketserver.StreamRequestHandler): + def handle(self): + self.connection.settimeout(5) + line = self.rfile.readline(4096) + if not line: # readiness probe + return + while self.rfile.readline(4096) not in (b"\r\n", b"\n", b""): + pass + print(json.dumps({"kind": "connect", "request": line.decode().strip()}), flush=True) + # No outbound connection, even for allowed names. + self.wfile.write(b"HTTP/1.1 200 Connection established\r\n\r\n") + self.wfile.flush() + data = self.rfile.read(1) + if data == b"G": + self.wfile.write(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nOK") + # For HTTP/2 the CONNECT log, not a successful response, is the oracle. + + +class MockDNS(socketserver.StreamRequestHandler): + def handle(self): + self.connection.settimeout(5) + header = self.rfile.read(2) + if not header: + return + packet = read_exact(self.rfile, struct.unpack("!H", header)[0]) + print(json.dumps({"kind": "dns", "questions": questions(packet)}), flush=True) + response = packet[:2] + b"\x81\x80" + packet[4:] + self.wfile.write(struct.pack("!H", len(response)) + response) + + +class Server(socketserver.ThreadingTCPServer): + allow_reuse_address = True + daemon_threads = True + + def handle_error(self, request, client_address): + print(json.dumps({"kind": "fixture_error"}), flush=True) + + +def serve(): + for port, handler in [(18080, MockProxy), (15353, MockDNS)]: + server = Server(("0.0.0.0", port), handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + print(json.dumps({"kind": "ready"}), flush=True) + threading.Event().wait() + + +def frame(kind, flags, stream_id, payload): + return len(payload).to_bytes(3, "big") + bytes([kind, flags]) + struct.pack("!I", stream_id) + payload + + +def probe(case, destination): + if case == "capability": + with socket.socket() as sock: + try: + sock.setsockopt(socket.SOL_SOCKET, 36, 114514) + except PermissionError: + print("SO_MARK denied") + return + raise AssertionError("SO_MARK unexpectedly permitted") + if case.startswith("dns-"): + items = { + "dns-allow": [("allowed.example", 1)], + "dns-deny": [("blocked.example", 1)], + "dns-chaos": [("blocked.example", 3)], + "dns-hesiod": [("blocked.example", 4)], + "dns-any-class": [("blocked.example", 255)], + "dns-multiple": [("allowed.example", 1), ("blocked.example", 1)], + **{name: [("allowed.example", 1)] for name in ( + "dns-txt", "dns-answer", "dns-authority", "dns-extra", "dns-edns-payload")}, + }[case] + packet = dns_packet(items) + if case == "dns-txt": + packet = packet[:-4] + struct.pack("!HH", 16, 1) + if case in {"dns-answer", "dns-authority", "dns-extra", "dns-edns-payload"}: + header = list(struct.unpack("!6H", packet[:12])) + header[{"dns-answer": 3, "dns-authority": 4}.get(case, 5)] = 1 + payload = b"fixture-payload" + if case == "dns-edns-payload": + payload = struct.pack("!HH", 65001, len(payload)) + payload + record = b"\0" + struct.pack("!HHIH", 41, 1232, 0, len(payload)) + payload + else: + payload = bytes([len(payload)]) + payload + record = b"\x07blocked\x07example\0" + struct.pack("!HHIH", 16, 1, 0, len(payload)) + payload + packet = struct.pack("!6H", *header) + packet[12:] + record + with socket.create_connection(("127.0.0.1", 1053), timeout=4) as sock: + sock.sendall(struct.pack("!H", len(packet)) + packet) + print("DNS response bytes:", len(sock.recv(4096))) + return + host = "allowed.example" if case.endswith("allow") else "blocked.example" + with socket.create_connection((destination, 443), timeout=4) as sock: + if case.startswith("tls-"): + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE # offline mock has no TLS origin + with context.wrap_socket(sock, server_hostname=None if case == "tls-no-sni" else host): + pass + elif case == "unknown-protocol": + sock.sendall(b"\x01\x02\x03unsupported\r\n\r\n") + elif case == "http-ip": + sock.sendall(b"GET / HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n") + elif case == "http-connect-mismatch": + sock.sendall(b"CONNECT blocked.example:443 HTTP/1.1\r\nHost: allowed.example\r\n\r\n") + elif case == "http-absolute-mismatch": + sock.sendall(b"GET http://blocked.example/ HTTP/1.1\r\nHost: allowed.example\r\n\r\n") + elif case.startswith("h2-"): + # HPACK static GET, https, /; literal :authority without Huffman. + if case == "h2-multiple": + host = "allowed.example" + headers = b"\x82\x87\x84\x01" + bytes([len(host)]) + host.encode() + sock.sendall(b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" + frame(4, 0, 0, b"") + frame(1, 5, 1, headers)) + if case == "h2-multiple": + host = "blocked.example" + headers = b"\x82\x87\x84\x01" + bytes([len(host)]) + host.encode() + sock.sendall(frame(1, 5, 3, headers)) + # Do not close on the peer's initial SETTINGS before it has had an + # opportunity to process our HEADERS and attempt the upstream dial. + while sock.recv(4096): + pass + else: + sock.sendall(f"GET / HTTP/1.1\r\nHost: {host}\r\nConnection: close\r\n\r\n".encode()) + print("response bytes:", len(sock.recv(1024))) + + +if __name__ == "__main__": + if sys.argv[1] == "serve": + serve() + else: + try: + probe(sys.argv[1], sys.argv[2]) + except (TimeoutError, ConnectionError, EOFError, ssl.SSLError) as error: + # Network failure is not a PASS. The runner inspects upstream logs. + print(type(error).__name__) diff --git a/scripts/tests/egress_s0/run.py b/scripts/tests/egress_s0/run.py new file mode 100644 index 0000000..74f83d6 --- /dev/null +++ b/scripts/tests/egress_s0/run.py @@ -0,0 +1,179 @@ +"""Explicit, offline release gate for Harbor's GOST candidate (not runtime). + +Requires preloaded images; never pulls, publishes ports, or calls external APIs. +Exit 1 means a negative probe reached the mock upstream; exit 2 means test error. +""" + +import argparse +import hashlib +import json +from pathlib import Path +import subprocess +import time +import uuid + + +FIXTURE = Path(__file__).with_name("fixture.py").resolve() + + +def docker(*args, check=True, timeout=60): + return subprocess.run(["docker", *args], check=check, text=True, + capture_output=True, timeout=timeout) + + +def events(name): + return [json.loads(line) for line in docker("logs", name).stdout.splitlines() if line.startswith("{")] + + +def wait_ready(name): + for _ in range(50): + if any(e.get("kind") == "ready" for e in events(name)): + return + time.sleep(.1) + raise RuntimeError("mock upstream did not become ready") + + +def configuration(upstream): + return { + "log": {"level": "error"}, + "services": [ + {"name": "transparent", "addr": ":12345", "bypass": "policy", + "metadata": {"so_mark": "114514"}, + "handler": {"type": "red", "chain": "upstream", "metadata": { + "sniffing": True, "sniffing.timeout": "2s", "sniffing.fallback": False}}, + "listener": {"type": "red"}}, + {"name": "dns", "addr": "127.0.0.1:1053", "bypass": "policy", + "metadata": {"so_mark": "114514"}, + "handler": {"type": "dns", "metadata": { + "dns": f"tcp://{upstream}:15353", "timeout": "2s"}}, + "listener": {"type": "dns", "metadata": {"mode": "tcp"}}}, + ], + "bypasses": [{"name": "policy", "whitelist": True, "matchers": ["allowed.example"]}], + "chains": [{"name": "upstream", "hops": [{"name": "proxy", "nodes": [ + {"name": "mock", "addr": f"{upstream}:18080", "metadata": {"so_mark": "114514"}, + "connector": {"type": "http"}, "dialer": {"type": "tcp"}}, + ]}]}], + } + + +def run(args): + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=False) + identity = "searchswe-s0-" + uuid.uuid4().hex[:12] + network, mock, gateway = identity, identity + "-mock", identity + "-gateway" + containers = [] + created_network = False + result = {"status": "infrastructure_error", "checks": [], "identity": identity} + try: + # Capture immutable IDs, and run those IDs rather than mutable tags. + images = {} + for role, image in [("gost", args.gost_image), ("python", args.python_image)]: + images[role] = json.loads(docker("image", "inspect", image).stdout)[0]["Id"] + result["images"] = images + result["fixture_sha256"] = hashlib.sha256(FIXTURE.read_bytes()).hexdigest() + docker("network", "create", "--internal", "--label", f"searchswe.s0={identity}", network) + created_network = True + containers.append(mock) + docker("run", "-d", "--pull", "never", "--name", mock, "--network", network, + "--cap-drop", "ALL", "--security-opt", "no-new-privileges:true", + "--label", f"searchswe.s0={identity}", + "-v", f"{FIXTURE}:/fixture.py:ro", "--entrypoint", "python", + images["python"], "-u", "/fixture.py", "serve") + wait_ready(mock) + info = json.loads(docker("inspect", mock).stdout)[0] + upstream = info["NetworkSettings"]["Networks"][network]["IPAddress"] + config = output / "gost.json" + config.write_text(json.dumps(configuration(upstream), indent=2) + "\n") + containers.append(gateway) + docker("run", "-d", "--pull", "never", "--name", gateway, "--network", network, + "--cap-add", "NET_ADMIN", "--cap-add", "NET_RAW", + "--label", f"searchswe.s0={identity}", + "-v", f"{config}:/gost.json:ro", "--entrypoint", "/bin/gost", + images["gost"], "-C", "/gost.json") + # Apply the image's original Harbor redirect/filter rules. Static policy + # in this candidate configuration equals the file policy in this helper. + docker("exec", gateway, "network-policy", "allow", "allowed.example") + client = identity + "-client" + containers.append(client) + docker("run", "-d", "--pull", "never", "--name", client, + "--network", f"container:{gateway}", "--cap-drop", "NET_RAW", + "--cap-drop", "NET_ADMIN", "--security-opt", "no-new-privileges:true", + "--label", f"searchswe.s0={identity}", "-v", f"{FIXTURE}:/fixture.py:ro", + "--entrypoint", "sleep", images["python"], "infinity") + for case, expected in [ + ("capability", False), ("http-allow", True), ("http-deny", False), + ("h2-allow", True), ("h2-deny", False), ("dns-allow", True), + ("dns-deny", False), ("dns-chaos", False), ("dns-multiple", False), + ("dns-hesiod", False), ("dns-any-class", False), + ("h2-multiple", True), ("tls-allow", True), ("tls-deny", False), + ("tls-no-sni", False), ("http-ip", False), ("unknown-protocol", False), + ("http-connect-mismatch", False), ("http-absolute-mismatch", False), + ("dns-txt", False), ("dns-answer", False), ("dns-authority", False), + ("dns-extra", False), ("dns-edns-payload", False), + ]: + before = len(events(mock)) + response = docker( + "exec", client, "python", "/fixture.py", case, upstream, + timeout=30, check=False) + if response.returncode: + raise RuntimeError(f"client {case} failed: {response.stderr}") + # Bounded observation window, not a substitute for positive controls. + time.sleep(.2) + observed = events(mock)[before:] + if any(e.get("kind") == "fixture_error" for e in observed): + raise RuntimeError(f"mock upstream failed during {case}") + reached = any(e.get("kind") in {"connect", "dns"} for e in observed) + # Multi-stream positives must not hide a forbidden second target. + forbidden = [e for e in observed if ( + e.get("kind") == "connect" and e.get("request") != "CONNECT allowed.example:443 HTTP/1.1" + ) or (e.get("kind") == "dns" and any(q["name"] != "allowed.example" or q["class"] != 1 + for q in e["questions"]))] + passed = reached == expected and not forbidden + if case == "capability": + passed = passed and "SO_MARK denied" in response.stdout + result["checks"].append({"case": case, "passed": passed, + "expected_upstream": expected, "upstream_events": observed, + "client_output": response.stdout.strip()}) + print(f"{case}: {'PASS' if passed else 'FAIL'}", flush=True) + if expected and not passed: + raise RuntimeError(f"positive control {case} failed; no security conclusion") + positives = [c for c in result["checks"] if c["expected_upstream"]] + if not all(c["passed"] for c in positives): + raise RuntimeError("positive control failed; no security conclusion") + result["status"] = "passed" if all(c["passed"] for c in result["checks"]) else "security_gate_failed" + return 0 if result["status"] == "passed" else 1 + except Exception as error: + result["error"] = str(error) + return 2 + finally: + cleanup_errors = [] + for name in reversed(containers): + inspected = docker("container", "inspect", name, check=False) + if inspected.returncode: + continue + info = json.loads(inspected.stdout)[0] + if info["Config"].get("Labels", {}).get("searchswe.s0") != identity: + cleanup_errors.append(f"ownership mismatch: {name}") + continue + if name in {gateway, mock}: + log = docker("logs", name, check=False) + (output / f"{'gateway' if name == gateway else 'upstream'}.log").write_text(log.stdout + log.stderr) + removed = docker("rm", "-f", name, check=False) + if removed.returncode: + cleanup_errors.append(f"container cleanup failed: {name}") + if created_network: + removed = docker("network", "rm", network, check=False) + if removed.returncode: + cleanup_errors.append(f"network cleanup failed: {network}") + result["cleanup_errors"] = cleanup_errors + (output / "result.json").write_text(json.dumps(result, indent=2) + "\n") + if cleanup_errors: + raise RuntimeError("S0 cleanup incomplete; see result.json") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--gost-image", required=True, help="Preloaded Harbor sidecar image containing network-policy") + parser.add_argument("--python-image", default="python:3.13-slim") + parser.add_argument("--output", required=True, type=Path, help="New evidence directory; must not exist") + raise SystemExit(run(parser.parse_args())) diff --git a/scripts/tests/egress_scenarios_fixture.py b/scripts/tests/egress_scenarios_fixture.py new file mode 100644 index 0000000..ec460a4 --- /dev/null +++ b/scripts/tests/egress_scenarios_fixture.py @@ -0,0 +1,151 @@ +"""Offline receiving oracle and bounded continuous HTTP clients. No target dialing.""" + +import http.client +import json +from pathlib import Path +import socket +import socketserver +import struct +import threading +import time + + +class Receiver(socketserver.ThreadingTCPServer): + allow_reuse_address = True + daemon_threads = True + # The positive-control burst has eight simultaneous clients. The default + # backlog of five can force a one-second SYN retry before the mock accepts. + request_queue_size = 64 + + def __init__(self, address, output, *, direct=False): + self.events = [] + self.lock = threading.Lock() + self.output = output + self.direct = direct + super().__init__(address, Handler) + + def record(self, **event): + with self.lock: + event["received_ns"] = time.monotonic_ns() + self.events.append(event) + with self.output.open("a") as stream: + stream.write(json.dumps(event) + "\n") + + def snapshot(self): + with self.lock: + return list(self.events) + + +class DirectReceiver(Receiver): + """Real direct HTTP + UDP DNS, bound only to ephemeral Docker-bridge ports.""" + + def __init__(self, address, output): + super().__init__(address, output, direct=True) + try: + self.dns = socketserver.ThreadingUDPServer((address[0], 0), DirectDNS) + except BaseException: + self.server_close() + raise + self.dns.origin = self + threading.Thread(target=self.dns.serve_forever, daemon=True).start() + + def shutdown(self): + self.dns.shutdown() + super().shutdown() + + def server_close(self): + if hasattr(self, "dns"): + self.dns.server_close() + super().server_close() + + def configuration(self, image): + host, port = self.dns.server_address + return {"version": 1, "mode": "direct", "image": image, "dns": {"servers": [f"{host}:{port}"]}} + + +class DirectDNS(socketserver.BaseRequestHandler): + def handle(self): + packet, sock = self.request + offset, labels = 12, [] + while packet[offset]: + size = packet[offset] + if size > 63: + raise ValueError("fixture expects an uncompressed DNS query") + labels.append(packet[offset+1:offset+1+size].decode()) + offset += size + 1 + question = packet[12:offset+5] + qtype, qclass = struct.unpack("!HH", packet[offset+1:offset+5]) + self.server.origin.record(kind="dns", host=".".join(labels), source=self.client_address[0]) + valid = qtype == 1 and qclass == 1 + response = packet[:2] + struct.pack("!5H", 0x8180, 1, int(valid), 0, 0) + question + if valid: + response += b"\xc0\x0c" + struct.pack("!HHIH", 1, 1, 0, 4) + response += socket.inet_aton(self.server.origin.server_address[0]) + sock.sendto(response, self.client_address) + + +class Handler(socketserver.StreamRequestHandler): + def handle(self): + self.connection.settimeout(5) + try: + line = self.rfile.readline(4096).decode().strip() + if not line: + return + method, target, version = line.split() + if self.server.direct: + request = line + else: + if method != "CONNECT" or version != "HTTP/1.1": + raise ValueError("unexpected proxy request") + self.headers() + self.server.record(kind="connect", target=target, source=self.client_address[0]) + self.wfile.write(b"HTTP/1.1 200 Connection established\r\n\r\n") + self.wfile.flush() + request = self.rfile.readline(4096).decode().strip() + if not request: + return + method, path, version = request.split() + headers = self.headers() + if (method != "GET" or version != "HTTP/1.1" + or (not self.server.direct and target != headers.get("host", "") + ":80")): + raise ValueError("origin/CONNECT mismatch in fixture") + _, actor, phase, sent = path.split("/") + self.server.record(kind="http", host=headers["host"].split(":")[0], actor=actor, phase=phase, + sent_ns=int(sent), source=self.client_address[0]) + self.wfile.write(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nOK") + self.wfile.flush() + except (OSError, http.client.HTTPException): + pass # Client cancellation during revocation is expected. + except Exception as error: + self.server.record(kind="fixture_error", error=type(error).__name__) + + def headers(self): + headers = {} + for _ in range(32): + line = self.rfile.readline(4096) + if line in {b"\r\n", b"\n"}: + return headers + key, value = line.decode().split(":", 1) + headers[key.lower()] = value.strip() + raise ValueError("unbounded fixture headers") + + +def request(host, actor, phase, timeout=.25): + row = {"host": host, "actor": actor, "phase": phase, "begin_ns": time.monotonic_ns(), "ok": False} + try: + # TEST-NET, not a real public API. The production transparent redirect + # must route the named Host through the mock CONNECT upstream. + port_file = Path("/tmp/searchswe-fixture-port") + port = int(port_file.read_text()) if port_file.exists() else 80 + authority = host if port == 80 else f"{host}:{port}" + with socket.create_connection(("198.51.100.42", port), timeout=timeout) as sock: + row["sent_ns"] = time.monotonic_ns() + packet = f"GET /{actor}/{phase}/{row['sent_ns']} HTTP/1.1\r\nHost: {authority}\r\nConnection: close\r\n\r\n" + sock.sendall(packet.encode()) + response = http.client.HTTPResponse(sock) + response.begin() + row["ok"] = response.status == 200 and response.read() == b"OK" + except (OSError, http.client.HTTPException) as error: + row["error_class"] = type(error).__name__ + row["end_ns"] = time.monotonic_ns() + return row diff --git a/scripts/tests/egress_transport/fixture.go b/scripts/tests/egress_transport/fixture.go new file mode 100644 index 0000000..b3c773f --- /dev/null +++ b/scripts/tests/egress_transport/fixture.go @@ -0,0 +1,357 @@ +// Offline transport fixture. CONNECT targets never select arbitrary sockets: +// approved fixture names map only to fixed loopback listeners in this container. +package main + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "encoding/binary" + "encoding/json" + "fmt" + "io" + "log" + "net" + "net/http" + "os" + "strings" + "sync" + "time" +) + +var output sync.Mutex +var eventFile string + +func event(fields map[string]any) { + output.Lock() + defer output.Unlock() + if eventFile != "" { + data, _ := json.Marshal(fields) + file, err := os.OpenFile(eventFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) + if err != nil { + panic(err) + } + defer file.Close() + if _, err := file.Write(append(data, '\n')); err != nil { + panic(err) + } + return + } + _ = json.NewEncoder(os.Stdout).Encode(fields) +} + +type settings struct { + Username string `json:"username"` + Password string `json:"password"` + Cert string `json:"cert"` + Key string `json:"key"` + DohCert string `json:"doh_cert"` + DohKey string `json:"doh_key"` + Events string `json:"events"` + DirectAPI bool `json:"direct_api"` +} + +type observedListener struct { + net.Listener + port string +} + +func (listener observedListener) Accept() (net.Conn, error) { + conn, err := listener.Listener.Accept() + if err == nil { + event(map[string]any{"kind": "tcp_accept", "port": listener.port}) + } + return conn, err +} + +func dnsReply(query []byte, address, ipv6 net.IP) ([]byte, string, uint16, error) { + if len(query) < 17 || binary.BigEndian.Uint16(query[4:6]) != 1 { + return nil, "", 0, fmt.Errorf("invalid fixture question") + } + labels, pos := []string{}, 12 + for pos < len(query) && query[pos] != 0 { + n := int(query[pos]) + pos++ + if n > 63 || pos+n >= len(query) { + return nil, "", 0, fmt.Errorf("invalid fixture label") + } + labels = append(labels, string(query[pos:pos+n])) + pos += n + } + if pos+5 > len(query) { + return nil, "", 0, fmt.Errorf("short fixture question") + } + typ := binary.BigEndian.Uint16(query[pos+1 : pos+3]) + name := strings.Join(labels, ".") + response := append([]byte(nil), query[:pos+5]...) + binary.BigEndian.PutUint16(response[2:4], 0x8180) + for i := 6; i < 12; i++ { + response[i] = 0 + } + if name == "nx.example" { + response[3] |= 3 + return response, name, typ, nil + } + var data []byte + if typ == 1 { + data = address.To4() + } else if typ == 28 { + data = ipv6.To16() + } + if data != nil { + response[7] = 1 + owner := []byte{0xc0, 0x0c} + if name == "alias.example" { + canonical := []byte("\x07allowed\x07example\x00") + response[7] = 2 + response = append(response, 0xc0, 0x0c, 0, 5, 0, 1, 0, 0, 0, 1, 0, byte(len(canonical))) + response = append(response, canonical...) + owner = canonical + } + response = append(response, owner...) + response = append(response, 0, byte(typ), 0, 1, 0, 0, 0, 1, 0, byte(len(data))) + response = append(response, data...) + } + return response, name, typ, nil +} + +func serve(path, role string) { + data, err := os.ReadFile(path) + if err != nil { + panic(err) + } + var cfg settings + if err := json.Unmarshal(data, &cfg); err != nil { + panic(err) + } + eventFile = cfg.Events + if err := os.WriteFile("/tmp/fixture-"+role+".pid", []byte(fmt.Sprint(os.Getpid())), 0600); err != nil { + panic(err) + } + addresses, err := net.InterfaceAddrs() + if err != nil { + panic(err) + } + var address net.IP + for _, item := range addresses { + ip := item.(*net.IPNet).IP + if ip.To4() != nil && !ip.IsLoopback() { + address = ip + } + } + quiet := log.New(io.Discard, "", 0) + apiHandler := func(kind string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + event(map[string]any{"kind": kind, "host": r.Host, "path": r.URL.Path, "protocol": r.Proto}) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"ok":true}`)) + } + } + api := &http.Server{Addr: "127.0.0.1:19443", ErrorLog: quiet, Handler: apiHandler("api")} + doh := &http.Server{Addr: "127.0.0.1:18443", ErrorLog: quiet, Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + query, _ := io.ReadAll(io.LimitReader(r.Body, 65536)) + _, pollution := os.Stat("/tmp/pollute") + answer := append(net.IP(nil), address...) + if pollution == nil { + answer[len(answer)-1] = 254 // unused, same internal subnet; never an external address + } + ipv6 := net.ParseIP("2001:db8::1") + _, pollution6 := os.Stat("/tmp/pollute6") + if pollution6 == nil { + ipv6 = net.ParseIP("2001:db8::bad") + } + reply, name, typ, err := dnsReply(query, answer, ipv6) + if err != nil { + http.Error(w, "invalid fixture query", 400) + return + } + event(map[string]any{"kind": "dns", "name": name, "type": typ, "polluted": pollution == nil, + "polluted_aaaa": pollution6 == nil, "aaaa": ipv6.String()}) + if name == "redirect.example" { + http.Redirect(w, r, "https://blocked.example/dns-query", 307) + return + } + if name == "timeout.example" { + time.Sleep(12 * time.Second) + return + } + w.Header().Set("Content-Type", "application/dns-message") + _, _ = w.Write(reply) + })} + proxy := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + expected := "Basic " + base64.StdEncoding.EncodeToString([]byte(cfg.Username+":"+cfg.Password)) + ok := r.Header.Get("Proxy-Authorization") == expected + event(map[string]any{"kind": "proxy", "target": r.Host, "auth_ok": ok}) + if !ok || r.Method != "CONNECT" { + w.Header().Set("Proxy-Authenticate", `Basic realm="fixture"`) + w.WriteHeader(407) + return + } + target := map[string]string{"allowed.example:443": "127.0.0.1:19443", "allowed.example:8443": "127.0.0.1:19443", "allowed.example:9443": "127.0.0.1:19443", "resolver.example:443": "127.0.0.1:18443", + "wrong-resolver.example:443": "127.0.0.1:18443"}[r.Host] + if target == "" { + w.WriteHeader(403) + return + } + upstream, err := net.DialTimeout("tcp", target, time.Second) + if err != nil { + http.Error(w, "fixture unavailable", 503) + return + } + defer upstream.Close() + client, buffered, err := w.(http.Hijacker).Hijack() + if err != nil { + return + } + defer client.Close() + _, _ = buffered.WriteString("HTTP/1.1 200 Connection established\r\n\r\n") + _ = buffered.Flush() + done := make(chan struct{}) + go func() { _, _ = io.Copy(upstream, buffered); _ = upstream.Close(); close(done) }() + _, _ = io.Copy(client, upstream) + _ = client.Close() + <-done + }) + type endpoint struct { + server *http.Server + secure bool + role string + } + endpoints := []endpoint{{api, true, "api"}, {doh, true, "doh"}, {&http.Server{Addr: ":18080", Handler: proxy, ErrorLog: quiet}, false, "proxy"}, + {&http.Server{Addr: ":18480", Handler: proxy, ErrorLog: quiet}, true, "proxy"}} + if cfg.DirectAPI { + endpoints = append(endpoints, endpoint{&http.Server{Addr: ":9443", Handler: apiHandler("api-direct"), ErrorLog: quiet}, true, "api"}) + } + for _, item := range endpoints { + if role != "all" && role != item.role { + continue + } + listener, err := net.Listen("tcp", item.server.Addr) + if err != nil { + panic(err) + } + _, port, _ := net.SplitHostPort(item.server.Addr) + listener = observedListener{Listener: listener, port: port} + go func(server *http.Server, listener net.Listener, secure bool) { + if secure { + cert, key := cfg.Cert, cfg.Key + if server == doh && cfg.DohCert != "" { + cert, key = cfg.DohCert, cfg.DohKey + } + _ = server.ServeTLS(listener, cert, key) + } else { + _ = server.Serve(listener) + } + }(item.server, listener, item.secure) + } + event(map[string]any{"kind": "ready", "role": role, "pid": os.Getpid()}) + select {} +} + +func client(mode, destination, ca string) error { + pem, err := os.ReadFile(ca) + if err != nil { + return err + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + return fmt.Errorf("invalid fixture CA") + } + transport := &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool}, ForceAttemptHTTP2: mode != "held-h1"} + port, authority := "443", "allowed.example" + if mode == "get-alt-port" { + port, authority = "8443", "allowed.example:8443" + } + if mode == "get-fallback" { + port, authority = "9443", "allowed.example:9443" + } + if mode == "get-ip6" { + transport.DialContext = func(ctx context.Context, _, address string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "tcp6", address) + } + } + if destination != "-" { + transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "tcp", net.JoinHostPort(destination, port)) + } + } + defer transport.CloseIdleConnections() + c := &http.Client{Transport: transport, Timeout: 5 * time.Second} + get := func(path string) error { + r, err := c.Get("https://" + authority + path) + if err != nil { + return err + } + defer r.Body.Close() + body, err := io.ReadAll(r.Body) + if err != nil || r.StatusCode != 200 || string(body) != `{"ok":true}` { + return fmt.Errorf("invalid fixture API response") + } + event(map[string]any{"kind": "response", "protocol": r.Proto, "path": path}) + return nil + } + if err := get("/before"); err != nil { + return err + } + if strings.HasPrefix(mode, "held-") { + event(map[string]any{"kind": "ready"}) + deadline := time.Now().Add(time.Minute) + for time.Now().Before(deadline) { + if _, err := os.Stat("/tmp/release"); err == nil { + _ = get("/after") // Only the origin log decides revocation success. + return nil + } + time.Sleep(50 * time.Millisecond) + } + return fmt.Errorf("fixture release timeout") + } + return nil +} + +func main() { + if strings.HasPrefix(os.Args[1], "delayed-") { + os.Args[1] = strings.TrimPrefix(os.Args[1], "delayed-") + event(map[string]any{"kind": "started"}) + deadline := time.Now().Add(time.Minute) + for { + if _, err := os.Stat("/tmp/begin"); err == nil { + break + } + if time.Now().After(deadline) { + panic("fixture start timeout") + } + time.Sleep(50 * time.Millisecond) + } + } + if os.Args[1] == "serve" { + serve(os.Args[2], "all") + return + } + if strings.HasPrefix(os.Args[1], "serve-") { + serve(os.Args[2], strings.TrimPrefix(os.Args[1], "serve-")) + return + } + if strings.HasPrefix(os.Args[1], "dns-") { + parts := strings.Split(os.Args[1], "-") + resolver := &net.Resolver{PreferGo: true, StrictErrors: true, Dial: func(ctx context.Context, _, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, parts[1], os.Args[2]) + }} + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + addresses, err := resolver.LookupIP(ctx, parts[2], os.Args[3]+".") + event(map[string]any{"kind": "resolved", "ok": err == nil && len(addresses) > 0, "addresses": addresses}) + return + } + if os.Args[1] == "resolve" { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, err := net.DefaultResolver.LookupIPAddr(ctx, os.Args[2]) + event(map[string]any{"kind": "resolved", "ok": err == nil}) + return + } + if err := client(os.Args[1], os.Args[2], os.Args[3]); err != nil { + event(map[string]any{"kind": "client_error", "error": err.Error()}) + os.Exit(1) + } +} diff --git a/scripts/tests/egress_transport/run.py b/scripts/tests/egress_transport/run.py new file mode 100644 index 0000000..01fecca --- /dev/null +++ b/scripts/tests/egress_transport/run.py @@ -0,0 +1,446 @@ +"""Offline TLS/auth/DoH integration. No public API, DNS or published port.""" + +import argparse +import base64 +import hashlib +import json +import os +from pathlib import Path +import runpy +import subprocess +import tempfile +import time +import uuid + + +ROOT = Path(__file__).resolve().parents[3] +SOURCE = Path(__file__).with_name("fixture.go") +helpers = runpy.run_path(str(Path(__file__).parents[1] / "egress_s0/run.py")) +docker, events, wait_ready = (helpers[n] for n in ("docker", "events", "wait_ready")) + + +class GateFailure(RuntimeError): + pass + + +def run(args): + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=False) + identity = "searchswe-transport-" + uuid.uuid4().hex[:12] + label = "searchswe.transport=" + identity + network, mock, image_tag = identity, identity + "-mock", identity + ":ca" + containers, networks = [], [] + created_image = False + private = tempfile.TemporaryDirectory(prefix="searchswe-transport-") + secret = Path(private.name) + username, password = "user-" + uuid.uuid4().hex, "canary-" + uuid.uuid4().hex + needles = [username, password, base64.b64encode(f"{username}:{password}".encode()).decode()] + result = {"status": "infrastructure_error", "identity": identity, "checks": [], + "source_sha256": hashlib.sha256(SOURCE.read_bytes()).hexdigest()} + try: + version = subprocess.check_output([str(args.go.resolve()), "version"], text=True) + if "go1.26.8 linux/amd64" not in version: + raise RuntimeError("Use the pinned Linux amd64 Go 1.26.8 toolchain") + binary = output / "fixture" + subprocess.run([str(args.go.resolve()), "build", "-trimpath", "-buildvcs=false", "-o", str(binary), str(SOURCE)], + env={**os.environ, "CGO_ENABLED": "0", "GOTOOLCHAIN": "local", "GOMAXPROCS": "4", + "GOCACHE": str(ROOT / "jobs/egress-build/gocache")}, check=True, timeout=180) + result["binary_sha256"] = hashlib.sha256(binary.read_bytes()).hexdigest() + base = json.loads(docker("image", "inspect", args.gateway_image).stdout)[0] + python = json.loads(docker("image", "inspect", args.python_image).stdout)[0]["Id"] + result["base_image"] = base["Id"] + def openssl(*arguments): + subprocess.run(["openssl", *arguments], cwd=secret, check=True, + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, timeout=30) + openssl("req", "-x509", "-newkey", "rsa:2048", "-nodes", "-days", "1", "-subj", "/CN=SearchSWE fixture CA", + "-keyout", "ca.key", "-out", "ca.crt") + openssl("req", "-newkey", "rsa:2048", "-nodes", "-subj", "/CN=proxy.example", "-keyout", "server.key", "-out", "server.csr") + (secret / "extensions").write_text("subjectAltName=DNS:proxy.example,DNS:resolver.example,DNS:allowed.example\n") + openssl("x509", "-req", "-in", "server.csr", "-CA", "ca.crt", "-CAkey", "ca.key", "-CAcreateserial", + "-days", "1", "-extfile", "extensions", "-out", "server.crt") + openssl("req", "-x509", "-newkey", "rsa:2048", "-nodes", "-days", "1", "-subj", "/CN=resolver.example", + "-addext", "subjectAltName=DNS:resolver.example", "-keyout", "bad.key", "-out", "bad.crt") + (secret / "ca.crt").chmod(0o644) + context = secret / "image" + context.mkdir() + (context / "ca.crt").write_bytes((secret / "ca.crt").read_bytes()) + (context / "Dockerfile").write_text( + 'ARG BASE\nFROM ${BASE}\nCOPY ca.crt /usr/local/share/ca-certificates/searchswe-fixture.crt\n' + 'RUN python3 -c "from pathlib import Path; p=Path(\'/etc/ssl/certs/ca-certificates.crt\'); ' + 'p.write_bytes(p.read_bytes()+Path(\'/usr/local/share/ca-certificates/searchswe-fixture.crt\').read_bytes())"\n' + f'LABEL searchswe.transport="{identity}"\n') + built = docker("build", "--pull=false", "--network=none", "--build-arg", "BASE=" + args.gateway_image, + "-t", image_tag, str(context), timeout=120, check=False) + (output / "image-build.log").write_text(built.stdout + built.stderr) + if built.returncode: + raise RuntimeError("Fixture CA image build failed; inspect image-build.log") + created_image = True + image = json.loads(docker("image", "inspect", image_tag).stdout)[0] + if image["RootFS"]["Layers"][:len(base["RootFS"]["Layers"])] != base["RootFS"]["Layers"]: + raise RuntimeError("Fixture CA image does not extend the inspected base") + result["fixture_image"] = image["Id"] + docker("network", "create", "--internal", "--label", label, network) + networks.append(network) + mock_settings = secret / "mock.json" + mock_settings.write_text(json.dumps({"username": username, "password": password, + "cert": "/private/server.crt", "key": "/private/server.key"})) + mock_settings.chmod(0o600) + mounts = ["-v", f"{binary}:/fixture:ro", "-v", f"{secret / 'ca.crt'}:/public/ca.crt:ro"] + containers.append(mock) + docker("run", "-d", "--pull", "never", "--name", mock, "--label", label, + "--network", network, "--cap-drop", "ALL", *mounts, "-v", f"{secret}:/private:ro", + "--user", f"{os.getuid()}:{os.getgid()}", "--entrypoint", "/fixture", python, "serve", "/private/mock.json") + wait_ready(mock) + ip = json.loads(docker("inspect", mock).stdout)[0]["NetworkSettings"]["Networks"][network]["IPAddress"] + common = {"upstream_ip": ip, "upstream_port": 18480, "upstream_addr": f"{ip}:18480", + "upstream_host": "proxy.example", "upstream_tls": True, + "auth": {"username": username, "password": password}, "doh_url": "https://resolver.example/dns-query"} + + clients = {} + file_event_sources = set() + def observe(name): + if name in file_event_sources: + text = docker("exec", name, "cat", "/tmp/events.jsonl").stdout + return [json.loads(line) for line in text.splitlines() if line.strip()] + return events(name) + def prepare_client(gateway): + name = identity + "-client-" + uuid.uuid4().hex[:8] + containers.append(name) + docker("run", "-d", "--pull", "never", "--name", name, + "--label", label, "--network", "container:" + gateway, "--cap-drop", "ALL", + "--security-opt", "no-new-privileges:true", *mounts, "--entrypoint", "sleep", python, "infinity") + clients[gateway] = name + + def start(case, overrides=None, trusted_ca=True): + name = identity + "-" + case + settings = {**common, **(overrides or {})} + config = secret / (case + ".json") + config.write_text(json.dumps(settings)) + config.chmod(0o600) + containers.append(name) + docker("run", "-d", "--pull", "never", "--name", name, "--label", label, + "--network", network, "--cap-drop", "ALL", "--cap-add", "NET_ADMIN", "--cap-add", "NET_RAW", + "--cap-add", "DAC_OVERRIDE", "--security-opt", "no-new-privileges:true", "--read-only", + "--tmpfs", "/run/searchswe:mode=0700", "--tmpfs", "/tmp:mode=1777", + "-v", f"{config}:/opt/searchswe/input.json:ro", "--entrypoint", "/opt/egress-sidecar/entrypoint.sh", + image["Id"] if trusted_ca else base["Id"]) + for _ in range(30): + if docker("exec", name, "network-policy", "show", check=False).returncode == 0: + break + time.sleep(.1) + else: + raise RuntimeError("gateway failed startup") + # Provision potentially slow Docker actors before granting a lease. + prepare_client(name) + docker("exec", name, "network-policy", "allow", "allowed.example", "nx.example", "redirect.example", "timeout.example") + return name + + def invoke(gateway, mode="get", destination=None, detached=False): + arguments = ([mode, *destination] if mode.startswith("dns-") else + [mode, destination] if mode == "resolve" else [mode, destination or ip, "/public/ca.crt"]) + if detached: + name = identity + "-client-" + uuid.uuid4().hex[:8] + containers.append(name) + response = docker("run", "-d", "--pull", "never", "--name", name, + "--label", label, "--network", "container:" + gateway, "--cap-drop", "ALL", + "--security-opt", "no-new-privileges:true", *mounts, + "--entrypoint", "/fixture", python, "delayed-" + arguments[0], *arguments[1:], timeout=60) + for _ in range(50): + if any(e.get("kind") == "started" for e in events(name)): + break + time.sleep(.1) + else: + raise RuntimeError("held client did not start") + docker("exec", gateway, "network-policy", "allow", "allowed.example") + docker("exec", name, "touch", "/tmp/begin") + return name, response + docker("exec", gateway, "network-policy", "lease") + name = clients[gateway] + response = docker("exec", name, "/fixture", *arguments, check=False, timeout=20) + return name, response + + def check(case, gateway, *, mode="get", destination=None, positive=False, deny_kinds=(), event_source=mock, + dns_expected=None): + before = len(observe(event_source)) + _, response = invoke(gateway, mode, destination) + docker("exec", gateway, "network-policy", "lease") + observed = observe(event_source)[before:] + has_api = any(e["kind"] == "api" for e in observed) + if positive and (response.returncode or not has_api): + raise RuntimeError("transport positive control failed: " + case) + if any(e["kind"] in deny_kinds for e in observed): + raise GateFailure("forbidden upstream event: " + case) + if dns_expected is not None: + if response.returncode or f'"ok":{str(dns_expected).lower()}' not in response.stdout: + raise RuntimeError("DNS fixture result mismatch: " + case) + if dns_expected and not any(e.get("kind") == "dns" and e.get("name") == destination[1] for e in observed): + raise RuntimeError("DNS positive never reached DoH: " + case) + result["checks"].append({"case": case, "passed": True, "upstream_events": observed, + "client": response.stdout.strip(), "returncode": response.returncode}) + + good = start("good") + check("tls-proxy-auth-api-h2", good, positive=True) + if not any(e.get("protocol") == "HTTP/2.0" for e in events(mock)): + raise RuntimeError("HTTP/2 positive control was not negotiated") + check("approved-private-endpoint-preserves-port", good, mode="get-alt-port", positive=True) + if not any(e.get("target") == "allowed.example:8443" for e in result["checks"][-1]["upstream_events"]): + raise RuntimeError("approved API port was not preserved") + check("docker-dns-doh-api", good, destination="-", positive=True) + if not any(e["kind"] == "dns" for e in result["checks"][-1]["upstream_events"]): + raise RuntimeError("system DNS positive did not traverse DoH") + for entry, server in (("local", "127.0.0.1:1053"), ("explicit", ip + ":53"), ("docker", "127.0.0.11:53")): + for protocol in ("tcp", "udp"): + for family in ("ip4", "ip6"): + docker("exec", good, "network-policy", "allow", "allowed.example") + mode = f"dns-{protocol}-{family}" + case = f"{entry}-{protocol}-{family}" + check(case + "-allowed", good, mode=mode, destination=(server, "allowed.example"), dns_expected=True) + check(case + "-forbidden", good, mode=mode, destination=(server, "blocked.example"), + dns_expected=False, deny_kinds=("proxy", "dns", "api")) + docker("exec", good, "network-policy", "allow", "alias.example") + check("dns-cache-not-a-grant-after-revocation", good, mode="dns-udp-ip6", destination=("127.0.0.11:53", "allowed.example"), + dns_expected=False, deny_kinds=("proxy", "dns", "api")) + docker("exec", good, "network-policy", "deny-all") + check("no-network-stops-docker-dns", good, mode="dns-udp-ip4", destination=("127.0.0.11:53", "allowed.example"), + dns_expected=False, deny_kinds=("proxy", "dns", "api")) + docker("exec", good, "network-policy", "allow", "allowed.example") + check("dns-restored-needs-fresh-doh", good, mode="dns-udp-ip4", destination=("127.0.0.11:53", "allowed.example"), dns_expected=True) + clear = start("clear", {"upstream_tls": False, "upstream_port": 18080, "upstream_addr": f"{ip}:18080"}) + check("http-proxy-auth-api", clear, positive=True) + # Wrong A answers on a reachable route must not change the checked + # hostname passed to the proxy. No public DNS/CDN address is embedded. + docker("exec", mock, "touch", "/tmp/pollute") + polluted = start("polluted") + check("polluted-a-still-reaches-approved-api", polluted, destination="-", positive=True) + if not any(e.get("kind") == "dns" and e.get("polluted") for e in result["checks"][-1]["upstream_events"]): + raise RuntimeError("polluted DNS positive did not reach its fixture") + docker("exec", mock, "rm", "/tmp/pollute") + docker("exec", mock, "touch", "/tmp/pollute6") + docker("exec", polluted, "network-policy", "allow", "allowed.example") + check("polluted-aaaa-exact-answer", polluted, mode="dns-udp-ip6", + destination=("127.0.0.1:1053", "allowed.example"), dns_expected=True) + if "2001:db8::bad" not in result["checks"][-1]["client"] or not any( + e.get("type") == 28 and e.get("polluted_aaaa") for e in result["checks"][-1]["upstream_events"]): + raise RuntimeError("polluted AAAA answer was not observed at both DNS ends") + docker("exec", polluted, "network-policy", "allow", "allowed.example") + check("polluted-aaaa-with-valid-a-api", polluted, destination="-", positive=True) + docker("exec", polluted, "network-policy", "allow", "allowed.example") + check("polluted-aaaa-only-no-ipv6-egress", polluted, mode="get-ip6", destination="-", deny_kinds=("api",)) + if result["checks"][-1]["returncode"] == 0 or not any( + e.get("type") == 28 and e.get("polluted_aaaa") for e in result["checks"][-1]["upstream_events"]): + raise RuntimeError("AAAA-only refusal lacked the required polluted resolution") + check("polluted-aaaa-does-not-grant-forbidden-name", polluted, mode="dns-udp-ip6", + destination=("127.0.0.1:1053", "blocked.example"), dns_expected=False, deny_kinds=("proxy", "dns", "api")) + docker("exec", mock, "rm", "/tmp/pollute6") + for case, overrides, trusted, deny in [ + ("bad-auth", {"auth": {"username": username, "password": "incorrect"}}, True, ("api", "dns")), + ("bad-proxy-name", {"upstream_host": "wrong-proxy.example"}, True, ("proxy", "api", "dns")), + ("bad-proxy-ca", {}, False, ("proxy", "api", "dns")), + ("bad-doh-name", {"doh_url": "https://wrong-resolver.example/dns-query"}, True, ("dns", "api")), + ]: + gateway = start(case, overrides, trusted) + check(case, gateway, destination="-" if case == "bad-doh-name" else None, deny_kinds=deny) + observed = result["checks"][-1]["upstream_events"] + if case == "bad-auth" and not any(e.get("kind") == "proxy" and e.get("auth_ok") is False for e in observed): + raise RuntimeError("wrong-auth negative never reached authentication") + if case.startswith("bad-proxy") and not any(e.get("kind") == "tcp_accept" and e.get("port") == "18480" for e in observed): + raise RuntimeError("certificate negative never reached the TLS proxy") + if case == "bad-doh-name" and not any(e.get("kind") == "tcp_accept" and e.get("port") == "18443" for e in observed): + raise RuntimeError("DoH certificate negative never reached the TLS origin") + bad_mock = identity + "-bad-doh-mock" + bad_settings = json.loads(mock_settings.read_text()) + bad_settings.update(doh_cert="/private/bad.crt", doh_key="/private/bad.key") + (secret / "bad-mock.json").write_text(json.dumps(bad_settings)) + (secret / "bad-mock.json").chmod(0o600) + containers.append(bad_mock) + docker("run", "-d", "--pull", "never", "--name", bad_mock, "--label", label, + "--network", network, "--cap-drop", "ALL", *mounts, "-v", f"{secret}:/private:ro", + "--user", f"{os.getuid()}:{os.getgid()}", "--entrypoint", "/fixture", python, "serve", "/private/bad-mock.json") + wait_ready(bad_mock) + bad_ip = json.loads(docker("inspect", bad_mock).stdout)[0]["NetworkSettings"]["Networks"][network]["IPAddress"] + bad_ca = start("bad-doh-ca", {"upstream_ip": bad_ip, "upstream_addr": f"{bad_ip}:18480"}) + check("bad-doh-ca", bad_ca, destination="-", deny_kinds=("api", "dns"), event_source=bad_mock) + if not any(e.get("kind") == "tcp_accept" and e.get("port") == "18443" for e in result["checks"][-1]["upstream_events"]): + raise RuntimeError("untrusted DoH CA negative never reached the TLS origin") + docker("exec", good, "network-policy", "allow", "alias.example") + check("cname-resolution", good, mode="resolve", destination="alias.example", deny_kinds=("api",)) + if '"ok":true' not in result["checks"][-1]["client"]: + raise RuntimeError("CNAME positive control failed") + check("cname-is-not-an-api-grant", good, deny_kinds=("api", "proxy")) + for name in ("nx.example", "redirect.example", "timeout.example"): + docker("exec", good, "network-policy", "allow", name) + check(name, good, mode="resolve", destination=name, deny_kinds=("api",)) + observed = result["checks"][-1]["upstream_events"] + if not any(e.get("kind") == "dns" and e.get("name") == name for e in observed): + raise RuntimeError("DNS fault did not reach its fixture") + if any(e.get("target") == "blocked.example:443" for e in observed): + raise GateFailure("DoH redirect reached a different origin") + if '"ok":false' not in result["checks"][-1]["client"]: + raise GateFailure("DNS fault unexpectedly resolved") + for mode in ("held-h1", "held-h2"): + docker("exec", good, "network-policy", "allow", "allowed.example") + held, _ = invoke(good, mode, detached=True) + wait_ready(held) + docker("exec", good, "network-policy", "deny-all") + before = len(events(mock)) + docker("exec", held, "touch", "/tmp/release") + if docker("wait", held, timeout=15).stdout.strip() != "0": + raise RuntimeError("held client failed") + observed = events(mock)[before:] + if observed: + raise GateFailure("old TLS/H2 pool reached upstream after revocation") + result["checks"].append({"case": mode + "-revoked", "passed": True, "upstream_events": observed}) + + # Separate processes in one dedicated internal-network container let us + # kill proxy/DoH independently while a reachable direct API witness stays + # alive. A client error alone is never the fallback-denial oracle. + fault_mock = identity + "-fault-mock" + fault_settings = {**json.loads(mock_settings.read_text()), "events": "/tmp/events.jsonl", "direct_api": True} + (secret / "fault-mock.json").write_text(json.dumps(fault_settings)) + (secret / "fault-mock.json").chmod(0o600) + containers.append(fault_mock) + docker("run", "-d", "--init", "--pull", "never", "--name", fault_mock, "--label", label, + "--network", network, "--cap-drop", "ALL", *mounts, "-v", f"{secret}:/private:ro", + "--entrypoint", "sleep", python, "infinity") + file_event_sources.add(fault_mock) + docker("exec", fault_mock, "touch", "/tmp/events.jsonl") + def role_start(role): + before = len(observe(fault_mock)) + docker("exec", "-d", fault_mock, "/fixture", "serve-"+role, "/private/fault-mock.json") + for _ in range(30): + ready = [e for e in observe(fault_mock)[before:] if e.get("kind") == "ready" and e.get("role") == role] + if ready: + return ready[-1]["pid"] + time.sleep(.1) + raise RuntimeError("fault fixture role failed startup: "+role) + def role_kill(role): + code = """import os,signal,sys,time +from pathlib import Path +role=sys.argv[1];pid=int(Path('/tmp/fixture-'+role+'.pid').read_text()) +assert Path(f'/proc/{pid}/cmdline').read_bytes().split(b'\\0')[1]==('serve-'+role).encode() +os.kill(pid,signal.SIGKILL) +for _ in range(100): + p=Path(f'/proc/{pid}/stat') + if not p.exists() or p.read_text().split()[2]=='Z': break + time.sleep(.02) +else: raise RuntimeError('fixture process did not exit') +print(pid) +""" + return int(docker("exec", fault_mock, "python", "-c", code, role).stdout) + pids = {role: role_start(role) for role in ("api", "doh", "proxy")} + fault_ip = json.loads(docker("inspect", fault_mock).stdout)[0]["NetworkSettings"]["Networks"][network]["IPAddress"] + witness = identity + "-direct-witness" + containers.append(witness) + docker("run", "-d", "--pull", "never", "--name", witness, "--label", label, "--network", network, + "--cap-drop", "ALL", *mounts, "--entrypoint", "sleep", python, "infinity") + def direct_control(case): + before = len(observe(fault_mock)) + response = docker("exec", witness, "/fixture", "get-fallback", fault_ip, "/public/ca.crt", check=False) + received = observe(fault_mock)[before:] + if response.returncode or not any(e.get("kind") == "api-direct" for e in received): + raise RuntimeError("direct fallback witness is not reachable") + result["checks"].append({"case": case, "passed": True, "upstream_events": received}) + direct_control("fallback-witness-positive") + fault = start("fault", {"upstream_ip": fault_ip, "upstream_addr": f"{fault_ip}:18480"}) + check("proxy-process-before-kill", fault, mode="get-fallback", destination=fault_ip, positive=True, + deny_kinds=("api-direct",), event_source=fault_mock) + if role_kill("proxy") != pids["proxy"]: + raise RuntimeError("proxy PID changed unexpectedly") + check("proxy-sigkill-no-direct-fallback", fault, mode="get-fallback", destination=fault_ip, + deny_kinds=("api", "api-direct", "dns", "proxy"), event_source=fault_mock) + direct_control("fallback-witness-alive-during-proxy-death") + docker("exec", fault, "network-policy", "deny-all") + new_pid = role_start("proxy") + if new_pid == pids["proxy"]: + raise RuntimeError("proxy restart did not establish a new process") + result.setdefault("process_faults", []).append({"role": "proxy", "signal": "SIGKILL", "old_pid": pids["proxy"], "new_pid": new_pid}) + check("proxy-restart-cannot-undo-deny-all", fault, mode="get-fallback", destination=fault_ip, + deny_kinds=("api", "api-direct", "proxy"), event_source=fault_mock) + docker("exec", fault, "network-policy", "allow", "nx.example") + check("proxy-restart-cannot-restore-old-host", fault, mode="get-fallback", destination=fault_ip, + deny_kinds=("api", "api-direct", "proxy"), event_source=fault_mock) + docker("exec", fault, "network-policy", "allow", "allowed.example") + check("proxy-restored-explicit-current-grant", fault, mode="get-fallback", destination="-", positive=True, + deny_kinds=("api-direct",), event_source=fault_mock) + if role_kill("doh") != pids["doh"]: + raise RuntimeError("DoH PID changed unexpectedly") + docker("exec", fault, "network-policy", "allow", "allowed.example") + check("resolver-sigkill-no-fallback", fault, mode="dns-udp-ip4", destination=("127.0.0.1:1053", "allowed.example"), + dns_expected=False, deny_kinds=("api", "api-direct", "dns"), event_source=fault_mock) + if not any(e.get("target") == "resolver.example:443" for e in result["checks"][-1]["upstream_events"]): + raise RuntimeError("failed DNS request never reached its live proxy") + if any(e.get("kind") == "proxy" and e.get("target") != "resolver.example:443" + for e in result["checks"][-1]["upstream_events"]): + raise GateFailure("resolver failure attempted a different upstream target") + check("resolver-dead-proxy-api-still-live", fault, mode="get-fallback", destination=fault_ip, positive=True, + deny_kinds=("api-direct",), event_source=fault_mock) + docker("exec", fault, "network-policy", "deny-all") + new_pid = role_start("doh") + if new_pid == pids["doh"]: + raise RuntimeError("DoH restart did not establish a new process") + result["process_faults"].append({"role": "doh", "signal": "SIGKILL", "old_pid": pids["doh"], "new_pid": new_pid}) + check("resolver-restart-cannot-undo-deny-all", fault, mode="dns-udp-ip4", destination=("127.0.0.1:1053", "allowed.example"), + dns_expected=False, deny_kinds=("proxy", "dns", "api", "api-direct"), event_source=fault_mock) + docker("exec", fault, "network-policy", "allow", "nx.example") + check("resolver-restart-cannot-restore-old-host", fault, mode="dns-udp-ip4", destination=("127.0.0.1:1053", "allowed.example"), + dns_expected=False, deny_kinds=("proxy", "dns", "api", "api-direct"), event_source=fault_mock) + docker("exec", fault, "network-policy", "allow", "allowed.example") + check("resolver-restored-explicit-current-grant", fault, mode="get-fallback", destination="-", positive=True, + deny_kinds=("api-direct",), event_source=fault_mock) + (output / "fault-events.json").write_text(json.dumps(observe(fault_mock), indent=2)+"\n") + result["status"] = "passed" + except GateFailure as error: + result.update(status="security_gate_failed", error=str(error)) + except Exception as error: + result["error"] = str(error) + finally: + cleanup = [] + for name in reversed(containers): + inspected = docker("container", "inspect", name, check=False) + if inspected.returncode: + continue + info = json.loads(inspected.stdout)[0] + if info["Config"].get("Labels", {}).get("searchswe.transport") != identity: + cleanup.append("container ownership mismatch") + continue + logs = docker("logs", name, check=False) + evidence = logs.stdout + logs.stderr + if any(needle in inspected.stdout + evidence for needle in needles): + result["status"] = "security_gate_failed" + result["error"] = "credential canary in container metadata/logs (withheld)" + else: + (output / (name.removeprefix(identity + "-") + ".log")).write_text(evidence) + if docker("rm", "-f", name, check=False).returncode: + cleanup.append("container cleanup failed") + for name in networks: + info = json.loads(docker("network", "inspect", name).stdout)[0] + if info["Labels"].get("searchswe.transport") != identity or info.get("Containers"): + cleanup.append("network ownership/membership mismatch") + elif docker("network", "rm", name, check=False).returncode: + cleanup.append("network cleanup failed") + if created_image: + info = json.loads(docker("image", "inspect", image_tag).stdout)[0] + if info["Config"]["Labels"].get("searchswe.transport") != identity: + cleanup.append("image ownership mismatch") + elif docker("image", "rm", image_tag, check=False).returncode: + cleanup.append("image cleanup failed") + private.cleanup() + result["cleanup_errors"] = cleanup + if cleanup: + result["status"] = "cleanup_error" + serialized = json.dumps(result, indent=2) + if any(needle in serialized for needle in needles): + serialized = json.dumps({"status": "security_gate_failed", "error": "credential canary in result (withheld)"}) + result["status"] = "security_gate_failed" + (output / "result.json").write_text(serialized + "\n") + print(json.dumps({"status": result["status"], "error": result.get("error"), "output": str(output)})) + return 0 if result["status"] == "passed" else 1 if result["status"] == "security_gate_failed" else 2 + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--gateway-image", required=True) + parser.add_argument("--python-image", default="python:3.13-slim") + parser.add_argument("--go", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + raise SystemExit(run(parser.parse_args())) diff --git a/scripts/tests/test_docker_dns.py b/scripts/tests/test_docker_dns.py index 0a3a254..9dfcc1b 100644 --- a/scripts/tests/test_docker_dns.py +++ b/scripts/tests/test_docker_dns.py @@ -1,8 +1,6 @@ -import json -import shutil import unittest -from scripts.docker_dns import create_overlay, parse_servers +from scripts.docker_dns import parse_servers class DockerDNSTests(unittest.TestCase): @@ -14,15 +12,3 @@ def test_reject_invalid_servers(self): for value in ('', '127.0.0.53', '::1', '1.1.1.1;echo bad', 'example.org', '0.0.0.0', '224.0.0.1'): with self.subTest(value=value), self.assertRaises(ValueError): parse_servers(value) - - def test_overlay_only_changes_sidecar(self): - path = create_overlay(['198.18.254.30']) - try: - obj = json.loads(path.read_text()) - self.assertEqual(list(obj['services']), ['harbor-docker-egress-control-sidecar']) - sidecar = obj['services']['harbor-docker-egress-control-sidecar'] - self.assertEqual(sidecar['dns'], ['198.18.254.30']) - self.assertTrue(all(v['read_only'] for v in sidecar['volumes'])) - self.assertTrue((path.parent/'network-policy-original').is_file()) - finally: - shutil.rmtree(path.parent) diff --git a/scripts/tests/test_egress_adapter.py b/scripts/tests/test_egress_adapter.py new file mode 100644 index 0000000..5446d96 --- /dev/null +++ b/scripts/tests/test_egress_adapter.py @@ -0,0 +1,196 @@ +"""Host lifecycle failure tests; these do not substitute for Docker gates.""" + +from pathlib import Path +import json +import subprocess +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import AsyncMock, Mock, patch + +from scripts.harbor_environments import PhaseScopedDocker, DockerEnvironment, gateway_image_info +from scripts.egress import ownership + + +class AdapterCleanupTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.root = tempfile.TemporaryDirectory() + self.addCleanup(self.root.cleanup) + self.env = object.__new__(PhaseScopedDocker) + self.env._private = tempfile.TemporaryDirectory() + self.private = Path(self.env._private.name) + self.env._instance = "fixture" + self.env._owner_lock = None + self.env._daemon_id = "fixture-daemon" + self.env._compose_started = True + self.env._rendered_document = {"name": "fixture"} + self.env._check_resource_ownership = Mock() + self.env.trial_paths = SimpleNamespace(trial_dir=Path(self.root.name) / "missing" / "trial") + self.env._force_closed = AsyncMock() + self.env._stop_heartbeat = AsyncMock() + self.env._require_daemon = AsyncMock() + self.env.prepare_logs_for_host = AsyncMock() + self.env._run_docker_compose_command = AsyncMock(return_value=SimpleNamespace(stdout="safe-event\n")) + for name in ("mounts", "resources", "env", "egress_control_services"): + setattr(self.env, f"_cleanup_{name}_compose_file", Mock()) + + async def test_missing_audit_parent_is_created_and_resources_are_removed(self): + await self.env.stop(delete=True) + self.assertFalse(self.private.exists()) + self.assertEqual((self.env.trial_paths.trial_dir / "egress/fixture.log").read_text(), "safe-event\n") + self.env._run_docker_compose_command.assert_any_await(["down", "--remove-orphans", "--volumes"], timeout_sec=60) + + async def test_audit_write_failure_does_not_skip_down(self): + with patch.object(Path, "mkdir", side_effect=OSError("fixture full disk")): + with self.assertRaisesRegex(RuntimeError, "audit collection failed"): + await self.env.stop(delete=True) + self.assertFalse(self.private.exists()) + self.env._run_docker_compose_command.assert_any_await(["down", "--remove-orphans", "--volumes"], timeout_sec=60) + + async def test_failed_prelaunch_never_runs_down_against_foreign_resources(self): + self.env._compose_started = False + await self.env.stop(delete=True) + self.env._run_docker_compose_command.assert_not_awaited() + self.env._force_closed.assert_not_awaited() + self.assertFalse(self.private.exists()) + + async def test_foreign_resource_blocks_down_and_preserves_manifest(self): + self.env._check_resource_ownership.side_effect = ValueError("not owned") + with self.assertRaisesRegex(ValueError, "not owned"): + await self.env.stop(delete=True) + self.env._run_docker_compose_command.assert_not_awaited() + self.assertTrue(self.private.exists()) + self.env._stop_heartbeat.assert_awaited_once() + self.env._private.cleanup() + + async def test_named_volume_ownership_collision(self): + doc = {"name": "fixture", "volumes": {"data": {"name": "fixture_data"}}} + response = SimpleNamespace(returncode=0, stdout='[{"Labels":{}}]', stderr="") + with patch("scripts.harbor_environments.subprocess.run", return_value=response), \ + patch("scripts.harbor_environments.docker_json", return_value="fixture-daemon"): + with self.assertRaisesRegex(ValueError, "not owned"): + PhaseScopedDocker._check_resource_ownership(self.env, doc) + self.env._private.cleanup() + + async def test_context_overrides_host_when_rejecting_remote_daemon(self): + with patch.dict("os.environ", {"DOCKER_CONTEXT": "remote", "DOCKER_HOST": "unix:///local.sock"}), \ + patch("scripts.harbor_environments.docker_json", return_value=[{"Endpoints": {"docker": {"Host": "ssh://remote"}}}]): + with self.assertRaisesRegex(ValueError, "local Unix-socket"): + self.env._preflight_gateway() + self.env._private.cleanup() + + async def test_missing_volume_and_network_are_not_foreign_resources(self): + doc = {"name": "fixture", "volumes": {"data": {"name": "fixture_data"}}, + "networks": {"default": {"name": "fixture_default"}}} + responses = [SimpleNamespace(returncode=1, stdout="[]", stderr=error) for error in ( + "Error response from daemon: get fixture_data: no such volume", + "Error response from daemon: network fixture_default not found")] + responses.append(SimpleNamespace(returncode=0, stdout="", stderr="")) + with patch("scripts.harbor_environments.subprocess.run", side_effect=responses), \ + patch("scripts.harbor_environments.docker_json", return_value="fixture-daemon"): + PhaseScopedDocker._check_resource_ownership(self.env, doc) + self.env._private.cleanup() + + async def test_daemon_change_refuses_resource_mutations(self): + with patch("scripts.harbor_environments.docker_json", return_value="different-daemon"), \ + patch("scripts.harbor_environments.subprocess.run") as command: + with self.assertRaisesRegex(ValueError, "identity changed"): + PhaseScopedDocker._check_resource_ownership(self.env, {"name": "fixture"}) + command.assert_not_called() + self.env._private.cleanup() + + async def test_different_daemon_kernel_is_rejected_before_resources(self): + responses = [[{"Endpoints": {"docker": {"Host": "unix:///fixture.sock"}}}], + {"OSType": "linux", "Architecture": "amd64", "KernelVersion": "different-kernel"}] + with patch("scripts.harbor_environments.docker_json", side_effect=responses), \ + patch("scripts.harbor_environments.ProjectLock") as lock: + with self.assertRaisesRegex(ValueError, "same native"): + self.env._preflight_gateway() + lock.assert_not_called() + self.env._private.cleanup() + + async def test_wrong_daemon_is_not_sent_a_policy_or_kill(self): + self.env._require_daemon.side_effect = RuntimeError("different daemon") + with patch("scripts.harbor_environments.asyncio.sleep", new_callable=AsyncMock) as wait: + with self.assertRaisesRegex(RuntimeError, "cleanup unverified"): + await PhaseScopedDocker._force_closed(self.env) + self.env._run_docker_compose_command.assert_not_awaited() + wait.assert_awaited_once_with(31) + self.env._private.cleanup() + + async def test_successful_deny_all_also_fences_delayed_remote_execs(self): + await PhaseScopedDocker._force_closed(self.env) + self.env._run_docker_compose_command.assert_any_await( + ["kill", "harbor-docker-egress-control-sidecar"], timeout_sec=30) + self.assertIsInstance(self.env._heartbeat_error, RuntimeError) + self.env._private.cleanup() + + async def test_retained_volumes_keep_manifest_but_not_proxy_secret(self): + self.env._private.cleanup() + with patch.object(ownership, "owner_root", return_value=Path(self.root.name)): + managed = ownership.PrivateDirectory("fixture", "fixture", "image") + self.env._private = managed + directory = Path(managed.name) + (directory / "input.json").write_text("fixture-only-private-settings") + self.env._rendered_document["volumes"] = {"data": {"name": "fixture_data"}} + await self.env.stop(delete=False) + self.assertTrue((directory / "owner.json").is_file()) + self.assertFalse((directory / "input.json").exists()) + managed.cleanup() + + async def test_gateway_bootstrap_then_baseline_then_task_start(self): + self.env._heartbeat = Mock() + self.env._gateway_image = "fixture" + self.env._engine_paths = () + self.env.egress = SimpleNamespace(path=Path("/private/fixture.json"), auth_path=None) + self.env._network_policy = Mock() + calls = [] + async def compose(command, *args, **kwargs): + calls.append(command) + return SimpleNamespace(stdout=json.dumps({"name": "fixture", "volumes": {}})) + async def baseline(policy): + self.assertIs(policy, self.env.network_policy) + calls.append("baseline") + self.env.set_network_policy = AsyncMock(side_effect=baseline) + with patch.object(DockerEnvironment, "_run_docker_compose_command", side_effect=compose), \ + patch("scripts.harbor_environments.validate_final"): + await PhaseScopedDocker._run_docker_compose_command(self.env, ["up", "--detach", "--wait"]) + self.assertEqual(calls[1:], [ + ["up", "--detach", "--wait", "--no-deps", "harbor-docker-egress-control-sidecar"], + "baseline", ["up", "--detach", "--wait"]]) + self.env._private.cleanup() + + +class GatewayImagePullTests(unittest.TestCase): + def test_cached_image_needs_no_registry(self): + with patch("scripts.harbor_environments.docker_json", return_value=[{"Id": "cached"}]), \ + patch("scripts.harbor_environments.subprocess.run") as run: + self.assertEqual(gateway_image_info("fixture:1"), {"Id": "cached"}) + run.assert_not_called() + + def test_missing_image_is_pulled_then_inspected(self): + missing = subprocess.CalledProcessError(1, "inspect", stderr="Error: No such image: fixture:1") + with patch("scripts.harbor_environments.docker_json", side_effect=[missing, [{"Id": "pulled"}]]) as inspect, \ + patch("scripts.harbor_environments.subprocess.run") as run: + self.assertEqual(gateway_image_info("fixture:1"), {"Id": "pulled"}) + self.assertEqual(inspect.call_count, 2) + run.assert_called_once_with(["docker", "pull", "--platform", "linux/amd64", "fixture:1"], + check=True, capture_output=True, text=True, timeout=300) + + def test_daemon_error_does_not_attempt_pull(self): + error = subprocess.CalledProcessError(1, "inspect", stderr="permission denied") + with patch("scripts.harbor_environments.docker_json", side_effect=error), \ + patch("scripts.harbor_environments.subprocess.run") as run: + with self.assertRaises(subprocess.CalledProcessError): + gateway_image_info("fixture:1") + run.assert_not_called() + + def test_failed_or_timed_out_pull_stops_startup(self): + missing = subprocess.CalledProcessError(1, "inspect", stderr="No such image: fixture:1") + for error in (subprocess.CalledProcessError(1, "pull"), subprocess.TimeoutExpired("pull", 300)): + with self.subTest(error=type(error).__name__), \ + patch("scripts.harbor_environments.docker_json", side_effect=missing) as inspect, \ + patch("scripts.harbor_environments.subprocess.run", side_effect=error): + with self.assertRaisesRegex(RuntimeError, "Could not pull egress gateway"): + gateway_image_info("fixture:1") + inspect.assert_called_once() diff --git a/scripts/tests/test_egress_build.py b/scripts/tests/test_egress_build.py new file mode 100644 index 0000000..e97701b --- /dev/null +++ b/scripts/tests/test_egress_build.py @@ -0,0 +1,85 @@ +import hashlib +import json +from pathlib import Path +import tempfile +import unittest + +from scripts.build_egress_gateway import stage_archives +from scripts.egress.build_audit import collect_notices, json_stream, runtime_inventory + + +class RuntimeBuildTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.cache = self.root / "cache" + self.cache.mkdir() + self.payload = b"fixture-only-archive" + self.lock = {"archives": {"fixture.apk": hashlib.sha256(self.payload).hexdigest()}} + + def test_missing_archive_does_not_silently_download(self): + with self.assertRaisesRegex(ValueError, "--download explicitly"): + stage_archives(self.lock, self.cache, self.root / "context") + + def test_corrupt_cache_is_rejected(self): + (self.cache / "fixture.apk").write_bytes(b"corrupt") + with self.assertRaisesRegex(ValueError, "checksum mismatch"): + stage_archives(self.lock, self.cache, self.root / "context") + + def test_valid_cache_is_copied_without_mutation(self): + (self.cache / "fixture.apk").write_bytes(self.payload) + stage_archives(self.lock, self.cache, self.root / "context") + self.assertEqual((self.root / "context/fixture.apk").read_bytes(), self.payload) + self.assertEqual((self.cache / "fixture.apk").read_bytes(), self.payload) + + def test_archive_path_traversal_is_rejected(self): + self.lock["archives"] = {"../fixture.apk": "invalid"} + with self.assertRaisesRegex(ValueError, "filename"): + stage_archives(self.lock, self.cache, self.root / "context") + + +class BuildAuditTests(unittest.TestCase): + def test_go_json_stream(self): + self.assertEqual(list(json_stream(' {"Path":"a"}\n{"Path":"b"} ')), [{"Path": "a"}, {"Path": "b"}]) + with self.assertRaises(json.JSONDecodeError): + list(json_stream('{"Path":')) + + def test_only_linked_modules_but_nested_notices_are_retained(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + for name in ("main", "dependency", "go"): + directory = root / name + directory.mkdir() + (directory / "LICENSE").write_text("fixture license\n") + (root / "dependency/nested").mkdir() + (root / "dependency/nested/NOTICE.txt").write_text("nested notice\n") + info = {"Main": {"Path": "example/main"}, "GoVersion": "fixture-go", "Deps": [{"Path": "example/dep", "Version": "v1"}]} + modules = [{"Path": "example/main", "Dir": str(root / "main")}, + {"Path": "example/dep", "Replace": {"Dir": str(root / "dependency")}}, + {"Path": "example/not-linked", "Dir": "/absent"}] + manifest = collect_notices(info, modules, root / "go", root / "notices") + self.assertEqual(len(manifest["modules"]), 3) + self.assertEqual(sum(len(m["notices"]) for m in manifest["modules"]), 4) + self.assertTrue((root / "notices/index.json").is_file()) + + def test_missing_license_and_symlink_are_rejected(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + module = root / "module" + module.mkdir() + info = {"Main": {"Path": "example/main"}, "GoVersion": "fixture-go"} + modules = [{"Path": "example/main", "Dir": str(module)}] + with self.assertRaisesRegex(ValueError, "no collected license"): + collect_notices(info, modules, root, root / "attempt1") + (module / "LICENSE").symlink_to(root / "somewhere") + with self.assertRaisesRegex(ValueError, "notice file"): + collect_notices(info, modules, root, root / "attempt2") + + def test_runtime_inventory_requires_declared_license_and_origin(self): + fields = "P:fixture\nV:1-r0\nL:MIT\no:fixture\nc:commit\nU:https://example.invalid\n" + row = runtime_inventory(fields)["packages"][0] + self.assertEqual(row["license_declared"], "MIT") + self.assertEqual(row["aports_commit"], "commit") + with self.assertRaisesRegex(ValueError, "Incomplete"): + runtime_inventory("P:fixture\nV:1-r0\n") diff --git a/scripts/tests/test_egress_cleanup.py b/scripts/tests/test_egress_cleanup.py new file mode 100644 index 0000000..378fdc9 --- /dev/null +++ b/scripts/tests/test_egress_cleanup.py @@ -0,0 +1,146 @@ +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from scripts import egress_cleanup +from scripts.egress import ownership + + +class OrphanCleanupTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + root = patch.object(ownership, "owner_root", return_value=Path(self.temp.name)) + root.start() + self.addCleanup(root.stop) + self.private = ownership.PrivateDirectory("fixture", "project", "image", daemon_id="fixture-daemon") + + def test_dry_run_does_not_remove_resources_or_private_files(self): + with patch.object(egress_cleanup, "inventory", return_value=[]), patch.object(egress_cleanup, "docker") as docker: + result = egress_cleanup.recover(self.private.name) + self.assertFalse(result["removed"]) + docker.assert_not_called() + self.assertTrue(Path(self.private.name).exists()) + + def test_live_project_owner_prevents_recovery(self): + lock = ownership.ProjectLock("project") + try: + with self.assertRaises(BlockingIOError): + egress_cleanup.recover(self.private.name, remove=True) + finally: + lock.close() + + def test_context_overrides_host_and_remote_cleanup_is_refused(self): + context = [{"Endpoints": {"docker": {"Host": "ssh://remote"}}}] + with patch.dict("os.environ", {"DOCKER_CONTEXT": "remote", "DOCKER_HOST": "unix:///local.sock"}), \ + patch.object(egress_cleanup, "docker", return_value=json.dumps(context)) as docker: + with self.assertRaisesRegex(ValueError, "local Unix-socket"): + egress_cleanup.inventory(self.private.manifest) + self.assertEqual(docker.call_count, 1) + + def test_gateway_removed_first_and_cleanup_is_verified(self): + resources = [{"kind": kind, "id": name, "gateway": gateway} for kind, name, gateway in + [("container", "task", False), ("network", "net", False), + ("container", "gateway", True), ("volume", "vol", False)]] + def remove(*arguments): + if arguments[:2] == ("container", "inspect"): + return json.dumps([{"State": {"Running": True}}]) + if arguments[0] == "logs": + return '{"event":"lease_expired","instance":"fixture"}\n' + if arguments[:2] == ("container", "kill"): + return "gateway" + resources[:] = [r for r in resources if r["id"] != arguments[-1]] + with patch.object(egress_cleanup, "inventory", side_effect=lambda _: resources[:]), \ + patch.object(egress_cleanup, "docker", side_effect=remove) as docker: + result = egress_cleanup.recover(self.private.name, remove=True) + self.assertTrue(result["removed"]) + calls = [call.args for call in docker.call_args_list] + self.assertEqual(calls[:4], [("container", "inspect", "gateway"), ("container", "kill", "gateway"), + ("logs", "gateway"), ("container", "rm", "-f", "gateway")]) + audit = Path(result["audit_paths"][0]) + self.assertIn("lease_expired", audit.read_text()) + self.assertEqual(audit.stat().st_mode & 0o777, 0o600) + self.assertFalse(Path(self.private.name).exists()) + + def test_audit_failure_preserves_stopped_gateway_and_manifest(self): + resource = {"kind": "container", "id": "gateway", "gateway": True} + replies = [json.dumps([{"State": {"Running": True}}]), "gateway", '{"event":"closed"}\n'] + with patch.object(egress_cleanup, "inventory", return_value=[resource]), \ + patch.object(egress_cleanup, "docker", side_effect=replies) as command, \ + patch.object(egress_cleanup.tempfile, "mkstemp", side_effect=OSError("fixture disk error")): + with self.assertRaises(OSError): + egress_cleanup.recover(self.private.name, remove=True) + self.assertEqual(command.call_args_list[1].args, ("container", "kill", "gateway")) + self.assertFalse(any("rm" in call.args for call in command.call_args_list)) + self.assertTrue((Path(self.private.name) / "owner.json").exists()) + + def test_docker_failure_preserves_recovery_manifest(self): + resources = [{"kind": "container", "id": "gateway", "gateway": True}] + with patch.object(egress_cleanup, "inventory", return_value=resources), \ + patch.object(egress_cleanup, "docker", side_effect=RuntimeError("fixture offline")): + with self.assertRaises(RuntimeError): + egress_cleanup.recover(self.private.name, remove=True) + self.assertTrue((Path(self.private.name) / "owner.json").is_file()) + + def test_foreign_project_label_is_rejected_before_mutation(self): + def inspect(*arguments): + if arguments[:2] == ("context", "inspect"): + return json.dumps([{"Endpoints": {"docker": {"Host": "unix:///fixture.sock"}}}]) + if arguments[:2] == ("info", "--format"): + return json.dumps("fixture-daemon") + if arguments[:2] == ("container", "ls"): + return "fixture-id\n" + if arguments[:2] == ("container", "inspect"): + return json.dumps([{"Config": {"Labels": {ownership.LABEL: "fixture", "com.docker.compose.project": "FOREIGN"}}}]) + raise AssertionError(arguments) + with patch.object(egress_cleanup, "docker", side_effect=inspect), patch.dict("os.environ", {"DOCKER_HOST": "unix:///fixture.sock"}): + with self.assertRaisesRegex(ValueError, "ownership mismatch"): + egress_cleanup.inventory(self.private.manifest) + + def test_wrong_local_daemon_does_not_discard_recovery_manifest(self): + responses = [json.dumps([{"Endpoints": {"docker": {"Host": "unix:///different.sock"}}}]), + json.dumps("different-daemon")] + with patch.object(egress_cleanup, "docker", side_effect=responses) as docker: + with self.assertRaisesRegex(ValueError, "daemon identity mismatch"): + egress_cleanup.recover(self.private.name, remove=True) + self.assertEqual(docker.call_count, 2) + self.assertTrue((Path(self.private.name) / "owner.json").exists()) + + +class OwnershipTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + root = patch.object(ownership, "owner_root", return_value=self.root) + root.start() + self.addCleanup(root.stop) + + def test_project_lock_excludes_concurrent_owners_and_releases(self): + first = ownership.ProjectLock("fixture-project") + self.addCleanup(first.close) + with self.assertRaises(BlockingIOError): + ownership.ProjectLock("fixture-project") + first.close() + second = ownership.ProjectLock("fixture-project") + second.close() + + def test_private_manifest_is_recoverable_without_gc_cleanup(self): + directory = ownership.PrivateDirectory("fixture", "project", "sha256:fixture") + name = directory.name + del directory + self.assertTrue(Path(name).is_dir()) + self.assertEqual(ownership.read_manifest(name)["instance"], "fixture") + + def test_manifest_change_or_symlink_prevents_removal(self): + directory = ownership.PrivateDirectory("fixture", "project", "sha256:fixture") + manifest = Path(directory.name) / "owner.json" + manifest.unlink() + target = self.root / "unrelated" + target.write_text("must survive") + manifest.symlink_to(target) + with self.assertRaises(OSError): + directory.cleanup() + self.assertEqual(target.read_text(), "must survive") diff --git a/scripts/tests/test_egress_config.py b/scripts/tests/test_egress_config.py new file mode 100644 index 0000000..8f89abb --- /dev/null +++ b/scripts/tests/test_egress_config.py @@ -0,0 +1,166 @@ +import copy +import json +from pathlib import Path +import tempfile +import unittest + +from scripts.egress.config import direct_config, load_config +from scripts.egress.compose import NETWORK, SERVICE, declared_services, validate_final + + +class ConfigTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.path = self.root / "egress.json" + self.config = {"version": 1, "image": "searchswe-egress:dev1", + "upstream": {"url": "https://proxy.example:8080", "address": "192.0.2.1"}, + "dns": {"doh_url": "https://resolver.example/dns-query"}} + + def load(self): + self.path.write_text(json.dumps(self.config)) + return load_config(self.path) + + def test_bootstrap_does_not_resolve_dns(self): + config = self.load() + self.assertEqual(config.settings["upstream_addr"], "192.0.2.1:8080") + self.assertEqual(config.settings["upstream_host"], "proxy.example") + del self.config["upstream"]["address"] + with self.assertRaisesRegex(ValueError, "explicit IPv4"): + self.load() + + def test_no_embedded_credentials_or_url_queries(self): + for url in ("https://user:sensitive-value@proxy.example", "http://proxy.example/?token=sensitive-value", + "socks5://proxy.example", "http://proxy.example/path", "http://[::1]", "http://proxy.example:bad"): + self.config["upstream"]["url"] = url + with self.subTest(url=url), self.assertRaises(ValueError) as caught: + self.load() + self.assertNotIn("sensitive-value", str(caught.exception)) + + def test_doh_requires_https(self): + self.config["dns"]["doh_url"] = "http://resolver.example/dns-query" + with self.assertRaises(ValueError): + self.load() + + def test_direct_defaults_and_explicit_dns(self): + settings = direct_config().private_settings() + self.assertEqual(settings, {"transport": "direct", "dns_servers": ["127.0.0.11"]}) + self.config = {"version": 1, "mode": "direct", "image": "fixture:direct", "dns": {"servers": ["192.0.2.53"]}} + self.assertEqual(self.load().settings["dns_servers"], ["192.0.2.53"]) + self.config["upstream"] = {"url": "http://192.0.2.1"} + with self.assertRaisesRegex(ValueError, "must not specify"): + self.load() + + def test_direct_dns_rejects_unusable_or_ambiguous_addresses(self): + for servers in ([], "192.0.2.53", [True], ["127.0.0.1"], ["0.0.0.0"], ["::1"], + ["resolver.example"], ["169.254.169.254"], ["224.0.0.1"], ["127.0.0.11:5353"], + ["192.0.2.53:٥٣"], ["192.0.2.53:"], ["192.0.2.53:0"], ["192.0.2.53:65536"]): + with self.subTest(servers=servers), self.assertRaises(ValueError): + direct_config(dns_servers=servers) + + def test_upstream_cannot_be_task_loopback_or_metadata(self): + for address in ("127.0.0.1", "0.0.0.0", "169.254.169.254", "224.0.0.1", True): + self.config["upstream"]["address"] = address + with self.subTest(address=address), self.assertRaises(ValueError): + self.load() + + def test_strict_schema(self): + for value in (True, 2, "1"): + self.config["version"] = value + with self.assertRaises(ValueError): + self.load() + self.config["version"] = 1 + self.config["upstream"]["password"] = "forbidden-inline-value" + with self.assertRaises(ValueError): + self.load() + + def test_secret_permissions_and_symlinks(self): + secret = self.root / "credentials.json" + secret.write_text(json.dumps({"username": "fixture", "password": "not-a-real-secret"})) + secret.chmod(0o600) + self.config["upstream"]["auth_file"] = "credentials.json" + config = self.load() + self.assertNotIn("not-a-real-secret", repr(config)) + self.assertNotIn("auth", config.settings) + self.assertEqual(config.private_settings()["auth"]["password"], "not-a-real-secret") + secret.chmod(0o644) + with self.assertRaisesRegex(ValueError, "0600"): + config.private_settings() + secret.chmod(0o600) + alias = self.root / "alias.json" + alias.symlink_to(secret) + self.config["upstream"]["auth_file"] = "alias.json" + with self.assertRaises(OSError): + self.load().private_settings() + + +class ComposeTests(unittest.TestCase): + def setUp(self): + self.document = {"name": "unique-trial", "networks": {"default": {"name": "unique-trial_default"}}, + "services": {SERVICE: {"image": "sha256:fixture", "networks": {"default": {}}, + "read_only": True, "tmpfs": ["/run/searchswe:mode=0700"], + "entrypoint": ["/opt/egress-sidecar/entrypoint.sh"]}, + "main": {"network_mode": NETWORK, "cap_drop": ["NET_RAW", "NET_ADMIN"], + "security_opt": ["no-new-privileges:true"], + "depends_on": {SERVICE: {"condition": "service_healthy"}}}}} + + def validate(self, document=None): + validate_final(document or self.document, private_directory="/tmp/private-egress-unique", image="sha256:fixture") + + def test_hardened_compose(self): + self.validate() + + def test_all_untrusted_services_checked(self): + self.document["services"]["helper"] = {"image": "python"} + with self.assertRaises(ValueError): + self.validate() + + def test_privilege_topology_and_control_mount_rejection(self): + # Keep each mutation independent of every other guard. + cases = [{"cap_add": ["SYS_ADMIN"]}, {"cap_drop": []}, {"privileged": True}, {"pid": "host"}, + {"ipc": "host"}, {"network_mode": "host"}, {"security_opt": ["seccomp:unconfined"]}] + cases += [{"volumes": [{"type": "bind", "source": source, "target": "/host"}]} + for source in ("/tmp/private-egress-unique", "/tmp", "/", "/proc/sys", "/sys", "/var", "/run/docker.sock")] + for change in cases: + document = copy.deepcopy(self.document) + document["services"]["main"].update(change) + with self.subTest(change=change), self.assertRaises(ValueError): + self.validate(document) + + def test_raw_explicit_topology_and_reserved_service_rejected(self): + for document in ({"services": {SERVICE: {}}}, {"services": {"side": {"network_mode": "host"}}}, + {"networks": {"custom": {}}}): + with self.assertRaises(ValueError): + declared_services([document]) + self.assertEqual(declared_services([{"services": {"helper": {"image": "python"}}}]), ["helper", "main"]) + + def test_agent_cannot_modify_config_used_by_next_verifier(self): + self.document["services"]["main"]["volumes"] = [{"type": "bind", "source": "/home/operator/settings", + "target": "/workspace"}] + with self.assertRaisesRegex(ValueError, "gateway configuration"): + validate_final(self.document, private_directory="/tmp/private-egress-unique", image="sha256:fixture", + protected_paths=["/home/operator/settings/egress.json"]) + + def test_task_build_cannot_copy_private_config_or_request_host_privileges(self): + for build in ({"context": "/tmp"}, {"context": "/safe", "network": "host"}, + {"context": "/safe", "additional_contexts": {"host": "/tmp"}}, + {"context": "/safe", "secrets": ["proxy-auth"]}): + self.document["services"]["main"]["build"] = build + with self.subTest(build=build), self.assertRaises(ValueError): + self.validate() + + def test_named_volume_cannot_hide_host_bind_or_external_driver(self): + for change in ({"driver_opts": {"type": "none", "o": "bind", "device": "/"}}, + {"driver": "external-plugin"}): + self.document["volumes"] = {"data": {"name": "unique-trial_data", **change}} + with self.subTest(change=change), self.assertRaises(ValueError): + self.validate() + + def test_plain_project_local_named_volume_is_supported(self): + self.document["volumes"] = {"data": {"name": "unique-trial_data", "driver": "local"}} + self.validate() + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_egress_contract.py b/scripts/tests/test_egress_contract.py new file mode 100644 index 0000000..5bd02bd --- /dev/null +++ b/scripts/tests/test_egress_contract.py @@ -0,0 +1,212 @@ +"""S0 contracts: real Harbor factory/phase paths, no Docker or model calls. + +ContractDocker is a test double, not the proposed production adapter. Only its +constructor uses real DockerEnvironment initialization; start/apply/stop record +events instead of running containers. These tests establish extension seams, +not enforcement correctness. +""" + +import asyncio +import logging +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from harbor.environments.docker.docker import DockerEnvironment +from harbor.environments.factory import EnvironmentFactory +from harbor.models.task.config import EnvironmentConfig as TaskEnvironment, NetworkPolicy, TaskConfig +from harbor.models.task.verifier_mode import resolve_effective_verifier_env_config, resolve_task_verifier_mode, resolve_step_verifier_mode +from harbor.models.trial.config import AgentConfig, EnvironmentConfig +from harbor.models.trial.paths import TrialPaths +from harbor.trial.network_policy import resolve_trial_network_plan +from harbor.trial.trial import Trial + +from scripts.tests.egress_s0.fixture import dns_packet, questions +from scripts.tests.egress_s0.run import configuration +from scripts.tests.egress_full_trial import task_config + + +REPO = Path(__file__).resolve().parents[2] + + +class ContractDocker(DockerEnvironment): + instances = [] + + def __init__(self, *args, s0_binding=None, **kwargs): + self.s0_binding = s0_binding + self.events = [] + self.fail_update = False + super().__init__(*args, **kwargs) + self.instances.append(self) + + async def start(self, force_build): + self.events.append(("start", self.network_policy)) + + async def stop(self, delete): + self.events.append(("stop", delete)) + + async def _apply_network_policy(self, policy): + if self.fail_update: + raise RuntimeError("fixture policy application failed") + self.events.append(("policy", policy)) + + +class EgressHarborContractTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + # Fixture text-file generation only; no task package is modified. + (self.root / "Dockerfile").write_text("FROM fixture-not-built\n") + self.paths = TrialPaths(trial_dir=self.root / "trial") + patcher = patch.object(DockerEnvironment, "_egress_control_kernel_support", return_value=True) + patcher.start() + self.addCleanup(patcher.stop) + ContractDocker.instances.clear() + self.environment_config = EnvironmentConfig( + import_path=f"{__name__}:ContractDocker", kwargs={"s0_binding": "host-config-path-only"}) + + def create(self, policy): + return EnvironmentFactory.create_environment_from_config( + config=self.environment_config, environment_dir=self.root, + environment_name="s0", session_id="s0-agent", trial_paths=self.paths, + task_env_config=TaskEnvironment(), network_policy=policy) + + def plan(self, task_name, model_host="model.example"): + config = TaskConfig.model_validate_toml((REPO / "tasks" / task_name / "task.toml").read_text()) + return resolve_trial_network_plan( + config, AgentConfig(extra_allowed_hosts=[model_host]), self.environment_config, None, + verifier_mode=resolve_task_verifier_mode(config), + env_config=resolve_effective_verifier_env_config(config, None)) + + def test_phase_policy_uses_harbor_inheritance_and_model_host_is_agent_only(self): + plan = self.plan("task-1-1") + self.assertEqual(plan.agent_env_baseline.allowed_hosts, ["openrouter.ai", "api.jina.ai"]) + self.assertIn("model.example", plan.agent_phase.allowed_hosts) + self.assertNotIn("model.example", plan.verifier_phase.allowed_hosts) + self.assertIn("api.deepseek.com", plan.verifier_phase.allowed_hosts) + self.assertNotIn("api.deepseek.com", plan.agent_phase.allowed_hosts) + offline = self.plan("task-2-1") + self.assertEqual(offline.agent_env_baseline.network_mode.value, "no-network") + self.assertEqual(offline.agent_phase.allowed_hosts, ["model.example"]) + + def test_real_separate_verifier_path_preserves_adapter_and_binding_not_overlay(self): + plan = self.plan("task-1-1") + agent = self.create(plan.agent_env_baseline) + external_overlay = self.root / "agent-only.json" + external_overlay.write_text('{"services": {}}\n') + self.environment_config.extra_docker_compose = [external_overlay] + fake_trial = SimpleNamespace( + config=SimpleNamespace(environment=self.environment_config), + task=SimpleNamespace(short_name="s0"), paths=self.paths, _id="fixture", + logger=logging.getLogger("s0"), _environment_build_timeout_sec=5, + _verifier_env_build_context=lambda step: self.root, + _separate_verifier_session_id=lambda key: "s0-verifier-" + key, + _verifier_env_mounts=lambda config: [], + _validate_separate_verifier_env_policies=lambda env, plan: None) + + async def exercise(): + async with Trial._separate_verifier_env( + fake_trial, TaskEnvironment(), key="first", plan=plan) as verifier: + self.assertIsInstance(verifier, ContractDocker) + self.assertIsNot(verifier, agent) + self.assertEqual(verifier.s0_binding, "host-config-path-only") + self.assertEqual(verifier.extra_docker_compose_paths, []) + self.assertEqual(verifier.network_policy, plan.verifier_env_baseline) + self.assertEqual(verifier.events[0][0], "start") + self.assertEqual(verifier.events[-1], ("stop", True)) + async with Trial._separate_verifier_env( + fake_trial, TaskEnvironment(), key="second", plan=plan) as second: + self.assertIsNot(second, verifier) + + asyncio.run(exercise()) + self.assertEqual(len(ContractDocker.instances), 3) + self.assertEqual(self.environment_config.extra_docker_compose, [external_overlay]) + + def test_real_phase_context_restores_baseline_on_error(self): + plan = self.plan("task-1-1") + environment = self.create(plan.agent_env_baseline) + + async def exercise(): + with self.assertRaisesRegex(RuntimeError, "agent failed"): + async with Trial._phase_network_policy( + None, environment, baseline_policy=plan.agent_env_baseline, + phase_policy=plan.agent_phase): + self.assertEqual(environment.network_policy, plan.agent_phase) + raise RuntimeError("agent failed") + + asyncio.run(exercise()) + self.assertEqual(environment.events, [("policy", plan.agent_phase), ("policy", plan.agent_env_baseline)]) + self.assertEqual(environment.network_policy, plan.agent_env_baseline) + + def test_failed_update_is_not_reported_as_applied(self): + baseline = NetworkPolicy(network_mode="no-network") + environment = self.create(baseline) + environment.fail_update = True + with self.assertRaisesRegex(RuntimeError, "application failed"): + asyncio.run(environment.set_network_policy(NetworkPolicy( + network_mode="allowlist", allowed_hosts=["allowed.example"]))) + self.assertEqual(environment.network_policy, baseline) + self.assertEqual(environment.events, []) + + def test_dns_fixture_encodes_non_in_and_multiple_questions(self): + self.assertEqual(questions(dns_packet([("blocked.example", 3)])), + [{"name": "blocked.example", "class": 3, "type": 1}]) + self.assertEqual(len(questions(dns_packet([ + ("allowed.example", 1), ("blocked.example", 1)]))), 2) + + def test_candidate_marks_are_strings_for_gost_json_metadata(self): + # GOST v0.10.9 GetInt does not handle the float64 produced by decoding + # arbitrary numeric JSON metadata. Losing the mark recurses into red. + config = configuration("192.0.2.10") + for service in config["services"]: + self.assertEqual(service["metadata"]["so_mark"], "114514") + node = config["chains"][0]["hops"][0]["nodes"][0] + self.assertEqual(node["metadata"]["so_mark"], "114514") + self.assertFalse(config["services"][0]["handler"]["metadata"]["sniffing.fallback"]) + + +if __name__ == "__main__": + unittest.main() + + +def plans(task, runtime): + return [resolve_trial_network_plan(task, AgentConfig(extra_allowed_hosts=["model.example"]), runtime, step, + verifier_mode=resolve_step_verifier_mode(task, step), + env_config=resolve_effective_verifier_env_config(task, step)) for step in task.steps] + + +class MultiStepPlansTests(unittest.TestCase): + def test_step_overrides_do_not_union_with_task_and_model_is_agent_only(self): + inherited, override, offline, shared = plans(task_config("alpha", "fixture"), EnvironmentConfig()) + self.assertEqual(inherited.agent_env_baseline.allowed_hosts, ["base-alpha.example"]) + self.assertEqual(inherited.agent_phase.allowed_hosts, ["task-alpha.example", "model.example"]) + self.assertEqual(override.agent_phase.allowed_hosts, ["step-alpha.example", "model.example"]) + self.assertEqual(override.verifier_phase.allowed_hosts, ["score-alpha.example"]) + self.assertEqual(override.verifier_env_baseline.network_mode.value, "no-network") + # Harbor deliberately adds the run's model endpoint to an otherwise + # offline agent phase. It never adds it to verifier policy. + self.assertEqual(offline.agent_phase.allowed_hosts, ["model.example"]) + self.assertEqual(offline.verifier_phase.network_mode.value, "no-network") + self.assertEqual(offline.verifier_env_baseline.allowed_hosts, ["vbase-alpha.example"]) + self.assertIsNone(shared.verifier_env_baseline) + self.assertEqual(shared.verifier_phase_baseline, inherited.agent_env_baseline) + self.assertEqual(shared.verifier_phase.allowed_hosts, ["shared-alpha.example"]) + + def test_different_task_plans_only_share_explicit_model_host(self): + a, b = [plans(task_config(name, "fixture"), EnvironmentConfig()) for name in ("alpha", "beta")] + for first, second in zip(a, b): + self.assertEqual(set(first.agent_phase.allowed_hosts) & set(second.agent_phase.allowed_hosts), {"model.example"}) + self.assertFalse(set(first.verifier_phase.allowed_hosts) & set(second.verifier_phase.allowed_hosts)) + + def test_runtime_environment_hosts_do_not_leak_into_explicit_step_environment(self): + inherited, override, offline, shared = plans(task_config("alpha", "fixture"), + EnvironmentConfig(extra_allowed_hosts=["trial-env.example"])) + self.assertEqual(inherited.agent_env_baseline.allowed_hosts, ["base-alpha.example", "trial-env.example"]) + self.assertEqual(inherited.verifier_env_baseline.allowed_hosts, ["vbase-alpha.example"]) + self.assertEqual(override.verifier_env_baseline.allowed_hosts, []) + self.assertEqual(override.agent_phase.allowed_hosts, ["step-alpha.example", "model.example"]) + self.assertEqual(offline.verifier_env_baseline.allowed_hosts, ["vbase-alpha.example"]) + self.assertEqual(shared.verifier_phase_baseline.allowed_hosts, ["base-alpha.example", "trial-env.example"]) diff --git a/scripts/tests/test_egress_credentials.py b/scripts/tests/test_egress_credentials.py new file mode 100644 index 0000000..cd01830 --- /dev/null +++ b/scripts/tests/test_egress_credentials.py @@ -0,0 +1,56 @@ +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from scripts.tests.egress_credentials import CredentialLeak, CredentialProbe + + +class CredentialEvidenceTests(unittest.TestCase): + def setUp(self): + self.root = tempfile.TemporaryDirectory() + self.addCleanup(self.root.cleanup) + patched = patch("scripts.tests.egress_credentials.owner_root", return_value=Path(self.root.name)) + patched.start() + self.addCleanup(patched.stop) + self.probe = CredentialProbe() + self.addCleanup(self.probe.close) + + def test_raw_json_url_and_basic_variants_are_detected_without_echo(self): + for pattern in self.probe.patterns: + with self.subTest(), self.assertRaises(CredentialLeak) as caught: + self.probe.scan(b"prefix=" + pattern, "fixture-surface") + for value in self.probe.auth.values(): + self.assertNotIn(value, str(caught.exception)) + + def test_private_source_is_owner_only_and_removed(self): + self.assertEqual(self.probe.path.stat().st_mode & 0o777, 0o600) + self.assertEqual(self.probe.path.parent.stat().st_mode & 0o777, 0o700) + self.assertEqual(json.loads(self.probe.path.read_text()), self.probe.auth) + self.probe.close() + self.assertFalse(self.probe.path.exists()) + + def test_artifact_hit_is_withheld_not_reported_as_clean(self): + root = Path(self.root.name) / "output" + root.mkdir() + artifact = root / "leaking-fixture.log" + artifact.write_text(json.dumps(self.probe.auth)) + with self.assertRaises(CredentialLeak): + self.probe.artifacts(root) + self.assertIn("Withheld", artifact.read_text()) + self.probe.scan(artifact.read_bytes(), "sanitized-fixture") + + def test_external_source_is_not_deleted_by_child_probe(self): + child = CredentialProbe(self.probe.path) + child.close() + self.assertTrue(self.probe.path.exists()) + + def test_safe_artifacts_pass_and_symlinks_are_rejected(self): + root = Path(self.root.name) / "output" + root.mkdir() + (root / "safe.json").write_text('{"kind":"generation_ready"}') + self.probe.artifacts(root) + (root / "unsafe-link").symlink_to(self.probe.path) + with self.assertRaisesRegex(RuntimeError, "symlink"): + self.probe.artifacts(root) diff --git a/scripts/tests/test_egress_gateway.py b/scripts/tests/test_egress_gateway.py new file mode 100644 index 0000000..88e6142 --- /dev/null +++ b/scripts/tests/test_egress_gateway.py @@ -0,0 +1,176 @@ +import importlib.util +import json +from pathlib import Path +import tempfile +import time +import unittest +from unittest import mock + + +SOURCE = Path(__file__).resolve().parents[2] / "environments/egress/gateway.py" +spec = importlib.util.spec_from_file_location("searchswe_test_gateway", SOURCE) +gateway = importlib.util.module_from_spec(spec) +spec.loader.exec_module(gateway) + + +class GatewayTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.run = mock.patch.object(gateway, "RUN", self.root) + self.ready = mock.patch.object(gateway, "READY", self.root / "ready") + self.run.start() + self.ready.start() + self.addCleanup(self.run.stop) + self.addCleanup(self.ready.stop) + audit = mock.patch.object(gateway.Controller, "audit") + audit.start() + self.addCleanup(audit.stop) + self.settings = {"upstream_ip": "192.0.2.1", "upstream_port": 8080, + "upstream_addr": "192.0.2.1:8080", "upstream_host": "proxy.example", + "upstream_tls": True, "doh_url": "https://resolver.example/dns-query", + "auth": {"username": "fixture", "password": "not-a-real-secret"}} + + def test_exact_host_policy_only(self): + for host in ("*.example", "1.2.3.4", "[::1]", "name.example:443", "UPPER.example", + "name.example\n", "a..example", "-a.example", "local", 1): + with self.subTest(host=host), self.assertRaises(ValueError): + gateway.policy("allowlist", [host]) + self.assertEqual(gateway.policy("allowlist", ["a.example", "a.example"])["hosts"], ["a.example"]) + for mode in ("public", "allow-all", "none"): + with self.assertRaises(ValueError): + gateway.policy(mode, []) + + def test_config_marks_tls_and_no_fallback(self): + config = gateway.gost_config(self.settings, ["allowed.example"]) + node = config["chains"][0]["hops"][0]["nodes"][0] + self.assertEqual(node["dialer"]["tls"], {"serverName": "proxy.example", "secure": True}) + self.assertEqual(node["metadata"]["so_mark"], "114514") + for service in config["services"]: + self.assertEqual(service["metadata"]["so_mark"], "114514") + self.assertEqual(service["handler"]["chain"], "operator") + self.assertFalse(config["services"][0]["handler"]["metadata"]["sniffing.fallback"]) + self.assertNotIn("api", config) + + def test_direct_routing_uses_filtered_resolver_without_proxy(self): + config = gateway.gost_config({"transport": "direct", "dns_servers": ["192.0.2.53"]}, ["allowed.example"]) + self.assertNotIn("chains", config) + self.assertEqual(config["services"][0]["resolver"], "phase-dns") + self.assertEqual(config["resolvers"][0]["nameservers"][0]["only"], "ipv4") + for service in config["services"]: + self.assertNotIn("chain", service["handler"]) + self.assertEqual(service["bypass"], "phase") + + def test_direct_public_is_leased_and_restriction_removes_public_mark(self): + controller = gateway.Controller({"transport": "direct", "dns_servers": ["127.0.0.11"]}) + commands = [] + with mock.patch.object(gateway, "nft", side_effect=commands.append): + controller.apply({"mode": "public", "hosts": [], "deadline_ns": time.monotonic_ns() + 30 * 10**9}) + self.assertTrue(any(", 0 timeout" in command for command in commands)) + self.assertEqual(commands[-1], "delete table inet searchswe_guard\n") + commands.clear() + controller.apply({"mode": "no-network", "hosts": [], "deadline_ns": time.monotonic_ns() + 30 * 10**9}) + self.assertFalse(any(", 0 timeout" in command for command in commands)) + self.assertFalse(any(command.startswith("delete table") for command in commands)) + + def test_generation_closes_gate_before_killing_and_opens_last(self): + commands = [] + worker = mock.Mock() + worker.poll.return_value = None + worker.terminate.side_effect = lambda: commands.append("terminate") + controller = gateway.Controller(self.settings) + with mock.patch.object(gateway, "nft", side_effect=commands.append), \ + mock.patch.object(gateway.subprocess, "Popen", return_value=worker), \ + mock.patch.object(gateway, "owns_listeners", return_value=True): + state = controller.apply({"mode": "allowlist", "hosts": ["allowed.example"], + "deadline_ns": time.monotonic_ns() + 30 * 10**9}) + self.assertTrue(state["ready"]) + self.assertNotIn("not-a-real-secret", json.dumps(state)) + self.assertEqual(commands[-1], "delete table inet searchswe_guard\n") + commands.clear() + state = controller.apply({"mode": "no-network", "hosts": [], + "deadline_ns": time.monotonic_ns() + 30 * 10**9}) + self.assertIn("priority -300", commands[0]) + self.assertEqual(commands[1], "terminate") + self.assertFalse(any(c.startswith("delete table") for c in commands)) + self.assertIsNone(controller.worker) + self.assertEqual(state["generation"], 2) + + def test_invalid_replacement_and_failed_worker_remain_closed(self): + controller = gateway.Controller(self.settings) + worker = mock.Mock() + worker.poll.return_value = 1 + with mock.patch.object(gateway, "nft") as nft, \ + mock.patch.object(gateway.subprocess, "Popen", return_value=worker): + for request in ({"mode": "allowlist", "hosts": ["*.example"]}, + {"mode": "allowlist", "hosts": ["allowed.example"]}): + with self.assertRaises((ValueError, RuntimeError)): + controller.apply({**request, "deadline_ns": time.monotonic_ns() + 30 * 10**9}) + self.assertFalse(controller.state["ready"]) + self.assertFalse(gateway.READY.exists()) + self.assertFalse(any(c.args[0].startswith("delete table") for c in nft.call_args_list)) + + def test_nft_interpolation_revalidates_settings(self): + with mock.patch.object(gateway, "nft") as nft: + for ip in ("::1", "192.0.2.1; flush ruleset", "proxy.example"): + with self.assertRaises(ValueError): + gateway.install_rules({**self.settings, "upstream_ip": ip}) + nft.assert_not_called() + + def test_upstream_cannot_target_shared_namespace(self): + with mock.patch.object(gateway.subprocess, "run") as run: + run.return_value.stdout = "local 192.0.2.1 dev lo src 192.0.2.1" + with self.assertRaises(ValueError): + gateway.validate_upstream_namespace(self.settings) + run.return_value.stdout = "192.0.2.1 via 172.18.0.1 dev eth0" + gateway.validate_upstream_namespace(self.settings) + + def test_expired_lease_cannot_be_renewed_into_old_permissions(self): + controller = gateway.Controller(self.settings) + controller.state = {"ready": True} + controller.deadline_ns = time.monotonic_ns() - 1 + with mock.patch.object(gateway, "nft") as nft: + with self.assertRaises(RuntimeError): + controller.renew(time.monotonic_ns() + 30 * 10**9) + self.assertFalse(controller.state["ready"]) + self.assertFalse(any("add element" in c.args[0] for c in nft.call_args_list)) + + def test_direct_dns_cannot_be_impersonated_inside_task_namespace(self): + with mock.patch.object(gateway.subprocess, "run") as run: + run.return_value.stdout = "local 192.0.2.53 dev lo src 192.0.2.53" + with self.assertRaisesRegex(ValueError, "outside the shared task"): + gateway.validate_upstream_namespace({"transport": "direct", "dns_servers": ["192.0.2.53"]}) + run.reset_mock() + gateway.validate_upstream_namespace({"transport": "direct", "dns_servers": ["127.0.0.11:53"]}) + run.assert_not_called() + with self.assertRaisesRegex(ValueError, "port 53"): + gateway.validate_upstream_namespace({"transport": "direct", "dns_servers": ["127.0.0.11:5353"]}) + run.return_value.stdout = "192.0.2.53 via 172.18.0.1 dev eth0" + gateway.validate_upstream_namespace({"transport": "direct", "dns_servers": ["192.0.2.53:5353"]}) + + def test_lease_uses_bounded_kernel_timeout_and_absolute_cutoff(self): + controller = gateway.Controller(self.settings) + with mock.patch.object(gateway, "nft") as nft: + controller.refresh_lease(time.monotonic_ns() + 30 * 10**9) + command = nft.call_args.args[0] + self.assertIn("timeout ", command) + self.assertIn('meta time < "', command) + for deadline in (0, True, time.monotonic_ns() + 60 * 10**9): + with self.assertRaises(ValueError): + controller.refresh_lease(deadline) + + def test_nft_failure_still_terminates_worker(self): + controller = gateway.Controller(self.settings) + worker = mock.Mock() + worker.poll.return_value = None + controller.worker = worker + with mock.patch.object(gateway, "nft", side_effect=RuntimeError("fixture failure")): + with self.assertRaises(RuntimeError): + controller.fail() + worker.terminate.assert_called_once() + self.assertFalse(controller.state["ready"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_egress_launcher.py b/scripts/tests/test_egress_launcher.py new file mode 100644 index 0000000..f649916 --- /dev/null +++ b/scripts/tests/test_egress_launcher.py @@ -0,0 +1,123 @@ +import json +import os +from pathlib import Path +import subprocess +import shutil +import sys +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] + + +class EgressLauncherTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.env_file = self.root / "empty.env" + self.env_file.touch() + self.config = self.root / "egress.json" + self.document = {"version": 1, "image": "searchswe-egress:dev3", + "upstream": {"url": "http://192.0.2.1:8080"}, + "dns": {"doh_url": "https://resolver.example/dns-query"}} + self.config.write_text(json.dumps(self.document)) + + def launch(self, *arguments, task="task-1-1", env=None, public=False): + repo = ROOT + if public: + # Public-policy coverage must not depend on a benchmark task's + # evolving network requirements (for example, task-2-3). + repo = self.root / "repo" + shutil.copytree(ROOT / "scripts", repo / "scripts", + ignore=shutil.ignore_patterns("__pycache__", "tests")) + task = "task-fixture" + fixture = repo / "tasks" / task + fixture.mkdir(parents=True) + (fixture / "task.toml").write_text( + '[environment]\nnetwork_mode = "no-network"\n' + '[agent]\nnetwork_mode = "public"\n' + '[verifier]\nnetwork_mode = "no-network"\n') + return subprocess.run( + [sys.executable, str(repo / "scripts/run_task.py"), "--task", task, "--agent", "pi", + "--model", "deepseek/deepseek-flash", "--env-file", str(self.env_file), "--dry-run", *arguments], + env={"PATH": os.environ["PATH"], "HOME": str(self.root), **(env or {})}, + capture_output=True, text=True, timeout=30) + + def test_default_restricted_task_uses_direct_gateway(self): + response = self.launch() + self.assertEqual(response.returncode, 0, response.stderr) + self.assertIn("--env scripts.harbor_environments:PhaseScopedDocker", response.stdout) + self.assertIn("egress_image=hanhainebula/search-swe-egress:1.0.0", response.stdout) + self.assertIn("Phase-scoped direct gateway", response.stdout) + + def test_direct_dns_and_image_override(self): + response = self.launch("--container-dns", "192.0.2.53", "--egress-image", "fixture:direct") + self.assertEqual(response.returncode, 0, response.stderr) + self.assertIn("egress_dns=192.0.2.53", response.stdout) + self.assertIn("egress_image=fixture:direct", response.stdout) + self.assertNotIn("--extra-docker-compose", response.stdout) + + def test_public_agent_with_restricted_verifier_uses_direct_gateway(self): + response = self.launch(public=True) + self.assertEqual(response.returncode, 0, response.stderr) + self.assertIn("PhaseScopedDocker", response.stdout) + + def test_direct_configuration_supports_public_phases(self): + self.config.write_text(json.dumps({"version": 1, "mode": "direct", "image": "fixture:direct", "dns": {}})) + response = self.launch("--egress-config", str(self.config), public=True) + self.assertEqual(response.returncode, 0, response.stderr) + self.assertIn("PhaseScopedDocker", response.stdout) + + def test_opt_in_uses_custom_adapter_and_public_path_only(self): + # Dry-run must not read this absent credential file or inspect Docker. + self.document["upstream"]["auth_file"] = "not-present.json" + self.config.write_text(json.dumps(self.document)) + response = self.launch("--egress-config", str(self.config)) + self.assertEqual(response.returncode, 0, response.stderr) + self.assertIn("--env scripts.harbor_environments:PhaseScopedDocker", response.stdout) + self.assertIn("--environment-kwarg egress_config=", response.stdout) + self.assertNotIn("not-present.json", response.stdout) + self.assertNotIn("CONTAINER_PROXY", response.stdout) + + def test_env_selection_and_cli_override(self): + response = self.launch(env={"EGRESS_CONFIG": str(self.config)}) + self.assertEqual(response.returncode, 0, response.stderr) + response = self.launch("--egress-config", str(self.config), env={"EGRESS_CONFIG": "/not/present"}) + self.assertEqual(response.returncode, 0, response.stderr) + + def test_dns_override_conflict_is_explicit(self): + response = self.launch("--egress-config", str(self.config), env={"CONTAINER_DNS": "192.0.2.53"}) + self.assertNotEqual(response.returncode, 0) + self.assertIn("clear them explicitly", response.stderr) + + def test_generic_proxy_cannot_bypass_restricted_path(self): + response = self.launch(env={"CONTAINER_PROXY": "http://192.0.2.1:8080"}) + self.assertNotEqual(response.returncode, 0) + self.assertIn("general proxy can bypass", response.stderr) + + def test_public_phase_rejected_without_changing_task(self): + response = self.launch("--egress-config", str(self.config), public=True) + self.assertNotEqual(response.returncode, 0, response.stdout + response.stderr) + self.assertIn("any public phase", response.stderr) + self.assertIn('network_mode = "public"', + (self.root / "repo/tasks/task-fixture/task.toml").read_text()) + + def test_rejected_url_does_not_echo_embedded_secret(self): + self.document["upstream"]["url"] = "https://user:do-not-leak-this@proxy.example" + self.config.write_text(json.dumps(self.document)) + response = self.launch("--egress-config", str(self.config)) + self.assertNotEqual(response.returncode, 0) + self.assertNotIn("do-not-leak-this", response.stdout + response.stderr) + + def test_proxy_cannot_be_exposed_as_an_allowed_model_host(self): + self.document["upstream"] = {"url": "https://api.deepseek.com:8443", "address": "192.0.2.1"} + self.config.write_text(json.dumps(self.document)) + response = self.launch("--egress-config", str(self.config)) + self.assertNotEqual(response.returncode, 0) + self.assertIn("cannot also be an allowed", response.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_network_probe.py b/scripts/tests/test_network_probe.py deleted file mode 100644 index 21f24b0..0000000 --- a/scripts/tests/test_network_probe.py +++ /dev/null @@ -1,27 +0,0 @@ -import subprocess -import unittest -from unittest.mock import patch - -from scripts.run_task import prepare_network_probe - - -class NetworkProbeTests(unittest.TestCase): - @patch("scripts.run_task.subprocess.run") - def test_cached_image(self, run): - run.return_value = subprocess.CompletedProcess([], 0, "", "") - prepare_network_probe() - self.assertEqual(run.call_count, 2) - self.assertEqual(run.call_args_list[1].args[0][1:3], ["run", "--rm"]) - - @patch("scripts.run_task.subprocess.run") - def test_pull_before_probe(self, run): - run.side_effect = [subprocess.CompletedProcess([], code, "", "") for code in (1, 0, 0)] - prepare_network_probe() - self.assertEqual(run.call_args_list[1].args[0][1], "pull") - self.assertEqual(run.call_args_list[1].kwargs["timeout"], 300) - - @patch("scripts.run_task.subprocess.run") - def test_fail_closed(self, run): - run.side_effect = [subprocess.CompletedProcess([], code, "", "") for code in (0, 1)] - with self.assertRaisesRegex(RuntimeError, "CONFIG_NFT_FIB_INET"): - prepare_network_probe() diff --git a/scripts/tests/test_release.py b/scripts/tests/test_release.py index 01e3f86..6efd759 100644 --- a/scripts/tests/test_release.py +++ b/scripts/tests/test_release.py @@ -34,7 +34,7 @@ def setUp(self): for name in ("environment/docker-compose.yaml", "tests/docker-compose.yaml"): (self.task / name).write_text("services: {}\n") (self.task / "task.toml").write_text( - '[task]\nname = "task-new"\nversion = "0.1"\n' + '[task]\nname = "fixture/task-new"\nversion = "0.1"\n' '[verifier.env]\nOPENAI_BASE_URL = "${OPENAI_BASE_URL:-}"\n' 'OPENAI_API_KEY = "${OPENAI_API_KEY:-}"\n' ) @@ -166,6 +166,26 @@ def test_launcher_separates_agent_and_verifier_keys_without_putting_them_in_argv self.assertIn("OPENAI_API_KEY=${AGENT_OPENAI_API_KEY}", argv) self.assertIn("OPENAI_API_KEY=${VERIFIER_OPENAI_API_KEY}", argv) + def test_step_restriction_selects_gateway_and_model_host(self): + (self.task / "task.toml").write_text( + '[environment]\nnetwork_mode = "public"\n' + '[agent]\nnetwork_mode = "public"\n' + '[[steps]]\nname = "online"\n' + '[[steps]]\nname = "offline"\n[steps.agent]\nnetwork_mode = "no-network"\n' + ) + command = [sys.executable, str(self.repo / "scripts/run_task.py"), "--task", "task-new", + "--agent", "pi", "--model", "deepseek/deepseek-flash", "--dry-run"] + result = subprocess.run(command, cwd=self.root, env=self.launcher_env(), capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + argv = shlex.split(result.stdout.splitlines()[-1]) + self.assertEqual(argv[argv.index("--env")+1], "scripts.harbor_environments:PhaseScopedDocker") + self.assertEqual(self.flag_values(argv, "--allow-agent-host"), ["api.deepseek.com"]) + result = subprocess.run(command, cwd=self.root, + env={**self.launcher_env(), "CONTAINER_PROXY": "http://192.0.2.1:7890"}, + capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn("general proxy can bypass", result.stderr) + def test_launcher_makes_repository_agents_importable(self): # A console script starts with its bin directory on sys.path, not cwd. # Import the real adapter in a new interpreter, not just inspect argv. @@ -245,7 +265,7 @@ def test_launcher_bypasses_harbor_022_gpu_preflight_for_docker(self): for gpus, expected in ((0, []), (1, ["0"])): with self.subTest(gpus=gpus): (self.task / "task.toml").write_text( - '[task]\nname = "task-new"\nversion = "0.1"\n' + '[task]\nname = "fixture/task-new"\nversion = "0.1"\n' f'[environment]\ngpus = {gpus}\n' '[verifier.env]\nOPENAI_BASE_URL = "${OPENAI_BASE_URL:-}"\n' 'OPENAI_API_KEY = "${OPENAI_API_KEY:-}"\n' @@ -558,7 +578,7 @@ def test_launcher_requires_only_the_selected_tasks_judge_groups(self): ]: with self.subTest(keys=keys, success=success): (self.task / "task.toml").write_text( - '[task]\nname = "task-new"\nversion = "0.1"\n[verifier.env]\n' + '[task]\nname = "fixture/task-new"\nversion = "0.1"\n[verifier.env]\n' + ''.join(f'{key} = "${{{key}:-}}"\n' for key in keys) ) (self.repo / ".env").write_text(values)