From 067df9aa5e8e2b1079f5e4f5953d501ebaba7d66 Mon Sep 17 00:00:00 2001 From: Tomer Glottman <83163454+tomergee@users.noreply.github.com> Date: Wed, 21 Oct 2026 16:38:58 -0700 Subject: [PATCH] integrations/nemo-gym: a runtime layer that injects the NeMo servers into any task image integrations/nemo-gym/layer is one OCI image, built from scratch the way integrations/openhands/layer is: python-build-standalone CPython 3.12.15, hash-pinned wheels installed with uv, and two servers started with the layer's own interpreter so the task image contributes only its kernel and filesystem: - nemo_runtime.sandbox_server serves NeMo Skills' local sandbox server (POST /execute for ipython, python, shell and lean4; IPython sessions by X-Session-ID; GET /health), vendored verbatim from NeMo-Skills bcf059a. Flask binds 127.0.0.1 by default, which the router cannot reach, so the wrapper binds 0.0.0.0. The layer's interpreter is appended to PATH as the fallback for language: python, so an image's own Python wins when it has one and an image without Python still executes code. - nemo_runtime.exec_server runs NeMo Gym's exec server (POST /exec, /upload, GET /download, /health), the EXEC_SERVER_SCRIPT of the ECS Fargate provider at nemo-gym d4fd5e8, written out verbatim. Both vendored files keep their Apache-2.0 license and a provenance README under layer/third_party. lock.sh regenerates the hash-pinned requirements for amd64 and arm64. The README section on injected daemons is rewritten around the real layer: build and push, the template command, the two servers, the trust note, and the limits (the NeMo Skills server no longer needs a Python in the image; the layer needs glibc 2.17 or newer). The base template must name a task image of its own, because the guest's image is Alpine-based and cannot run the layer's interpreter, so a base without one never bakes its golden snapshot; and a recipe selects the layered base through create.template, not a per-sandbox provider_options template, which would select the base itself and skip the task image. hack/runtime-layer-demo.sh registers the template and calls both servers from inside the environment, through the header form and through the path form. tests/test_e2e.py gains test_runtime_layer_endpoint, gated by SUBSTRATE_E2E_RUNTIME_TEMPLATE and SUBSTRATE_E2E_RUNTIME_IMAGE. RUNTIMES.md and the top-level README point at the layer. No provider code changes. Verified on a GKE cluster (gVisor pool, Substrate ddb46651, ate-env-api with the port proxy): environments from python:3.12-slim and from debian (no Python inside) served 4 to 7 s after create; /execute answered from inside, through the header form and the path form; an IPython session kept its state; language: python ran on the layer's 3.12.15 in the Debian image; the exec server answered /exec; the e2e test passed in 6.4 s. Layer image 87 MB compressed. --- README.md | 2 +- docs/rootfs-images/RUNTIMES.md | 10 + integrations/nemo-gym/README.md | 84 +- .../nemo-gym/hack/runtime-layer-demo.sh | 107 +++ integrations/nemo-gym/layer/.dockerignore | 16 + integrations/nemo-gym/layer/Dockerfile | 66 ++ .../nemo-gym/layer/build-constraints.in | 15 + .../nemo-gym/layer/build-constraints.txt | 6 + integrations/nemo-gym/layer/lock.sh | 54 ++ .../nemo-gym/layer/nemo_runtime/__init__.py | 17 + .../layer/nemo_runtime/exec_server.py | 44 + .../layer/nemo_runtime/sandbox_server.py | 65 ++ integrations/nemo-gym/layer/requirements.in | 22 + integrations/nemo-gym/layer/requirements.txt | 295 +++++++ .../layer/third_party/nemo-gym/LICENSE | 201 +++++ .../layer/third_party/nemo-gym/README.md | 13 + .../layer/third_party/nemo-gym/exec_server.py | 82 ++ .../layer/third_party/nemo-skills/LICENSE | 201 +++++ .../layer/third_party/nemo-skills/README.md | 13 + .../nemo-skills/local_sandbox_server.py | 794 ++++++++++++++++++ integrations/nemo-gym/tests/test_e2e.py | 81 +- 21 files changed, 2166 insertions(+), 22 deletions(-) create mode 100755 integrations/nemo-gym/hack/runtime-layer-demo.sh create mode 100644 integrations/nemo-gym/layer/.dockerignore create mode 100644 integrations/nemo-gym/layer/Dockerfile create mode 100644 integrations/nemo-gym/layer/build-constraints.in create mode 100644 integrations/nemo-gym/layer/build-constraints.txt create mode 100755 integrations/nemo-gym/layer/lock.sh create mode 100644 integrations/nemo-gym/layer/nemo_runtime/__init__.py create mode 100644 integrations/nemo-gym/layer/nemo_runtime/exec_server.py create mode 100644 integrations/nemo-gym/layer/nemo_runtime/sandbox_server.py create mode 100644 integrations/nemo-gym/layer/requirements.in create mode 100644 integrations/nemo-gym/layer/requirements.txt create mode 100644 integrations/nemo-gym/layer/third_party/nemo-gym/LICENSE create mode 100644 integrations/nemo-gym/layer/third_party/nemo-gym/README.md create mode 100644 integrations/nemo-gym/layer/third_party/nemo-gym/exec_server.py create mode 100644 integrations/nemo-gym/layer/third_party/nemo-skills/LICENSE create mode 100644 integrations/nemo-gym/layer/third_party/nemo-skills/README.md create mode 100644 integrations/nemo-gym/layer/third_party/nemo-skills/local_sandbox_server.py diff --git a/README.md b/README.md index cf18493e..e86dfd02 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ while this project adds the environment-shaped API on top. - **`cmd/ate-env-guest`** — The daemon server running inside each actor serving command executions, file read/write, and built-in MCP tools. - **`clients/go`** — The Go client library to manage environments, run commands, and perform file operations. - **`clients/python`** — Python client for environment lifecycle management and batch sandbox orchestration ([README](clients/python/README.md)). -- **`integrations/nemo-gym`** — A [NeMo Gym](https://github.com/NVIDIA-NeMo/Gym) sandbox provider that runs rollout sandboxes as environments, built on the Python client ([README](integrations/nemo-gym/README.md)). +- **`integrations/nemo-gym`** — A [NeMo Gym](https://github.com/NVIDIA-NeMo/Gym) sandbox provider that runs rollout sandboxes as environments, built on the Python client, plus a runtime layer that injects NeMo Skills' sandbox server and NeMo Gym's exec server into any task image ([README](integrations/nemo-gym/README.md)). ## Installation diff --git a/docs/rootfs-images/RUNTIMES.md b/docs/rootfs-images/RUNTIMES.md index cd50f31d..b7aebd43 100644 --- a/docs/rootfs-images/RUNTIMES.md +++ b/docs/rootfs-images/RUNTIMES.md @@ -78,6 +78,16 @@ but Docker Hub, is in [`examples/runtime-layers`](../../examples/runtime-layers) the static `busybox:1.37-musl` mounted as the layer and its `httpd` as the sidecar, run under two different rootfs images. +### NeMo runtime layer + +[`integrations/nemo-gym/layer`](../../integrations/nemo-gym/layer) is a +complete layer of this shape: a relocatable CPython with NeMo Skills' local +sandbox server and NeMo Gym's exec server installed, started with +`--sidecar '/opt/nemo/python/bin/python3 -m nemo_runtime.sandbox_server --host 0.0.0.0 --port 6000'` +and gated with `--sidecar-readyz http://127.0.0.1:6000/health`. The NeMo Gym +provider hands the port out as `endpoint()`; see +[integrations/nemo-gym](../../integrations/nemo-gym/README.md#services-inside-the-sandbox-the-nemo-runtime-layer). + ## Who can reach the second runtime - **Processes inside the environment** reach it at `127.0.0.1:` right diff --git a/integrations/nemo-gym/README.md b/integrations/nemo-gym/README.md index 77cb0eaa..a7f30fa4 100644 --- a/integrations/nemo-gym/README.md +++ b/integrations/nemo-gym/README.md @@ -33,6 +33,7 @@ One gRPC endpoint, `ate-env-api`, carries everything (see the | `upload_file` / `download_file` | `Env.write_file` / `Env.read_file_bytes` | `FileSystemService` in the guest | | `status(handle)` | `Client.get(id)` → lifecycle status | `EnvironmentService`; does **not** wake a parked sandbox | | `close(handle)` | `Client.delete(id)`, NOT_FOUND-safe | `EnvironmentService` | +| `endpoint(handle, port)` (nemo-gym 0.7) | `Client.get(id)` for existence, then a static `SandboxEndpoint` | ate-env-api's port proxy: each request goes through the router to that port in the sandbox, where a runtime layer's daemon listens (see below) | Environment ids are `gym-<10 hex>`; each handle remembers the atespace it was created in. @@ -206,30 +207,65 @@ To share one `ate_env.Client` between the provider and other code, pass it in: Unknown keys are rejected at `create()` time, as are unknown keys anywhere in the config block. -### Services inside the sandbox: an injected daemon +### Services inside the sandbox: the NeMo runtime layer Some NeMo components talk to a service inside the sandbox instead of running commands through -`exec`: an exec server, a code-execution sandbox server, a tool daemon. The task images do not -contain it, and rebuilding hundreds of thousands of SWE-bench images to add it is not an option. -ate-env attaches such a daemon at runtime the same way it attaches its own guest: as a read-only -OCI image volume, a *runtime layer*, that the guest starts and supervises next to itself +`exec`: NeMo Skills' code-execution sandbox server (what `ns_tools` and the math recipes call for +`ipython` and `python` tool calls), NeMo Gym's own exec server (what the ECS Fargate provider +injects), a tool daemon. The task images do not contain them, and rebuilding hundreds of thousands +of SWE-bench images to add them is not an option. ate-env attaches such a daemon at runtime the +same way it attaches its own guest: as a read-only OCI image volume, a *runtime layer*, that the +guest starts and supervises next to itself ([docs/rootfs-images/RUNTIMES.md](../../docs/rootfs-images/RUNTIMES.md)). The provider implements nemo-gym 0.7's `SupportsSandboxEndpoint`, so Gym reaches the daemon from outside the cluster's actor network. +[`layer/`](layer/) is that layer for NeMo: one OCI image, built from scratch, that holds a +relocatable CPython 3.12 ([python-build-standalone](https://github.com/astral-sh/python-build-standalone)) +with hash-pinned wheels and two servers, each started with the layer's own interpreter so the task +image contributes nothing but its kernel and filesystem: + +| server | command (as a guest sidecar) | protocol | from | +|---|---|---|---| +| NeMo Skills local sandbox server | `/opt/nemo/python/bin/python3 -m nemo_runtime.sandbox_server --host 0.0.0.0 --port 6000` | `POST /execute` (`generated_code`, `language` = `ipython`/`python`/`shell`/`lean4`, `timeout`, `std_input`), IPython sessions by `X-Session-ID`, `GET /health` | `nemo_skills/code_execution/local_sandbox/local_sandbox_server.py`, vendored verbatim ([provenance](layer/third_party/nemo-skills/README.md)) | +| NeMo Gym exec server | `/opt/nemo/python/bin/python3 -m nemo_runtime.exec_server --host 0.0.0.0 --port 19542` | `POST /exec` (`cmd`, `timeout`), `POST /upload`, `GET /download?path=`, `GET /health` | the `EXEC_SERVER_SCRIPT` of `nemo_gym/sandbox/providers/ecs_fargate/engine.py`, vendored verbatim ([provenance](layer/third_party/nemo-gym/README.md)) | + +Both execute whatever they are sent, with the task image's user and filesystem, and neither +authenticates its caller; see the trust note below. `ipython` and `python` code runs on the +layer's interpreter and its packages (IPython, psutil, Flask and their dependencies, nothing +else); `shell` runs the task image's `/bin/sh`; a task image's own Python is used only when the +code calls it by path. + +Build and push it once per registry (linux/amd64 and linux/arm64; `layer/lock.sh` refreshes the +hash-pinned requirements): + +```bash +docker buildx build --platform linux/amd64,linux/arm64 --provenance=false --sbom=false \ + --push -t /nemo-runtime-layer: integrations/nemo-gym/layer +crane digest /nemo-runtime-layer: # templates take the digest, not the tag +``` + 1. **Put the daemon on the base template, once.** Every environment created from this base with a task image (`rootfs_image`) gets the daemon; the task images stay unmodified. + [`hack/runtime-layer-demo.sh`](hack/runtime-layer-demo.sh) does this and the steps below. ```bash - ate-env manifest template --template gym-base \ - --layer nemo=/nemo-daemon@sha256:=/opt/nemo \ - --sidecar '/opt/nemo/daemon --port 6000' \ + ate-env manifest template --template gym-nemo \ + --rootfs-image docker.io/library/python@sha256: \ + --layer nemo=/nemo-runtime-layer@sha256:=/opt/nemo \ + --sidecar '/opt/nemo/python/bin/python3 -m nemo_runtime.sandbox_server --host 0.0.0.0 --port 6000' \ --sidecar-readyz http://127.0.0.1:6000/health \ --snapshots-bucket gs:///ate-env/ | kubectl-ate create actor-template -f - ``` - The actor reports ready only once the daemon answers its readiness URL, so `create()` returns - a sandbox whose daemon is up. + Add a second `--sidecar`/`--sidecar-readyz` pair for the exec server when a recipe needs it. + The actor reports ready only once every sidecar answers its readiness URL, so `create()` + returns a sandbox whose daemon is up. Give the base a task image of its own + (`--rootfs-image`): the guest's image is Alpine-based and cannot run the layer's glibc + interpreter, so a base without one never bakes its golden snapshot and shows `ERROR` in + `kubectl-ate get actor-template`, although the templates derived from it for real task images + work. Point `create.template` (not a per-sandbox `provider_options` template, which selects the + base itself) at it, so each task image derives from the base and inherits the layer. 2. **Select the base and declare the port.** @@ -240,7 +276,7 @@ actor network. api_url: ate-env-api.ate-env:7777 # endpoint_url: http://ate-env-api.ate-env:7777 # default: api_url create: - template: gym-base + template: gym-nemo ``` ```python @@ -265,21 +301,30 @@ actor network. the guest's port 80. Set `connection.endpoint_url` when the code that calls the daemon reaches ate-env-api at another address than the provider does. +Verified 2026-10-21 on a GKE cluster (gVisor pool, Substrate `ddb46651`, ate-env-api with the port +proxy) with `hack/runtime-layer-demo.sh` and `tests/test_e2e.py -k runtime_layer`: environments from +unmodified `python:3.12-slim` and `debian:bookworm-slim` (no Python inside) were serving 4 to 5 s +after `create`, with readiness gated on both sidecars' `/health`; `POST /execute` answered from +inside the environment, through the header form and through the path form, an IPython session kept +its state across two calls, `language: python` ran on the layer's interpreter where the image had +none, and the exec server answered `POST /exec`. The layer image is 87 MB compressed. + Limits: - **HTTP/1.1 only.** The router tunnels HTTP to the port, not raw TCP. Streamed responses are flushed as they arrive and WebSocket upgrades pass through; gRPC to the daemon is not supported yet. - **The daemon must be self-contained.** A layer cannot rely on the task image's libraries: ship a - static binary, or put the interpreter and its packages in the layer. NeMo Skills' sandbox server - (Flask on CPython, port 6000, `POST /execute`, `GET /health`) fits only with a relocatable Python - inside its layer; Gym's ECS exec server needs `python3` in the task image. + static binary, or put the interpreter and its packages in the layer, as `layer/` does for the two + NeMo servers. The layer's CPython is a glibc build, so a task image needs glibc 2.17 or newer + (every manylinux-era distribution; not Alpine or another musl image). - **The daemon must listen on `0.0.0.0`.** The router reaches the port on the actor's address; a daemon bound to `127.0.0.1` only answers processes inside the sandbox, and requests through the endpoint fail with 502. - **No authentication in front of the daemon.** Anyone who can reach ate-env-api can call any port - in any sandbox through it. Keep ate-env-api inside the cluster, and give the daemon its own - credential when the sandbox runs untrusted code. + in any sandbox through it, and both NeMo servers run what they are sent. Keep ate-env-api inside + the cluster; the sandbox is the trust boundary, the same as with NeMo Skills' own sandbox + container and the ECS provider's exec server. - Port 80 belongs to the guest. The daemon shares the task image's user, environment and filesystem. - Each distinct task image yields one derived ActorTemplate on its base, created on first use and @@ -302,6 +347,13 @@ python3 -m venv .venv kubectl port-forward -n ate-env svc/ate-env-api 7777:7777 & SUBSTRATE_E2E_API_URL=127.0.0.1:7777 .venv/bin/python -m pytest tests/test_e2e.py -q # SUBSTRATE_E2E_TEMPLATE= picks the ActorTemplate (default: default-template) + +# The runtime-layer test needs a base template with the NeMo runtime layer (register one +# with hack/runtime-layer-demo.sh); it creates a sandbox on it, resolves endpoint() and +# calls NeMo Skills' sandbox server through ate-env-api's port proxy: +SUBSTRATE_E2E_API_URL=127.0.0.1:7777 SUBSTRATE_E2E_RUNTIME_TEMPLATE=gym-nemo \ + SUBSTRATE_E2E_RUNTIME_IMAGE=docker.io/library/python@sha256: \ + .venv/bin/python -m pytest tests/test_e2e.py -q -k runtime_layer ``` ## Benchmark diff --git a/integrations/nemo-gym/hack/runtime-layer-demo.sh b/integrations/nemo-gym/hack/runtime-layer-demo.sh new file mode 100755 index 00000000..ddbf2237 --- /dev/null +++ b/integrations/nemo-gym/hack/runtime-layer-demo.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Demo of the NeMo runtime layer (integrations/nemo-gym/layer): registers a +# base template that mounts the layer at /opt/nemo and starts NeMo Skills' +# sandbox server (and, with EXEC_PORT set, NeMo Gym's exec server) as guest +# sidecars, creates an environment from an unmodified task image on that +# base, and calls the servers three ways: from inside the environment, from +# outside through ate-env-api's port proxy with the routing headers, and +# through the path form. Deletes the environment on exit; keeps the template. +# +# Required: LAYER_IMAGE (digest-pinned nemo-runtime-layer), GUEST_IMAGE +# (digest-pinned ate-env-guest), SNAPSHOTS_BUCKET, ROOTFS_IMAGE +# (digest-pinned task image, e.g. python:3.12-slim@sha256:...) +# Optional: TEMPLATE (gym-nemo), ATESPACE (ate-env), SUBSTRATE_ENV_API +# (127.0.0.1:7777, ate-env-api reached by port-forward), PORT +# (6000), EXEC_PORT (unset: no exec server), FORCE=1 (recreate the +# template), ATE_ENV, KUBECTL_ATE +set -euo pipefail + +: "${LAYER_IMAGE:?set LAYER_IMAGE to the digest-pinned nemo-runtime-layer image}" +: "${GUEST_IMAGE:?set GUEST_IMAGE to the digest-pinned ate-env-guest image}" +: "${SNAPSHOTS_BUCKET:?set SNAPSHOTS_BUCKET to the snapshot object-storage URL}" +: "${ROOTFS_IMAGE:?set ROOTFS_IMAGE to a digest-pinned task image}" +TEMPLATE=${TEMPLATE:-gym-nemo} +ATESPACE=${ATESPACE:-ate-env} +export SUBSTRATE_ENV_API=${SUBSTRATE_ENV_API:-127.0.0.1:7777} +PORT=${PORT:-6000} +EXEC_PORT=${EXEC_PORT:-} +ATE_ENV=${ATE_ENV:-ate-env} +KUBECTL_ATE=${KUBECTL_ATE:-kubectl-ate} +PY=/opt/nemo/python/bin/python3 + +step() { printf '\n== %s\n' "$*"; } +now() { date +%s; } + +sidecars=(--sidecar "${PY} -m nemo_runtime.sandbox_server --host 0.0.0.0 --port ${PORT}" + --sidecar-readyz "http://127.0.0.1:${PORT}/health") +if [ -n "${EXEC_PORT}" ]; then + sidecars+=(--sidecar "${PY} -m nemo_runtime.exec_server --host 0.0.0.0 --port ${EXEC_PORT}" + --sidecar-readyz "http://127.0.0.1:${EXEC_PORT}/health") +fi + +step "template ${ATESPACE}/${TEMPLATE}: guest + NeMo runtime layer at /opt/nemo" +if "${KUBECTL_ATE}" get actor-template "${TEMPLATE}" --atespace "${ATESPACE}" >/dev/null 2>&1 && [ "${FORCE:-0}" != 1 ]; then + echo "exists; FORCE=1 recreates it" +else + [ "${FORCE:-0}" = 1 ] && "${KUBECTL_ATE}" delete actor-template "${TEMPLATE}" --atespace "${ATESPACE}" >/dev/null 2>&1 || true + # The base names a task image too (--rootfs-image): the guest's own image + # is Alpine-based and cannot run the layer's glibc interpreter, so a base + # without one never bakes its golden snapshot. Other task images derive + # from this base and inherit the layer. + "${ATE_ENV}" manifest template --template "${TEMPLATE}" --atespace "${ATESPACE}" \ + --guest-image "${GUEST_IMAGE}" --snapshots-bucket "${SNAPSHOTS_BUCKET}" \ + --rootfs-image "${ROOTFS_IMAGE}" \ + --layer "nemo=${LAYER_IMAGE}=/opt/nemo" "${sidecars[@]}" \ + | "${KUBECTL_ATE}" create actor-template -f - >/dev/null +fi +"${KUBECTL_ATE}" get actor-template "${TEMPLATE}" --atespace "${ATESPACE}" + +ID="nemo-$(date +%s)" +cleanup() { "${ATE_ENV}" delete "${ID}" --atespace "${ATESPACE}" >/dev/null 2>&1 || true; } +trap cleanup EXIT + +step "create ${ID} from ${ROOTFS_IMAGE} on ${TEMPLATE}" +t0=$(now) +"${ATE_ENV}" create "${ID}" --atespace "${ATESPACE}" --template "${TEMPLATE}" --rootfs-image "${ROOTFS_IMAGE}" +for _ in $(seq 1 60); do "${ATE_ENV}" "${ID}" shell 'true' >/dev/null 2>&1 && break; sleep 3; done +"${ATE_ENV}" "${ID}" shell 'true' >/dev/null +echo "serving after $(( $(now) - t0 ))s (readiness waited for the sidecars' /health)" + +step "inside the environment: POST /execute to 127.0.0.1:${PORT}" +"${ATE_ENV}" "${ID}" shell "${PY} -c \"import json,urllib.request; r=urllib.request.urlopen(urllib.request.Request('http://127.0.0.1:${PORT}/execute', data=json.dumps({'generated_code':'print(6*7)','language':'python','timeout':10}).encode(), headers={'Content-Type':'application/json'})); print(r.read().decode())\"" + +api="http://${SUBSTRATE_ENV_API}" +body='{"generated_code":"import sys; print(sys.version.split()[0]); print(6*7)","language":"python","timeout":10}' +step "outside, header form: ${api} with X-Env-Id/X-Env-Port" +t1=$(now) +curl -fsS -X POST "${api}/execute" -H "X-Env-Id: ${ID}" -H "X-Env-Atespace: ${ATESPACE}" -H "X-Env-Port: ${PORT}" \ + -H 'Content-Type: application/json' -d "${body}"; echo " ($(( $(now) - t1 ))s)" + +step "outside, path form: ${api}/v1alpha/atespaces/${ATESPACE}/envs/${ID}/ports/${PORT}/execute" +curl -fsS -X POST "${api}/v1alpha/atespaces/${ATESPACE}/envs/${ID}/ports/${PORT}/execute" \ + -H 'Content-Type: application/json' -d '{"generated_code":"print(\"ipython session\"); x = 40","language":"ipython","timeout":10}' -H 'X-Session-ID: demo'; echo +curl -fsS -X POST "${api}/v1alpha/atespaces/${ATESPACE}/envs/${ID}/ports/${PORT}/execute" \ + -H 'Content-Type: application/json' -d '{"generated_code":"print(x + 2)","language":"ipython","timeout":10}' -H 'X-Session-ID: demo'; echo + +if [ -n "${EXEC_PORT}" ]; then + step "NeMo Gym exec server on ${EXEC_PORT}: POST /exec" + # shellcheck disable=SC2016 # $(hostname) expands inside the sandbox, not here + curl -fsS -X POST "${api}/exec" -H "X-Env-Id: ${ID}" -H "X-Env-Atespace: ${ATESPACE}" -H "X-Env-Port: ${EXEC_PORT}" \ + -H 'Content-Type: application/json' -d '{"cmd":"echo hello from $(hostname); id -u","timeout":10}'; echo +fi + +step "done; deleting ${ID} (the template ${TEMPLATE} stays)" diff --git a/integrations/nemo-gym/layer/.dockerignore b/integrations/nemo-gym/layer/.dockerignore new file mode 100644 index 00000000..f1f4408f --- /dev/null +++ b/integrations/nemo-gym/layer/.dockerignore @@ -0,0 +1,16 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__pycache__/ +*.pyc diff --git a/integrations/nemo-gym/layer/Dockerfile b/integrations/nemo-gym/layer/Dockerfile new file mode 100644 index 00000000..bb9ba1d6 --- /dev/null +++ b/integrations/nemo-gym/layer/Dockerfile @@ -0,0 +1,66 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# The NeMo runtime layer: a relocatable CPython with NeMo Skills' local sandbox +# server and NeMo Gym's exec server installed, built as an OCI image with +# nothing else in it. Mounted read-only into an environment (ate-env manifest +# template --layer nemo==/opt/nemo), the guest starts either server as +# a sidecar; the task image contributes only the kernel and its filesystem. +# Same recipe as integrations/openhands/layer: python-build-standalone plus +# hash-pinned wheels installed with uv. linux/amd64 and linux/arm64. +FROM curlimages/curl:8.16.0@sha256:463eaf6072688fe96ac64fa623fe73e1dbe25d8ad6c34404a669ad3ce1f104b6 AS fetch +ARG TARGETARCH +ARG PBS_RELEASE=20261013 +ARG PYTHON_VERSION=3.12.15 +ARG PBS_SHA256_AMD64=5d546fc004fc2435fa142eb8f42ed8de5eb4725963dc22f7e6671014a80eb560 +ARG PBS_SHA256_ARM64=34a43010cc05982d8b5debbad7d09f7b1200117ebf018f7acfd656895e408205 +SHELL ["/bin/ash", "-euo", "pipefail", "-c"] +RUN case "${TARGETARCH}" in \ + amd64) arch=x86_64; sum="${PBS_SHA256_AMD64}" ;; \ + arm64) arch=aarch64; sum="${PBS_SHA256_ARM64}" ;; \ + *) echo "unsupported architecture ${TARGETARCH}" >&2; exit 1 ;; \ + esac; \ + f="cpython-${PYTHON_VERSION}+${PBS_RELEASE}-${arch}-unknown-linux-gnu-install_only.tar.gz"; \ + curl -fsSLo "/tmp/${f}" "https://github.com/astral-sh/python-build-standalone/releases/download/${PBS_RELEASE}/${f}"; \ + echo "${sum} /tmp/${f}" | sha256sum -c -; \ + mkdir -p /tmp/nemo; \ + tar -xzf "/tmp/${f}" -C /tmp/nemo + +FROM ghcr.io/astral-sh/uv:0.11.18@sha256:78bc42400d77b0678ba95765305c826652ed5431f399257271dda681d0318f03 AS uv + +FROM debian:trixie-slim@sha256:a29215f6a35e51e22adffa17f89e9d2ef06214e64a2bad10d765c46aea49f11f AS build +ARG TARGETARCH +ENV UV_NO_CACHE=1 UV_PYTHON_DOWNLOADS=never +COPY --from=uv /uv /usr/local/bin/uv +COPY --from=fetch /tmp/nemo/ /opt/nemo/ +COPY requirements.txt build-constraints.txt /tmp/lock/ +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN case "${TARGETARCH}" in \ + amd64) platform=x86_64-manylinux_2_31 ;; \ + arm64) platform=aarch64-manylinux_2_31 ;; \ + *) echo "unsupported architecture ${TARGETARCH}" >&2; exit 1 ;; \ + esac \ + && uv pip install --index-url https://pypi.org/simple --require-hashes \ + --python /opt/nemo/python/bin/python3 --python-platform "${platform}" --python-version 3.12 \ + --target /opt/nemo/python/lib/python3.12/site-packages \ + --build-constraints /tmp/lock/build-constraints.txt -r /tmp/lock/requirements.txt +# The entry points and the vendored servers live in site-packages, so +# python3 -m nemo_runtime. works wherever the layer is mounted. +COPY nemo_runtime/ /opt/nemo/python/lib/python3.12/site-packages/nemo_runtime/ +COPY third_party/ /opt/nemo/python/lib/python3.12/site-packages/nemo_runtime/third_party/ +RUN /opt/nemo/python/bin/python3 -c "import flask, psutil, IPython, traitlets; import nemo_runtime.sandbox_server, nemo_runtime.exec_server; nemo_runtime.sandbox_server.load_app()" \ + && find /opt/nemo/python -name __pycache__ -prune -exec rm -rf {} + \ + && /opt/nemo/python/bin/python3 -m compileall -q -j 0 --invalidation-mode unchecked-hash /opt/nemo/python/lib + +FROM scratch +COPY --from=build /opt/nemo/ / diff --git a/integrations/nemo-gym/layer/build-constraints.in b/integrations/nemo-gym/layer/build-constraints.in new file mode 100644 index 00000000..9b0ad7d3 --- /dev/null +++ b/integrations/nemo-gym/layer/build-constraints.in @@ -0,0 +1,15 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +setuptools diff --git a/integrations/nemo-gym/layer/build-constraints.txt b/integrations/nemo-gym/layer/build-constraints.txt new file mode 100644 index 00000000..14a9220d --- /dev/null +++ b/integrations/nemo-gym/layer/build-constraints.txt @@ -0,0 +1,6 @@ +# Generated by ./lock.sh from build-constraints.in for CPython 3.12, linux/amd64 and +# linux/arm64, manylinux_2_31. Do not edit; run ./lock.sh. +setuptools==84.2.1 \ + --hash=sha256:3ff8b3b5492a0aa66960f4419365a86b84e013070d4313e4d4a6894f5d64d6bc \ + --hash=sha256:ca140b45d5325a9478f880e351484f62fef15f336462e449ca0c03c873d589c2 + # via -r build-constraints.in diff --git a/integrations/nemo-gym/layer/lock.sh b/integrations/nemo-gym/layer/lock.sh new file mode 100755 index 00000000..52910c28 --- /dev/null +++ b/integrations/nemo-gym/layer/lock.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Regenerates requirements.txt and build-constraints.txt, the hash-locked +# dependency set the Dockerfile installs. It resolves for CPython 3.12 on +# linux/amd64 and linux/arm64, each restricted to manylinux_2_31 wheels (glibc +# 2.31), and fails when the two resolutions differ, because the Dockerfile +# installs one file on both. Needs uv. +# +# ./lock.sh +set -euo pipefail +cd "$(dirname "$0")" + +INDEX_URL=${INDEX_URL:-https://pypi.org/simple} +tmp=$(mktemp -d) +trap 'rm -rf "${tmp}"' EXIT + +compile() { # compile [uv args...] + local in=$1 out=$2 + shift 2 + uv pip compile --quiet --no-header --generate-hashes --index-url "${INDEX_URL}" \ + --python-version 3.12 "$@" "${in}" -o "${out}" +} + +for arch in x86_64 aarch64; do + compile requirements.in "${tmp}/${arch}.txt" --python-platform "${arch}-manylinux_2_31" +done +if ! diff -q "${tmp}/x86_64.txt" "${tmp}/aarch64.txt" >/dev/null; then + diff "${tmp}/x86_64.txt" "${tmp}/aarch64.txt" >&2 || true + echo "lock.sh: amd64 and arm64 resolve differently; split requirements.txt per architecture" >&2 + exit 1 +fi +compile build-constraints.in "${tmp}/build.txt" + +header='# Generated by ./lock.sh from %s for CPython 3.12, linux/amd64 and +# linux/arm64, manylinux_2_31. Do not edit; run ./lock.sh. +' +# shellcheck disable=SC2059 # the header is the format +{ printf "${header}" requirements.in; cat "${tmp}/x86_64.txt"; } >requirements.txt +# shellcheck disable=SC2059 +{ printf "${header}" build-constraints.in; cat "${tmp}/build.txt"; } >build-constraints.txt +echo "wrote requirements.txt ($(grep -c '==' requirements.txt) packages) and build-constraints.txt" diff --git a/integrations/nemo-gym/layer/nemo_runtime/__init__.py b/integrations/nemo-gym/layer/nemo_runtime/__init__.py new file mode 100644 index 00000000..25e936ce --- /dev/null +++ b/integrations/nemo-gym/layer/nemo_runtime/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Entry points of the NeMo runtime layer: the NeMo Skills sandbox server and +the NeMo Gym exec server, served from the layer's own CPython so a task image +needs nothing but a kernel. See integrations/nemo-gym/README.md.""" diff --git a/integrations/nemo-gym/layer/nemo_runtime/exec_server.py b/integrations/nemo-gym/layer/nemo_runtime/exec_server.py new file mode 100644 index 00000000..b6213e9d --- /dev/null +++ b/integrations/nemo-gym/layer/nemo_runtime/exec_server.py @@ -0,0 +1,44 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Serve NeMo Gym's exec server (POST /exec, /upload, GET /download, /health) +from the layer: ``python3 -m nemo_runtime.exec_server --host 0.0.0.0 --port 19542``. + +The vendored script reads TB_EXEC_BIND and TB_EXEC_PORT; this wrapper sets them +from its flags and runs the script as __main__.""" + +from __future__ import annotations + +import argparse +import os +import runpy +import sys +from pathlib import Path + +_SERVER = Path(__file__).resolve().parent / "third_party" / "nemo-gym" / "exec_server.py" + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--host", default="0.0.0.0", help="bind address; 0.0.0.0 so the router reaches it") + ap.add_argument("--port", type=int, default=19542, help="TCP port (NeMo Gym's default is 19542)") + args = ap.parse_args(argv) + os.environ["TB_EXEC_BIND"] = args.host + os.environ["TB_EXEC_PORT"] = str(args.port) + runpy.run_path(str(_SERVER), run_name="__main__") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/integrations/nemo-gym/layer/nemo_runtime/sandbox_server.py b/integrations/nemo-gym/layer/nemo_runtime/sandbox_server.py new file mode 100644 index 00000000..e5560e1c --- /dev/null +++ b/integrations/nemo-gym/layer/nemo_runtime/sandbox_server.py @@ -0,0 +1,65 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Serve NeMo Skills' local sandbox server (POST /execute, GET /health) from +the layer: ``python3 -m nemo_runtime.sandbox_server --host 0.0.0.0 --port 6000``. + +The vendored module is loaded from third_party/nemo-skills so it stays +byte-identical to upstream; this wrapper only chooses the bind address (Flask's +default is 127.0.0.1, which the router cannot reach) and the port.""" + +from __future__ import annotations + +import argparse +import importlib.util +import logging +import os +import sys +from pathlib import Path + +_SERVER = Path(__file__).resolve().parent / "third_party" / "nemo-skills" / "local_sandbox_server.py" + + +def load_app(): + """Import the vendored Flask application under its upstream module name.""" + spec = importlib.util.spec_from_file_location("local_sandbox_server", _SERVER) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load {_SERVER}") + module = importlib.util.module_from_spec(spec) + sys.modules["local_sandbox_server"] = module + spec.loader.exec_module(module) + return module.app + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--host", default="0.0.0.0", help="bind address; 0.0.0.0 so the router reaches it") + ap.add_argument("--port", type=int, default=6000, help="TCP port (NeMo Skills' default is 6000)") + args = ap.parse_args(argv) + os.environ.setdefault("WORKER_NUM", "0") + # `language: python` runs `python -c ...` through PATH. A task image's + # own python wins when it has one; the layer's interpreter is the + # fallback, so an image without python still executes code. + layer_bin = str(Path(sys.executable).resolve().parent) + os.environ["PATH"] = os.environ.get("PATH", "/usr/local/bin:/usr/bin:/bin") + os.pathsep + layer_bin + logging.getLogger("werkzeug").setLevel(logging.WARNING) + app = load_app() + # One process, many threads: each IPython session already runs in its own + # child process, and the guest restarts this server if it exits. + app.run(host=args.host, port=args.port, threaded=True, use_reloader=False) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/integrations/nemo-gym/layer/requirements.in b/integrations/nemo-gym/layer/requirements.in new file mode 100644 index 00000000..90a8233a --- /dev/null +++ b/integrations/nemo-gym/layer/requirements.in @@ -0,0 +1,22 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# What the NeMo Skills local sandbox server imports (third_party/nemo-skills), +# pinned to NeMo Skills' requirements/sandbox.lock at the vendored commit. The +# NeMo Gym exec server needs only the standard library. Run ./lock.sh after a +# change here. +flask==3.1.3 +psutil==7.2.2 +ipython==8.39.0 +traitlets==5.14.3 diff --git a/integrations/nemo-gym/layer/requirements.txt b/integrations/nemo-gym/layer/requirements.txt new file mode 100644 index 00000000..3fd96496 --- /dev/null +++ b/integrations/nemo-gym/layer/requirements.txt @@ -0,0 +1,295 @@ +# Generated by ./lock.sh from requirements.in for CPython 3.12, linux/amd64 and +# linux/arm64, manylinux_2_31. Do not edit; run ./lock.sh. +asttokens==3.0.2 \ + --hash=sha256:3ecdbd8f2cc195f53ccada3a613538bb5f9ef6f6869129f13e03c30a677b8fe2 \ + --hash=sha256:9da13157f5b28becde0bd374fc677dcd3c290614264eff096f167c469cd9f933 + # via stack-data +blinker==1.9.0 \ + --hash=sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf \ + --hash=sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc + # via flask +click==8.5.0 \ + --hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 \ + --hash=sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34 + # via flask +decorator==5.3.1 \ + --hash=sha256:4cbcdd55a6efadb9dbea26b858f4fb3264567b52d69ca0d25b721b553f60ea82 \ + --hash=sha256:f47fe6fdbd2edd623ecfe36875d37aba411624e2670dd395dddae1358689bb3c + # via ipython +executing==2.3.0 \ + --hash=sha256:15919cb5d667e5cb4e099511971d00d659573fff2dd5c4e6cd8b71636c7858d2 \ + --hash=sha256:736e859c9f8701f11fcf516856f26f562e04776387824b43a35a1dfe21c84122 + # via stack-data +flask==3.1.3 \ + --hash=sha256:0ef0e52b8a9cd932855379197dd8f94047b359ca0a78695144304cb45f87c9eb \ + --hash=sha256:f4bcbefc124291925f1a26446da31a5178f9483862233b23c0c96a20701f670c + # via -r requirements.in +ipython==8.39.0 \ + --hash=sha256:4110ae96012c379b8b6db898a07e186c40a2a1ef5d57a7fa83166047d9da7624 \ + --hash=sha256:bb3c51c4fa8148ab1dea07a79584d1c854e234ea44aa1283bcb37bc75054651f + # via -r requirements.in +itsdangerous==2.2.0 \ + --hash=sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef \ + --hash=sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173 + # via flask +jedi==0.20.1 \ + --hash=sha256:0fb16d86c4a4c73c37ba518c77419975e30fcc620658a8d14fbb5720cdd34142 \ + --hash=sha256:2f71208c3f9c1bca057c0e90d3f272aba44ace88fc4067d7587e9e069331b7e5 + # via ipython +jinja2==3.1.6 \ + --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 + # via flask +markupsafe==3.0.4 \ + --hash=sha256:007e1ffd9bf65bb6ee96df7b258fc632a4868dd5566037986c64781f35a36e98 \ + --hash=sha256:02fa4acbc6a3fc5c693c34d4dd8c1130b7fe99cc915181b0ddd6f72aeb296002 \ + --hash=sha256:03470d1a8268e692ecf79ecd565593e59d44219377a7ead61f1f1b94c1f7ff6b \ + --hash=sha256:04e7902ba80ee4bac1d50a549606527a1dcf0476cd81403db41099d3b60ec653 \ + --hash=sha256:051417f74bcaaefa316276e0ff723f541616ca51043d070da00249d9bddd3e3c \ + --hash=sha256:05295589e619b9bed252a86b532b8e27350abc372d18ba89b59375325e91ec1e \ + --hash=sha256:06de8ef6331f6e822c28d577dc8bf43fe398800477c49498f38fc38b67ff33fc \ + --hash=sha256:0764a13d34cae40db7bbf3a09b7e9b491bf4603e20b263a7a9d6b8e324975d0a \ + --hash=sha256:077293e425f28ec737dbcad442a71752e28f8ae27cde3d68acd1fb212091cd92 \ + --hash=sha256:0930db9bdc62d22944e10b066448bb65dc9abe9112880c7cab8da54db4284d5f \ + --hash=sha256:0cee7cb0f9a1b6892ea482237d9403b3d1b4603aee057d0ff01f0fac2d019a97 \ + --hash=sha256:0d9c47709875fdb321452056622e930c52afbc07a7d780762fbb8b4d91ce6fa4 \ + --hash=sha256:11935df9bf455ed0c04eb87bcd720f02b1fe5e02128a9430f23aed6f93336fc7 \ + --hash=sha256:12a606a492de952afcb43b59a14aaaaad120e708d3663dd0fdf2d738d427a691 \ + --hash=sha256:14bd2d845d62ab678eaf81da89d7b621b51756c72346745c1a594c09d49207a2 \ + --hash=sha256:15ba9e28640feef770374b116a6f019c21f52404aeabe516aa7f800587b98cfc \ + --hash=sha256:18a801868a884f216e784d7d14db2a4077143ce7610440aee2ce8f734e7cfcde \ + --hash=sha256:1c0df495a977d10460a94941799c72d5b5ab03d3858d949b55b5a66c8f371c99 \ + --hash=sha256:1caa2fa5a6184fb233153b35f654e6687bd555476f6170f29d8ee9be1a8b0af9 \ + --hash=sha256:1e1451fab512d1bcc3dc26988ec1edb0b82c2db909132872cd9356070a6b63df \ + --hash=sha256:1f1f9477e174582b0a1b583d60b66e1f2cf5d3fe12cee985e4aedf44766600e5 \ + --hash=sha256:2628d3a8cb648ecebb3c5d6b0a1052d400e4d8b7ac0fb786be8d285b50040d17 \ + --hash=sha256:26e9867520db70d37f7fb421a7f0d8adb40171011fb84ce869afa1a83370dfa8 \ + --hash=sha256:2a6ef68ae94aed8721934072b27a3b654ea2100b97e4ab864cf1489c90926fbc \ + --hash=sha256:2b2b1e18af909b448bb3cf9e3433366f7a8726271fc214e8b10e0f62a78c724b \ + --hash=sha256:2cb3dd71fc6be918ad4264346a8ed69485f9b7ed7bf35495d8e22807cd6b8bea \ + --hash=sha256:2d1b7d9308288661f56672b1b157d75fc536714d3638487bbea17b6318a78248 \ + --hash=sha256:2dad610540cb2e6272855c178f08ae9a1c7ac258a7fb71660553a5f104b42741 \ + --hash=sha256:2e5a7cd7fdd14fcb1ae5d7d8bf23d24fbd1daefd1fbca2580132e1ea75f098b5 \ + --hash=sha256:2e9ad7dd851bf45fab9f75cbff4cb493fee9979e8d8c7c9c3ee119022518edd6 \ + --hash=sha256:340cbb1957ba99929cbf19a75626d36ba1ae21d1730b287d1cf7f824a20c4fc7 \ + --hash=sha256:34bdde374c5932765d7dc685c4a1d191a3207852d67e8e0a9eb6ea85156181f1 \ + --hash=sha256:353bd63081912ab8cfa6a0c7d185934cdf8426f04c618bba6bc4b394f2069b67 \ + --hash=sha256:387d8cd30e69b3f0a72877b9ae717033396404e19095b17fe89753a981fda44f \ + --hash=sha256:3882fb412298575bae3b9c46868251f15cc69307359f87bb1b382e53d6e5a2c9 \ + --hash=sha256:38fc55594dab834470b6733dead2ee9e3f657fb0608c769dcafa0ba5ab52f45c \ + --hash=sha256:396ec4e65cc889f69786b3b89478b471cee5a3bcf468b9d9bb03e1a30fb291fc \ + --hash=sha256:39dbacefc411633db5b4378b066a9aca70a3d7e2922c9e578d825f844026eeba \ + --hash=sha256:3a93d9616ddecfb393727a0041a562cf0b15a244e20f2bd25efc7949be4c4f17 \ + --hash=sha256:3d23795802fc8bd72534836d64489bbf0f67c088959091bdb22e10735a5107bf \ + --hash=sha256:434139499bb20b502ed3baa1f169e618f924a97e7a777fea1a49446d80106cf6 \ + --hash=sha256:436e3ffc6310d3c41878c601db29098102fe5d8a467c49da4a4125254e0980f2 \ + --hash=sha256:489505b03f692c3f376394e49194fa7a7f9e8558d6e293a7056a0032b0c38163 \ + --hash=sha256:4a540e2d3192792fc84eced57bef37851ccb2b41f73291bb17408eea77bcd278 \ + --hash=sha256:4a7cdc2a420ca01058182da4253329764d4bfa055564d1eced90e6ba1e8b1d3d \ + --hash=sha256:4bced6e2a6dba6a28f7dd3c6ce14df1b2dd495923f16ea484cad03decd463b2b \ + --hash=sha256:4cf3468d5ec187ffffcaca8e61929a37448f215dafc1386a12c750a72fe53634 \ + --hash=sha256:4e2c4809c14559aa7ef426f27fb35afbb38104c349a903bf8f3600456764bb38 \ + --hash=sha256:4ed644d75aa94a2baf7ec3a96eaa160ea58c742eb9d27c6506053c5c40fc84ed \ + --hash=sha256:4f6e0852a0283b1b1fd776eeb7b766a5f440b3e2bd31ab51af3b400585f3965c \ + --hash=sha256:5066b244f576f91afc8ee3ba029a89f99d39c79b1853fe9d39bea9f0afbec148 \ + --hash=sha256:5086f9975abb1ab531ee6afca1761e4b59a19b446f3f6522ed776963228cfe5a \ + --hash=sha256:50b5bedc9ed8a94fc8857a42ef4f84a81ea88f8d4f05dc8705fb23ee6d8dcca7 \ + --hash=sha256:52704c5d36eb6dda8866493decd61111fff86244c9b1ad225ca01b9e91e5970f \ + --hash=sha256:55ffd6ce583d97dc71dc92e930324c8c0d25aea7e3ade6ae54ef77cedb096811 \ + --hash=sha256:569d65055d367e3dcdf30c3f41119467b73d9ee9faf332bdf40402644f5ac08e \ + --hash=sha256:57f9947a7e57a081c1e3e0a2dd0d2dcf290a4531450e6f611e30084c222a7295 \ + --hash=sha256:5989cb26b2e1efc6a42216a9f6b5ee495ce5ace2e5b352a9af489976b32d1ee2 \ + --hash=sha256:5c22873ad1f0532ba40fa1727f3c0fc1bbbaab6d373d4cbe3f0dc74b2e2521c7 \ + --hash=sha256:5e8b3d0b18fd623afa12ecb2ce8d8becef69f9b5440c6330c7972200e0bb84b0 \ + --hash=sha256:61631e08084be9e21a8967ec3139c7616ed7c5e9368e05c86d1b39562c8a57b6 \ + --hash=sha256:64511c54db4e4987aef4c41923235927428729e8174c5dba488429be70a998ed \ + --hash=sha256:6669c1bf34080161ce49c589cc512ef24d4c704ac9d2b2d3667f519c60418378 \ + --hash=sha256:672d207103e6b16ca098611b0f9efad6bc00afd47c03d6ef62186495ca677dc0 \ + --hash=sha256:6768d67d1bce64270e0fdc2e69309d68b9b18ae56ddf6c711d168e9d051c2cac \ + --hash=sha256:6a45c3d514f2436064db00d7fc8778d888f0236ebfed649b53d13a59e69ad51b \ + --hash=sha256:6bd9e1788e15bfcf6a9082de42e30387e7b85d211ab21e57a939bb8cfaaf8d96 \ + --hash=sha256:6d2a9efe686f9de00d0d1ea32a4a5a86d558a2277501bd78d964214eab625e59 \ + --hash=sha256:6da83a088f8ef93b2d483a8232a4dbf4d69d3d8496b568a03c56becac43e1808 \ + --hash=sha256:7018d4af1cd272e847aa5917983ab5e83e4f6579f9dbfecd4a79c0ca80b144c2 \ + --hash=sha256:71f88e749ea29f67f21f3b36433c1dc54c7729ed2a6d9e2da2e0d9e0d7b224eb \ + --hash=sha256:737c9c3981998eba27f11786f84fddcbabc74068b72a4a1f454ea02094b57b65 \ + --hash=sha256:73e77980c7207854f00fc4e71fb1626868d5740ab4012623d55c7a99ad122a72 \ + --hash=sha256:799c39bdf5e2f1292fedd3009f7b3c9e760f10b2420cb9638d56920840ff6db8 \ + --hash=sha256:7a83aa6e4805df46fed18e989d3d16f86ef60cb50bbc8d9ce3a6be89165fbf6e \ + --hash=sha256:7d3391b2188d18737cb2fa147028b1096236eaa7e156446c650a489fa2cadc91 \ + --hash=sha256:7e1636da3d8dfc220b6dd10264db5f2b165e4888c4518594898fbe381049af8a \ + --hash=sha256:805c8b84534fa10891890f0e4be39f3a99e94615d93e8836bf9fa1fdca2feeb2 \ + --hash=sha256:811d02d5122171c1941357efd8f9bf4ffe907b7f0a1a4e729a880e4be3f46e3e \ + --hash=sha256:8138eb83940ec7299024d92d4dee45f601b9e6c5ffde9d25f4e35e326203c707 \ + --hash=sha256:83b3944fea42a8400edf92fd1770fb8d0d4f7de651353bd2d8525a92dba69a21 \ + --hash=sha256:849dd2bb0e5e4ab2b71c7191726a4a8d5aa8a610daa584728cbee0b710ddc4ef \ + --hash=sha256:8698d70a8081ee8c090dbb394768b5789a1da8b131b5499f89d071dd3cfaf6be \ + --hash=sha256:8781a792a070cf2bd1b86d3aa943894115faaba6e88122a7bf32d62072742453 \ + --hash=sha256:88d59b473bfb03259722600839af9bbd7fa13a2eb514beefeedb95997882f69a \ + --hash=sha256:8909c2f1c6dd65e054ac4b573a91c8384d1492281e55d82d159d653f7a13adf6 \ + --hash=sha256:8965520ac587c94a4ac48b729be3d8b8de00af39699b17585dfb599babe77977 \ + --hash=sha256:8b5d563170ff8ba3181caa967c99a3c804d1dedb702c7cb93a6a7c32247da978 \ + --hash=sha256:8e124f974786f831d6043728e38296969d3579db8896fe004682f5758e613581 \ + --hash=sha256:8f0fac8b13d14bb06c68195f849371924ae53dd7b1c00fed24650f704383b692 \ + --hash=sha256:9240187afb63d2f9ddc3e032c670356fe941f6e20662ea168a5dc3f1f317e1b3 \ + --hash=sha256:925f929d6b59a8b3f8b8c6ac363cd0af7eecc81efb3071770b3c6717c450a369 \ + --hash=sha256:9348cbb300d224fe3b89793262cb093504d4ae927004468463f745188a193e4a \ + --hash=sha256:9388003072b95f2f1e3fd908604194d653ba21330d811961a78b7da1a77e9e36 \ + --hash=sha256:9438a2648b2195980cb2dd8e53ed7b8df91319e2d0b70ae61a9e1d1bc8d3bec9 \ + --hash=sha256:94e4c421742086aeee4c32a506eec8859d7634aad943f7e6aacf70f813478768 \ + --hash=sha256:94f5407f7bc64fa6463906b896f9904beeeb7dd8dc116ee8e9056c8714ff9916 \ + --hash=sha256:971a3bbb75d97ae4e2e8f7d4834236f86f85f0c85e04ab2e191db1123b04f80b \ + --hash=sha256:9e227f3dbe6bde7491cf0a9965d00b88c6b1a4a95d11480ddf88bb96d397c19f \ + --hash=sha256:9e25feb9e330b63edb0278a0acdf85e50d0cb0fbf49c3084abbe4e24ae195346 \ + --hash=sha256:9f098115c247e11d138ab83a28fa0323c77015007ea2df73ba5fd714dfefd67c \ + --hash=sha256:a18f38cafc329bac5e3c2b96c765b4c96d3d103421ed22ab7988c1e3fce27464 \ + --hash=sha256:a4bbd2d87dd233b9fc5812160c3d0ffbe42edc22a26ce0469f58479ede633fe9 \ + --hash=sha256:a5fcffb37e602b0b3c1638a97746b9b96125caa9bcf6fa41d337a9261de231ee \ + --hash=sha256:a8e9f292fcda89b324f2f5c91d13f1424a153e40fc2756f38ee23b15835ff300 \ + --hash=sha256:a9f54054101545a9a9cccefddf54316aa6e4491611fcbef9e91b3b6bebec04f6 \ + --hash=sha256:aa2c838cc024642cc04c6854232f32b43e5e22833dd11119c1766c7873b8370d \ + --hash=sha256:ac0c7c9f1609b0c4c114feb1d7a3409564c7fb77e360bed9e97e5d25dfeaf868 \ + --hash=sha256:add96447a86d205ab616665d53b2950ee81083757f56e6ea833c8b2917646b46 \ + --hash=sha256:ae9dcb8fbe244cb82f8a6458b455b927a03685e383d9bacf1ea5ce180b96dc97 \ + --hash=sha256:b4a635a0487774f841cb1fb62e907e7195cc95bc761e053184b8acc3ceb20733 \ + --hash=sha256:b4d12837e0203bbace818ff4a7461afdcd78bcd782351cea148139180d7bcffe \ + --hash=sha256:b61687d0828e72bf5cda24a2690188f37170bd31c9359ac97e4e66569f120a16 \ + --hash=sha256:b807e598953730f82e4eae3bd30f6a122cf6b31c398c6b504c0e04c13c170429 \ + --hash=sha256:b8cd1f918b26fd7b1832ece557cc18f2d8747309ff8b3f0ef9d4250c5ad67a39 \ + --hash=sha256:b91cc9d336957239ff200f30097e6fea2dc6d6fb3c81e853eaa09eac904fd894 \ + --hash=sha256:bd3ce56ae2cbae3ba82b683bc425cd7e48d2ed8b10f3e818186b6f5646d9271c \ + --hash=sha256:be6cb0c799abb0e2ba3e618e6d28ddddf7e485f6c2ce938dfa237daf3905072c \ + --hash=sha256:befb4158af32106b9a93db8d6d1d1cbbd418c0d5aca0cabb7b1780abf0c89169 \ + --hash=sha256:bf053da3c97a4bc5ecfbb218cdd2983febd91c617be8367d139882aa11e490aa \ + --hash=sha256:c02e8f18bdedba082cef725942ac823b9b60656db07f7e265cb31618dfd00d77 \ + --hash=sha256:c1bc67752d5f21013cfe430df4062441714eab79f65a6a05e01505957e9c35fe \ + --hash=sha256:c61750fadcd119d0825bcb7d7d675dd264dcc89cc05292aab5be68ebdbb374ad \ + --hash=sha256:c90d5b3d4e944e065a301d741b3c1d784f6bd1f503aa68b4967e32b2ba313d85 \ + --hash=sha256:c9a7f43c0b202b334cc9184af09bb8f21d3a209e038efaf106936fb69e6b026e \ + --hash=sha256:cb96e6e088d6cf71c1ea977510948320234824cf226e32f6f6e044f7a9c82b34 \ + --hash=sha256:cf63c214fe879a65e69a386f915e36104fc84254ab141240f8854602d8e0be2a \ + --hash=sha256:d1aca03ede943eb80ab3d63bb082c84b7aab85ea83bd0fd0c200260945fb49d9 \ + --hash=sha256:d2e56fd3b00222722abfb3f5f0759ddbae4b90811b5ad4343c64030ad1bde70c \ + --hash=sha256:d5f93ebbeb8032d47e349328ec8662d973d9b05a70b3c35df1f91fe419b84749 \ + --hash=sha256:d882a373d8093c2941e01291b7ced96e9cbe4781da9a7751ca7e6c70385e5214 \ + --hash=sha256:d920abdfa61279ba1a2ef9484aab07bf03331f8c08a10120fa332353d06e6932 \ + --hash=sha256:da2af0d7aebfc2074080d72efa6ab8317c62481ef1f896f65d9999c1c01f4494 \ + --hash=sha256:dd8ea6ebee7aedbf7c749fa80521d9ccf1ba473e0d1e14805caafbaad281c889 \ + --hash=sha256:de8b364c423ef0a4bad9069657d617f9a5d2b2062457a89b1fa16ee199c399c1 \ + --hash=sha256:df1ae86ff54725a01fa1a0510b914ca53a161b7050be74f6204e24aded5971d0 \ + --hash=sha256:dff05cb7016dff1e9fd68f4122c127b65dfc59de5306cfb7ad92f956f230bee2 \ + --hash=sha256:e1a622f13970d81f95d0c72f9dc090dce9085fccfa4c9f2174377ee32bd15786 \ + --hash=sha256:e49fb0d1ce92cfa0cb198cc5b1b11cdf9d0638658e2a2db2687e39db7c87fc78 \ + --hash=sha256:e5c802729725bd07e2bc3ab7b76dc7e0bbfc53129d8f1eb1c002c24cf774717e \ + --hash=sha256:e841068dc0be4cb6dfb5c890eb88cbdcff2f4a332393c7ec94e8e618bd32c1a8 \ + --hash=sha256:e916035e3e9930cbdfdd10abf48861340221857f45509565898e012263f7b289 \ + --hash=sha256:eba154571c16e032112afac0dc2dfe9e63c2ceb7aedd07bb7eecf2ce26d4dd4c \ + --hash=sha256:f03460ff076f70ab595bb45a0205ccea1971443575b6920c52e755dec2b3fbfe \ + --hash=sha256:f0ec3b750b59375eab5b0fb2b9254810c00a3375be6d789899f1055a1d556237 \ + --hash=sha256:f291bcf42ae98eb5107edb162c3c998b4a89648fd8e99ed4cbd12705292788cd \ + --hash=sha256:f61efe1d2fe0de16158a5fe1d1cf3c14bdb6aecd54d8938fd26512c525c1f624 \ + --hash=sha256:f68edfc67aabac33708941f26f22a7b8e9f81429bc0cf249fcf7d66b23af8d19 \ + --hash=sha256:fa95848c929b6a75f6848d3c9793e59db365ee436776e57db835cdbfa79ba977 \ + --hash=sha256:fd9f8797427910198f95bced71ddfed61130d7e349213bfb8466c9c99e2c46a8 \ + --hash=sha256:fdb4ca07ab75ffadab4a8b135ad59cdbb3156b99310f3d565370da74a15d6bd3 + # via + # flask + # jinja2 + # werkzeug +matplotlib-inline==0.2.2 \ + --hash=sha256:3c821cf1c209f59fb2d2d64abbf5b23b67bcb2210d663f9918dd851c6da1fcf6 \ + --hash=sha256:72f3fe8fce36b70d4a5b612f899090cd0401deddc4ea90e1572b9f4bfb058c79 + # via ipython +parso==0.8.7 \ + --hash=sha256:a8926eb2a1b915486941fdbd31e86a4baf88fe8c210f25f2f35ecec5b574ca1c \ + --hash=sha256:eaaac4c9fdd5e9e8852dc778d2d7405897ec510f2a298071453e5e3a07914bb1 + # via jedi +pexpect==4.9.0 \ + --hash=sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523 \ + --hash=sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f + # via ipython +prompt-toolkit==3.0.53 \ + --hash=sha256:01c0891d7f9237d5e339f7d3e42cdae80b7534abb1c7c0e3352efba6231492f2 \ + --hash=sha256:9ec8a0ad96d5c56148b3f914aa79c1564c3fde5d2e6b876e7bc327e353cf8fa6 + # via ipython +psutil==7.2.2 \ + --hash=sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372 \ + --hash=sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9 \ + --hash=sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841 \ + --hash=sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63 \ + --hash=sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979 \ + --hash=sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a \ + --hash=sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b \ + --hash=sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9 \ + --hash=sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee \ + --hash=sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312 \ + --hash=sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b \ + --hash=sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9 \ + --hash=sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e \ + --hash=sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc \ + --hash=sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1 \ + --hash=sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf \ + --hash=sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea \ + --hash=sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988 \ + --hash=sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486 \ + --hash=sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00 \ + --hash=sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8 + # via -r requirements.in +ptyprocess==0.7.0 \ + --hash=sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35 \ + --hash=sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220 + # via pexpect +pure-eval==0.2.4 \ + --hash=sha256:260c2774686e651b79f8b8e7fc9d80b3599ea6a66334b47d5f4abb69fc2c0ea1 \ + --hash=sha256:96cae060a313cfaad51bb761278bfb0e62dc0248d9315a81173752dc546cd37a + # via stack-data +pygments==2.21.0 \ + --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 \ + --hash=sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c + # via ipython +stack-data==0.6.3 \ + --hash=sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9 \ + --hash=sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695 + # via ipython +traitlets==5.14.3 \ + --hash=sha256:9ed0579d3502c94b4b3732ac120375cda96f923114522847de4b3bb98b96b6b7 \ + --hash=sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f + # via + # -r requirements.in + # ipython + # matplotlib-inline +wcwidth==0.9.3 \ + --hash=sha256:16d09df90b3a8f799ea14c726a41d9b906c3104f95f46d158e1ebe45cf4d547b \ + --hash=sha256:2f875a4c9e62df2ecfa70d99b24c551f2245609858e179c3a40e74631b2d08a4 \ + --hash=sha256:3b119ef60177a92b4f298b80e98464c2922a1fba3ed77fe9a2bcb69edf1b9af4 \ + --hash=sha256:3e3e31d6c17d6da3051f7eb854d5e4bebdc52a08a7a52adc2da6f15af51304e8 \ + --hash=sha256:457f986e4867111cb037ccbf0d97ae100b1ecad030d43a020e7207f1e29649d5 \ + --hash=sha256:52a13869704d43a4add59c68ebc892198e53063883114783bf0b7457e250e3f4 \ + --hash=sha256:537ea00b6792bb243cd09582322c54dbc848879e5d34ea4c5fee0147529685ab \ + --hash=sha256:569a0cdb826d3097b6dbacfaa45aa0002d72a2ba6fa8a63251ea43ac4dd7b6a4 \ + --hash=sha256:5d3b0b6c37f40e8d8298941ec373a426a892bb9a514328abcc9226cdc60ae6eb \ + --hash=sha256:658dcbce546e7f28ee361f88e4a8e591dd56f46bff6084e11b4ad8310ed99946 \ + --hash=sha256:6f3e855d3a09af8b7b6d56e76bc270681032f8a37d527a8d5707a441958e3730 \ + --hash=sha256:70a2ca239b85a219324968993c76c117fe423d4e1a7c9d860c999c66192f2570 \ + --hash=sha256:7a95d8c784cbccbbcb808eaadfd156cd282b2bdd95e89b32505e3443381ce61d \ + --hash=sha256:80557db1f77148b78e1b3c2a9cc4ebc4feabef42aaf69a555565057031c32367 \ + --hash=sha256:9470169fcb4e5340269175ebb3642dd5bfc8f82da8c86f29fc1fefc6c2b9bc1b \ + --hash=sha256:9ab8bfee107e08de09e15f9f6afb9fe0f64c8d64fd296d0d8557bafe71ea3f93 \ + --hash=sha256:9e1cbe18c18d152e1e68e4e2561dab6c2bc988f8fd337a1fb2e1e0809c3fa568 \ + --hash=sha256:a44acbb23b525070a0b8c343dd7cab7a4020aff73b38804f7a66d6a66e16e3fc \ + --hash=sha256:a7c0405a3bf3c564e481d6b2d55c99cc231bb68cf8f4a0995944bbce5ebf1a5d \ + --hash=sha256:aab90ab489850f87633922c3064f6d0442d8f6ae5407a62f82bdb1f424179c05 \ + --hash=sha256:c12abd75ffcbad417e7770a6feb82413c102ccce5c435068c39a82e2218b00b3 \ + --hash=sha256:c1bbc8ebf133422cadfefea95a9f89d6c9e9a1ae19930bef7862a9b59b20bbad \ + --hash=sha256:c4531d93344441a7948e665854f0c40e3a8342ee106436d1531efa7a9fa7641e \ + --hash=sha256:dfe7cfbd4cb59db9fe0931bd6091ab4a95b3acc5d56bfb50fa9fef0ccba5dc5c \ + --hash=sha256:e3c9069c54a2da0bf4f6c7e65a735cb1da0f47601a15cb5fef7e4b6ffca4cf60 \ + --hash=sha256:e9362be098c43d076aa43754face3cc0180fa81cea9ba304240b15490f2617e8 \ + --hash=sha256:eef39f3260991d8767189d296f0116fa4b45ad9c3a9a16d5af9187fb66b27564 \ + --hash=sha256:f11789e520ade8017fa93b01a4b8985571bb433eaefc2f857b57e36875e2e1d8 \ + --hash=sha256:f2f943c4ef14e9baba730740265f4e5bf5fc8473550e36cd92828cd5663561a6 + # via prompt-toolkit +werkzeug==3.1.9 \ + --hash=sha256:55ca7c70a75689be937aa27f8ff4b018f06ff4838fc73045560bf0f5a1291060 \ + --hash=sha256:6392e50c78460ba618e5b21f08a71f59c99ce99cdc6cf6e3dd7e6ccca8754fab + # via flask diff --git a/integrations/nemo-gym/layer/third_party/nemo-gym/LICENSE b/integrations/nemo-gym/layer/third_party/nemo-gym/LICENSE new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/integrations/nemo-gym/layer/third_party/nemo-gym/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/integrations/nemo-gym/layer/third_party/nemo-gym/README.md b/integrations/nemo-gym/layer/third_party/nemo-gym/README.md new file mode 100644 index 00000000..f6336b4c --- /dev/null +++ b/integrations/nemo-gym/layer/third_party/nemo-gym/README.md @@ -0,0 +1,13 @@ +# NeMo Gym exec server + +`exec_server.py` is the `EXEC_SERVER_SCRIPT` string from +https://github.com/NVIDIA-NeMo/Gym, path +`nemo_gym/sandbox/providers/ecs_fargate/engine.py`, at commit `d4fd5e8` +(2026-10), written out as a file; the text is unchanged. Licensed by NVIDIA +under Apache-2.0 (`LICENSE`). It is the zero-dependency HTTP exec server the ECS +Fargate provider injects into its sandboxes: `POST /exec` with `cmd` and +`timeout`, `POST /upload` with `path`, base64 `content` and optional `mode`, +`GET /download?path=`, `GET /health`. It reads `TB_EXEC_BIND` and +`TB_EXEC_PORT`; `nemo_runtime.exec_server` sets them from its flags and runs +the script. To update, copy the string out of the new `engine.py` and bump the +commit here. diff --git a/integrations/nemo-gym/layer/third_party/nemo-gym/exec_server.py b/integrations/nemo-gym/layer/third_party/nemo-gym/exec_server.py new file mode 100644 index 00000000..7d342bb8 --- /dev/null +++ b/integrations/nemo-gym/layer/third_party/nemo-gym/exec_server.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Zero-dependency HTTP exec server for sandbox containers.""" +from __future__ import annotations +import base64, json, os, shutil, subprocess +from http.server import BaseHTTPRequestHandler, HTTPServer, ThreadingHTTPServer +from urllib.parse import parse_qs, urlparse +_BASH = shutil.which("bash") +_PORT = int(os.environ.get("TB_EXEC_PORT", "19542")) +_BIND = os.environ.get("TB_EXEC_BIND", "127.0.0.1") +class _H(BaseHTTPRequestHandler): + def log_message(self, fmt, *a): pass + def do_GET(self): + p = urlparse(self.path) + if p.path == "/health": self._ok({"ok": True}) + elif p.path == "/download": + qs = parse_qs(p.query) + paths = qs.get("path", []) + if not paths: self._err(400, "missing ?path=") + else: self._dl(paths[0]) + else: self._err(404, f"not found: {p.path}") + def do_POST(self): + p = urlparse(self.path) + body = self._body() + if p.path == "/exec": self._exec(body) + elif p.path == "/upload": self._up(body) + else: self._err(404, f"not found: {p.path}") + def _exec(self, b): + cmd = b.get("cmd") + if not cmd: self._err(400, "missing 'cmd'"); return + t = b.get("timeout", 300) + try: + cp = subprocess.run(cmd, shell=True, executable=_BASH, capture_output=True, timeout=t) + self._ok({"stdout": cp.stdout.decode("utf-8", errors="replace"), + "stderr": cp.stderr.decode("utf-8", errors="replace"), + "rc": cp.returncode}) + except subprocess.TimeoutExpired: + self._ok({"stdout":"","stderr":f"timed out after {t}s","rc":124}) + except Exception as e: + self._ok({"stdout":"","stderr":str(e),"rc":-1}) + def _up(self, b): + path, c = b.get("path"), b.get("content") + if not path or c is None: self._err(400, "missing path/content"); return + try: + data = base64.b64decode(c) + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + with open(path, "wb") as f: f.write(data) + m = b.get("mode") + if m: os.chmod(path, int(m, 8)) + self._ok({"ok": True}) + except Exception as e: self._err(500, str(e)) + def _dl(self, path): + if not os.path.isfile(path): self._err(404, f"not found: {path}"); return + try: + with open(path, "rb") as f: data = f.read() + self.send_response(200) + self.send_header("Content-Type", "application/octet-stream") + self.send_header("Content-Length", str(len(data))) + self.end_headers(); self.wfile.write(data) + except Exception as e: self._err(500, str(e)) + def _body(self): + n = int(self.headers.get("Content-Length", 0)) + if n == 0: return {} + try: return json.loads(self.rfile.read(n)) + except Exception: return {} + def _ok(self, obj): + p = json.dumps(obj).encode() + self.send_response(200) + self.send_header("Content-Type","application/json") + self.send_header("Content-Length",str(len(p))) + self.end_headers(); self.wfile.write(p) + def _err(self, code, msg): + p = json.dumps({"error": msg}).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(p))) + self.end_headers(); self.wfile.write(p) +if __name__ == "__main__": + s = ThreadingHTTPServer((_BIND, _PORT), _H) + print(f"exec_server on {_BIND}:{_PORT}", flush=True) + try: s.serve_forever() + except KeyboardInterrupt: pass + finally: s.server_close() diff --git a/integrations/nemo-gym/layer/third_party/nemo-skills/LICENSE b/integrations/nemo-gym/layer/third_party/nemo-skills/LICENSE new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/integrations/nemo-gym/layer/third_party/nemo-skills/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/integrations/nemo-gym/layer/third_party/nemo-skills/README.md b/integrations/nemo-gym/layer/third_party/nemo-skills/README.md new file mode 100644 index 00000000..346dd910 --- /dev/null +++ b/integrations/nemo-gym/layer/third_party/nemo-skills/README.md @@ -0,0 +1,13 @@ +# NeMo Skills local sandbox server + +`local_sandbox_server.py` is copied verbatim from +https://github.com/NVIDIA/NeMo-Skills, path +`nemo_skills/code_execution/local_sandbox/local_sandbox_server.py`, at commit +`bcf059a` (2026-10). Licensed by NVIDIA under Apache-2.0 (`LICENSE`). It is the +Flask application behind NeMo Skills' `LocalSandbox` client: `POST /execute` +with `generated_code`, `language` (`ipython`, `python`, `shell`, `lean4`), +`timeout`, optional `std_input` and `session_id`; `GET /health`; and the +IPython session routes. `nemo_runtime.sandbox_server` loads it and serves it +on the host and port the sidecar command names. To update, copy the new file, +bump the commit here, and re-lock `requirements.in` against NeMo Skills' +`requirements/sandbox.lock`. diff --git a/integrations/nemo-gym/layer/third_party/nemo-skills/local_sandbox_server.py b/integrations/nemo-gym/layer/third_party/nemo-skills/local_sandbox_server.py new file mode 100644 index 00000000..9faf4a2a --- /dev/null +++ b/integrations/nemo-gym/layer/third_party/nemo-skills/local_sandbox_server.py @@ -0,0 +1,794 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import io +import logging +import multiprocessing as mp +import os +import re +import resource +import signal +import subprocess +import tempfile +import threading +import time +import traceback +from contextlib import redirect_stderr, redirect_stdout + +import psutil +from flask import Flask, request +from IPython.terminal.interactiveshell import TerminalInteractiveShell +from traitlets.config import Config + +app = Flask(__name__) + +# Identify worker and configure logging so messages are visible per-worker +worker_id = os.getenv("WORKER_NUM", "unknown") +logging.basicConfig( + level=logging.INFO, + format=f"[worker {worker_id}] %(asctime)s %(levelname)s: %(message)s", +) + +# ============================================================================= +# Network Blocking Configuration (Defense in Depth) +# ============================================================================= +# When NEMO_SKILLS_SANDBOX_BLOCK_NETWORK=1, we use TWO layers of protection +# to prevent LLM-generated code from making network requests: +# +# LAYER 1: libblock_network.so (via /etc/ld.so.preload, set by start-with-nginx.sh) +# - Intercepts socket() syscalls at the C library level +# - Blocks any NEW PROCESS that exec()'s (curl, wget, python3 subprocess, etc.) +# - System-enforced by the dynamic linker - user code CANNOT bypass it +# - Limitation: Doesn't affect forked processes (no exec = no linker = no preload) +# +# LAYER 2: Python socket patch (below, in shell_worker function) +# - Patches socket.socket and _socket.socket at Python level +# - Blocks code running in the IPython shell_worker (which is forked, not exec'd) +# - Limitation: Only works for Python code in the same process +# +# WHY BOTH ARE NEEDED: +# | Attack Vector | Python Patch | ld.so.preload | +# |--------------------------------------------|--------------|---------------| +# | socket.socket() in IPython | Blocked | (forked) | +# | import _socket; _socket.socket() in IPython| Blocked | (forked) | +# | subprocess.run(["curl", url]) | Can't help | Blocked | +# | subprocess.run(["python3",...], env={}) | Can't help | Blocked | +# | requests.get() in IPython | Blocked | (forked) | +# +# Neither layer alone is sufficient - together they cover all adversarial scenarios. +# ============================================================================= +BLOCK_NETWORK = os.getenv("NEMO_SKILLS_SANDBOX_BLOCK_NETWORK", "0") == "1" +if BLOCK_NETWORK: + logging.info("Network blocking mode enabled") + + +# Worker that runs inside the shell process and owns a TerminalInteractiveShell() +def shell_worker(conn): + # LAYER 2: Python-level socket blocking for IPython sessions + # The shell_worker is forked (not exec'd) from the uWSGI worker, so it does NOT + # get the ld.so.preload library loaded. We must patch Python's socket module directly. + # This blocks: socket.socket(), _socket.socket(), requests.get(), urllib, etc. + if BLOCK_NETWORK: + import _socket as _socket_module # Low-level C extension + import socket as socket_module # High-level module + + class BlockedSocket(_socket_module.socket): + def __init__(self, family=-1, type=-1, proto=-1, fileno=None): + # Allow Unix domain sockets (needed for IPC) + # Check both AF_UNIX and AF_LOCAL for consistency with C implementation + af_local = getattr(_socket_module, "AF_LOCAL", _socket_module.AF_UNIX) + if family in (_socket_module.AF_UNIX, af_local): + super().__init__(family, type, proto, fileno) + # Block IPv4/IPv6 + elif family in (_socket_module.AF_INET, _socket_module.AF_INET6): + raise OSError(101, "Network is unreachable (blocked by sandbox)") + else: + super().__init__(family, type, proto, fileno) + + # Patch BOTH modules to prevent bypass attempts + _socket_module.socket = BlockedSocket # Blocks: import _socket; _socket.socket() + socket_module.socket = BlockedSocket # Blocks: import socket; socket.socket() + + # Keep IPython history process-local so workers and sessions do not contend on a shared history.sqlite file. + shell_config = Config() + shell_config.HistoryManager.hist_file = ":memory:" + shell = TerminalInteractiveShell(config=shell_config) + # TerminalInteractiveShell installs a SIGINT handler that calls sys.exit(0) + # instead of raising KeyboardInterrupt when _executing is False (which is + # the case when run_cell is called programmatically). Restore the default + # handler so SIGINT raises KeyboardInterrupt and our except clause catches it. + signal.signal(signal.SIGINT, signal.SIG_DFL) + try: + while True: + try: + msg = conn.recv() + except EOFError: + break + if not isinstance(msg, dict): + continue + cmd = msg.get("cmd") + if cmd == "exec": + code = msg.get("code", "") + exec_id = msg.get("id") + traceback_verbosity = msg.get("traceback_verbosity", "Plain") + + # Set traceback verbosity for this execution + shell.InteractiveTB.set_mode(mode=traceback_verbosity) + + stdout_buf = io.StringIO() + stderr_buf = io.StringIO() + try: + # Capture stdout/stderr so we can send back outputs even if the caller times out + with redirect_stdout(stdout_buf), redirect_stderr(stderr_buf): + res = shell.run_cell(code) + conn.send( + { + "status": "ok", + "id": exec_id, + "result_repr": repr(getattr(res, "result", None)), + "stdout": stdout_buf.getvalue(), + "stderr": stderr_buf.getvalue(), + "has_error": res.error_before_exec or res.error_in_exec, + } + ) + except KeyboardInterrupt: + conn.send( + { + "status": "interrupted", + "id": exec_id, + "stdout": stdout_buf.getvalue(), + "stderr": stderr_buf.getvalue(), + } + ) + except Exception: + conn.send( + { + "status": "error", + "id": exec_id, + "traceback": traceback.format_exc(), + "stdout": stdout_buf.getvalue(), + "stderr": stderr_buf.getvalue(), + } + ) + elif cmd == "shutdown": + break + finally: + try: + conn.close() + except Exception: + pass + + +class ShellManager: + def __init__(self): + """ + Manages IPython shell processes with proper timeout and cancellation support. + """ + self.shells = {} # shell_id -> dict(proc, conn, lock, created, last_used, restart_pending) + self.manager_lock = threading.Lock() # Protects shells dict + + def start_shell(self, shell_id): + parent_conn, child_conn = mp.Pipe(duplex=True) + proc = mp.Process(target=shell_worker, args=(child_conn,), daemon=True) + proc.start() + current_time = time.time() + with self.manager_lock: + self.shells[shell_id] = { + "proc": proc, + "conn": parent_conn, + "lock": threading.Lock(), + "created": current_time, + "last_used": current_time, + "restart_pending": False, # Flag to indicate this shell was restarted and needs new_session_created=True + } + + def stop_shell(self, shell_id): + with self.manager_lock: + entry = self.shells.pop(shell_id, None) + + if not entry: + return + proc, conn = entry["proc"], entry["conn"] + try: + conn.send({"cmd": "shutdown"}) + except Exception: + pass + try: + conn.close() + except Exception: + pass + proc.terminate() + proc.join(timeout=2.0) + + def _finish_restart(self, shell_id): + """Start a fresh shell and mark it as needing session restore. + + IMPORTANT: Must be called OUTSIDE except handlers. start_shell() uses + multiprocessing.Process.start() → os.fork(). A child forked inside an + except block inherits sys.exc_info() from the parent, causing Python's + implicit exception chaining to attach the parent's exception context to + every subsequent exception in the child process. + """ + self.start_shell(shell_id) + with self.manager_lock: + if shell_id in self.shells: + self.shells[shell_id]["restart_pending"] = True + + def _cleanup_shell_resources(self, proc, conn): + # Best-effort teardown for the current shell process and pipe. + try: + proc.terminate() + except Exception: + try: + os.kill(proc.pid, signal.SIGKILL) + except Exception: + pass + try: + proc.join(timeout=2.0) + except Exception: + pass + try: + conn.close() + except Exception: + pass + + def run_cell(self, shell_id, code, timeout=1.0, grace=2.0, traceback_verbosity="Plain"): + """ + Execute `code` on shell `shell_id`. + - timeout: seconds to wait for normal completion + - grace: seconds to wait after sending SIGINT + - traceback_verbosity: IPython traceback verbosity mode + Returns a dict with status and outputs. If it times out and we kill the shell, + status will be 'timeout_killed' and the shell will be restarted (memory dropped). + """ + current_time = time.time() + with self.manager_lock: + entry = self.shells.get(shell_id) + shell_was_created = False + + if entry is None: + shell_was_created = True + else: + # Update last_used timestamp + entry["last_used"] = current_time + + # Check if this shell has a restart pending and clear the flag + shell_was_recently_restarted = False + if not shell_was_created and shell_id in self.shells: + with self.manager_lock: + if self.shells[shell_id].get("restart_pending", False): + shell_was_recently_restarted = True + self.shells[shell_id]["restart_pending"] = False + + # Create shell outside the lock to avoid blocking other operations + if shell_was_created: + self.start_shell(shell_id) + with self.manager_lock: + entry = self.shells[shell_id] + + proc = entry["proc"] + conn = entry["conn"] + lock = entry["lock"] + + exec_id = time.time_ns() + with lock: + _need_restart = False + # send execution request + try: + conn.send({"cmd": "exec", "id": exec_id, "code": code, "traceback_verbosity": traceback_verbosity}) + except Exception as exc: + logging.warning(f"Shell process for {shell_id} failed before execution request was sent, restarting") + # clean up the shell process and connection - best-effort + self._cleanup_shell_resources(proc, conn) + + # remove the old shell entry + with self.manager_lock: + self.shells.pop(shell_id, None) + + _need_restart = True + _exc_msg = str(exc) + _restart_result = { + "status": "error", + "msg": f"send failed: {_exc_msg}", + "shell_was_created": shell_was_created, + "shell_was_restarted": True, + "shell_was_recently_restarted": shell_was_recently_restarted, + } + + if _need_restart: + self._finish_restart(shell_id) + return _restart_result + + # wait for the result up to `timeout` + _need_restart = False + if conn.poll(timeout): + try: + result = conn.recv() + result["shell_was_created"] = shell_was_created + result["shell_was_recently_restarted"] = shell_was_recently_restarted + return result + except EOFError: + # Connection closed - shell process died, need to restart + logging.warning(f"Shell process for {shell_id} died during execution, restarting") + + # clean up the shell process and connection - best-effort + self._cleanup_shell_resources(proc, conn) + + # remove the old shell entry + with self.manager_lock: + self.shells.pop(shell_id, None) + + _need_restart = True + _restart_result = { + "status": "error", + "msg": "connection closed", + "shell_was_created": shell_was_created, + "shell_was_restarted": True, + "shell_was_recently_restarted": shell_was_recently_restarted, + } + + if _need_restart: + self._finish_restart(shell_id) + return _restart_result + + # no reply yet -> try gentle interrupt (SIGINT) + try: + # Process.send_signal exists on Unix; fallback to os.kill if necessary + try: + proc.send_signal(signal.SIGINT) + except AttributeError: + os.kill(proc.pid, signal.SIGINT) + except Exception: + # If we couldn't send SIGINT, fall through to termination after grace + pass + + # wait short grace period for the shell to handle the interrupt + _need_restart = False + if conn.poll(grace): + try: + result = conn.recv() + result["shell_was_created"] = shell_was_created + result["shell_was_recently_restarted"] = shell_was_recently_restarted + return result + except EOFError: + # Connection closed - shell process died, need to restart + logging.warning(f"Shell process for {shell_id} died during interrupt, restarting") + # clean up the shell process and connection - best-effort + self._cleanup_shell_resources(proc, conn) + + # remove the old shell entry + with self.manager_lock: + self.shells.pop(shell_id, None) + + _need_restart = True + _restart_result = { + "status": "interrupted", + "msg": "connection closed after interrupt", + "shell_was_created": shell_was_created, + "shell_was_restarted": True, + "shell_was_recently_restarted": shell_was_recently_restarted, + } + + if _need_restart: + self._finish_restart(shell_id) + return _restart_result + + # still stuck -> terminate the shell and restart it (drop memory) + # clean up the shell process and connection - best-effort + self._cleanup_shell_resources(proc, conn) + + # remove and restart a fresh shell for this id + with self.manager_lock: + self.shells.pop(shell_id, None) + self._finish_restart(shell_id) + + return { + "status": "timeout_killed", + "id": exec_id, + "shell_was_restarted": True, + "shell_was_recently_restarted": shell_was_recently_restarted, + } + + +def log_session_count(prefix: str = "") -> None: + try: + with shell_manager.manager_lock: + session_count = len(shell_manager.shells) + logging.info("%sactive_sessions=%d", prefix, session_count) + except Exception: + pass + + +# Global shell manager for IPython sessions +shell_manager = ShellManager() +SESSION_TIMEOUT = float(os.getenv("NEMO_SKILLS_SANDBOX_SESSION_TIMEOUT", 0)) # disabled by default + + +def cleanup_expired_sessions(): + """Remove IPython sessions that haven't been used recently""" + if SESSION_TIMEOUT <= 0: + return # Session expiration disabled + + current_time = time.time() + expired_sessions = [] + + with shell_manager.manager_lock: + for session_id, session_data in shell_manager.shells.items(): + if current_time - session_data["last_used"] > SESSION_TIMEOUT: + expired_sessions.append(session_id) + + for session_id in expired_sessions: + try: + shell_manager.stop_shell(session_id) + logging.info(f"Cleaned up expired session: {session_id}") + except Exception as e: + logging.warning(f"Error cleaning up session {session_id}: {e}") + + +def postprocess_output(output, traceback_verbosity): + if traceback_verbosity not in ["Minimal", "Plain"]: + return output + + def strip_ansi_codes(text): + ansi_escape = re.compile(r"\x1B(?:[@-Z\-_]|\[[0-?]*[ -/]*[@-~])") + return ansi_escape.sub("", text) + + output = strip_ansi_codes(output) + lines = [] + for line in output.split("\n"): + if line.strip().startswith("File 0: + cleanup_expired_sessions() + + # Execute code using ShellManager with proper timeout and cancellation + result = shell_manager.run_cell( + session_id, generated_code, timeout=timeout, grace=2.0, traceback_verbosity=traceback_verbosity + ) + + # Determine if this is effectively a new session + new_session_created = ( + result.get("shell_was_created", False) # Shell was created for this request + or result.get("shell_was_restarted", False) # Shell was restarted during this request + or result.get("shell_was_recently_restarted", False) # Shell was restarted in a previous request + ) + + # Map ShellManager results to expected format + if result["status"] == "ok": + process_status = "error" if result.get("has_error", False) else "completed" + return { + "process_status": process_status, + "stdout": postprocess_output(result.get("stdout", ""), traceback_verbosity), + "stderr": postprocess_output(result.get("stderr", ""), traceback_verbosity), + "new_session_created": new_session_created, + } + elif result["status"] == "timeout_killed": + logging.warning(f"IPython session {session_id} timed out after {timeout}s, shell was restarted") + return { + "process_status": "timeout", + "stdout": "", + "stderr": f"Execution timed out after {timeout} seconds\n", + "new_session_created": True, # Shell was restarted, so it's effectively new + } + elif result["status"] == "interrupted": + # For timeout scenarios, treat interruption as timeout since it happened due to timeout + # Check if shell was restarted during interruption + if result.get("shell_was_restarted", False): + new_session_created = True + logging.warning(f"IPython session {session_id} was interrupted and restarted after {timeout}s timeout") + else: + logging.warning(f"IPython session {session_id} was interrupted after {timeout}s timeout") + return { + "process_status": "timeout", + "stdout": postprocess_output(result.get("stdout", ""), traceback_verbosity), + "stderr": postprocess_output( + result.get("stderr", "") + f"Execution timed out after {timeout} seconds\n", traceback_verbosity + ), + "new_session_created": new_session_created, + } + else: # error status + # Check if shell was restarted during error + if result.get("shell_was_restarted", False): + new_session_created = True + logging.warning(f"IPython session {session_id} had an error and was restarted") + error_msg = result.get("traceback", result.get("msg", "Unknown error")) + return { + "process_status": "error", + "stdout": postprocess_output(result.get("stdout", ""), traceback_verbosity), + "stderr": postprocess_output(result.get("stderr", "") + f"\n{error_msg}\n", traceback_verbosity), + "new_session_created": new_session_created, + } + + except Exception as e: + logging.error(f"Error in execute_ipython_session for session {session_id}: {e}") + return { + "process_status": "error", + "stdout": "", + "stderr": f"Session error: {e}\n", + "new_session_created": False, + } + + +# Log per-request session count after each response +@app.after_request +def _after_log_session_count(response): + log_session_count() + return response + + +MEM_LIMIT_BYTES = int(os.environ.get("NEMO_SKILLS_SANDBOX_MEM_LIMIT", 50 * 1024**3)) # 50 GiB default + +# Set per-worker memory limit for ipython session +resource.setrlimit(resource.RLIMIT_AS, (2 * MEM_LIMIT_BYTES, 2 * MEM_LIMIT_BYTES)) +resource.setrlimit(resource.RLIMIT_DATA, (2 * MEM_LIMIT_BYTES, 2 * MEM_LIMIT_BYTES)) +logging.info("Applied worker memory limit (RLIMIT_AS/RLIMIT_DATA): %d bytes", 2 * MEM_LIMIT_BYTES) + + +# Code to kill the process tree for lean4 code execution +def kill_process_tree(proc): + """ + Safely and aggressively kills a process and all its descendants. + This is the recommended approach for ensuring cleanup. + """ + try: + parent = psutil.Process(proc.pid) + # Get all children of the process, recursively. + children = parent.children(recursive=True) + # Add the parent to the list of processes to be killed. + all_processes = children + [parent] + + # Kill all processes in the tree. + for p in all_processes: + try: + # SIGKILL is a forceful, non-ignorable kill signal. + p.kill() + except psutil.NoSuchProcess: + # The process might have already died, which is fine. + pass + + # Wait for all processes to be terminated. + gone, alive = psutil.wait_procs(all_processes, timeout=3) + if alive: + # If any processes are still alive, they are likely zombies + # or in an unkillable state. This is a last resort. + for p in alive: + logging.warning("Process %s could not be killed.", p.pid) + except psutil.NoSuchProcess: + # The main process already died before we could kill it. + pass + except Exception as e: + logging.error("Error in kill_process_tree: %s", e) + + +def set_limits(mem_bytes: int = MEM_LIMIT_BYTES) -> None: + """ + Apply RLIMITs and start a new session for the child process. + + Called via `preexec_fn` (subprocess) or directly in a forked worker. + """ + resource.setrlimit(resource.RLIMIT_AS, (mem_bytes, mem_bytes)) + resource.setrlimit(resource.RLIMIT_DATA, (mem_bytes, mem_bytes)) + os.setsid() # isolate PGID / signals + + +def execute_python(generated_code, std_input, timeout, language): + execution_command = [language, "-c", generated_code] + try: + process = subprocess.Popen( + execution_command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + universal_newlines=True, + preexec_fn=set_limits, + ) + stdout, stderr = process.communicate(input=std_input, timeout=timeout) + return {"process_status": "completed", "stdout": stdout, "stderr": stderr} + except subprocess.TimeoutExpired: + try: + # kill the whole process group + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=1) # reap, no extra timeout needed + return {"process_status": "timeout", "stdout": "", "stderr": f"Execution timed out after {timeout} seconds\n"} + + +def execute_lean4(generated_code, timeout): + temp_file_name = None + proc = None # <-- Keep track of the process object + try: + project_path = "/lean4/my_project" + # Use a with statement for the temp file to ensure it's closed + with tempfile.NamedTemporaryFile(dir=project_path, delete=False, suffix=".lean") as temp_file: + temp_file_name = temp_file.name + temp_file.write(generated_code.encode("utf-8")) + temp_file.flush() # Ensure data is written to disk + + # Use subprocess.Popen for more control + proc = subprocess.Popen( + ["lake", "env", "--dir", project_path, "lean", temp_file_name], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=project_path, + preexec_fn=os.setsid, + ) + + # Communicate with the process, which waits for it to finish + # This will raise TimeoutExpired if the timeout is reached + stdout, stderr = proc.communicate(timeout=timeout) + + if proc.returncode == 0: + process_status = "completed" + else: + process_status = "failed" + + return { + "process_status": process_status, + "stdout": stdout.decode("utf-8"), + "stderr": stderr.decode("utf-8"), + } + + except subprocess.TimeoutExpired: + # kill the process tree + kill_process_tree(proc) + # Now we can safely get any output that was generated before the kill. + stdout, stderr = proc.communicate() + + final_stderr = stderr.decode("utf-8") + f"Execution timed out after {timeout} seconds\n" + return { + "process_status": "timeout", + "stdout": stdout.decode("utf-8"), + "stderr": final_stderr, + } + + except Exception as e: + logging.error("Error executing Lean4 code: %s", e) + return {"process_status": "error", "stdout": "", "stderr": str(e) + "\n"} + finally: + # Safely remove the temporary file if it was created + if temp_file_name and os.path.exists(temp_file_name): + os.remove(temp_file_name) + + +def execute_shell(command, timeout): + tmp_path = None + try: + # Write the full script to a temp file so /bin/bash can read it from disk + with tempfile.NamedTemporaryFile(delete=False, suffix=".sh", mode="w") as tmp: + tmp.write(command) + tmp_path = tmp.name + os.chmod(tmp_path, 0o755) + + result = subprocess.run( + ["/bin/bash", tmp_path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=timeout, + preexec_fn=set_limits, + ) + process_status = "completed" if result.returncode == 0 else "error" + return {"process_status": process_status, "stdout": result.stdout, "stderr": result.stderr} + except subprocess.TimeoutExpired: + return {"process_status": "timeout", "stdout": "", "stderr": f"Execution timed out after {timeout} seconds\n"} + finally: + if tmp_path and os.path.exists(tmp_path): + os.remove(tmp_path) + + +# Main Flask endpoint to handle execution requests +@app.route("/execute", methods=["POST"]) +def execute(): + generated_code = request.json["generated_code"] + timeout = request.json["timeout"] + language = request.json.get("language", "ipython") + std_input = request.json.get("std_input", "") + max_output_characters = request.json.get("max_output_characters", 1000) + traceback_verbosity = request.json.get("traceback_verbosity", "Plain") + + session_id = request.headers.get("X-Session-ID") + + if language == "ipython": + if session_id is None: + return {"error": "X-Session-ID header required for ipython sessions"}, 400 + result = execute_ipython_session(generated_code, session_id, timeout, traceback_verbosity) + elif language == "lean4": + result = execute_lean4(generated_code, timeout) + elif language == "shell": + result = execute_shell(generated_code, timeout) + else: + result = execute_python(generated_code, std_input, timeout, language) + + if len(result["stdout"]) > max_output_characters: + result["stdout"] = result["stdout"][:max_output_characters] + "" + if len(result["stderr"]) > max_output_characters: + result["stderr"] = result["stderr"][:max_output_characters] + "" + + return result + + +# Session management endpoints +@app.route("/sessions", methods=["GET"]) +def list_sessions(): + """List all active IPython sessions""" + try: + session_info = {} + + # Get sessions from ShellManager with proper locking + with shell_manager.manager_lock: + for session_id, session_data in shell_manager.shells.items(): + session_info[session_id] = { + "backend": "ipython", + "created": session_data["created"], + "last_used": session_data["last_used"], + "alive": True, # All shells in manager are alive + } + return {"sessions": session_info, "backend": "ipython"} + except Exception as e: + import traceback + + error_msg = f"Error in list_sessions: {str(e)}\n{traceback.format_exc()}" + logging.error(error_msg) + return {"error": error_msg}, 500 + + +@app.route("/sessions/", methods=["DELETE"]) +def delete_session(session_id): + """Delete a specific IPython session""" + try: + with shell_manager.manager_lock: + session_exists = session_id in shell_manager.shells + + if session_exists: + shell_manager.stop_shell(session_id) + return {"message": f"IPython session {session_id} deleted successfully"} + else: + return {"error": f"IPython session {session_id} not found"}, 404 + except Exception as e: + return {"error": f"Error deleting IPython session {session_id}: {e}"}, 500 + + +@app.route("/health", methods=["GET"]) +def health(): + return {"status": "healthy", "worker": os.environ.get("WORKER_NUM", "unknown")} + + +if __name__ == "__main__": + log = logging.getLogger("werkzeug") + log.setLevel(logging.WARNING) + app.run(port=6000) diff --git a/integrations/nemo-gym/tests/test_e2e.py b/integrations/nemo-gym/tests/test_e2e.py index d5605366..a7a00990 100644 --- a/integrations/nemo-gym/tests/test_e2e.py +++ b/integrations/nemo-gym/tests/test_e2e.py @@ -12,16 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""End-to-end test against a live ate-env deployment. +"""End-to-end tests against a live ate-env deployment. Skipped unless SUBSTRATE_E2E_API_URL points at a reachable ate-env-api (e.g. `kubectl port-forward -n ate-env svc/ate-env-api 7777:7777` then `SUBSTRATE_E2E_API_URL=127.0.0.1:7777 pytest tests/test_e2e.py`; an http:// URL works too). SUBSTRATE_E2E_TEMPLATE picks the ActorTemplate (default `default-template`, ate-env's own default). -Exercises the full provider lifecycle on a real Substrate actor: -create → exec (incl. nonzero exit, env/cwd) → file round-trip → status → -close → status STOPPED. + +test_full_lifecycle exercises the provider on a real Substrate actor: +create -> exec (incl. nonzero exit, env/cwd) -> file round-trip -> status -> +close -> status STOPPED. + +test_runtime_layer_endpoint needs a base template that carries the NeMo +runtime layer (integrations/nemo-gym/layer, registered as in +hack/runtime-layer-demo.sh) in SUBSTRATE_E2E_RUNTIME_TEMPLATE, and the port +NeMo Skills' sandbox server listens on in SUBSTRATE_E2E_RUNTIME_PORT (default +6000), and a digest-pinned task image in SUBSTRATE_E2E_RUNTIME_IMAGE: the +sandbox is created from that image on the layered base, the way a recipe does +it, because the base's own container is the ate-env guest image, which cannot +host the layer's glibc interpreter. """ from __future__ import annotations @@ -31,7 +41,6 @@ from pathlib import Path import pytest - from nemo_gym_substrate._compat import SandboxSpec, SandboxStatus from nemo_gym_substrate.provider import SubstrateSandboxProvider @@ -86,3 +95,65 @@ async def scenario() -> None: await provider.aclose() asyncio.run(scenario()) + + +RUNTIME_TEMPLATE = os.environ.get("SUBSTRATE_E2E_RUNTIME_TEMPLATE") +RUNTIME_PORT = int(os.environ.get("SUBSTRATE_E2E_RUNTIME_PORT", "6000")) +RUNTIME_IMAGE = os.environ.get("SUBSTRATE_E2E_RUNTIME_IMAGE") + + +def _post_json(url: str, headers: dict[str, str], body: dict, timeout: float = 120.0) -> dict: + import json + import urllib.request + + req = urllib.request.Request( + url, + data=json.dumps(body).encode(), + method="POST", + headers={"Content-Type": "application/json", **headers}, + ) + with urllib.request.urlopen(req, timeout=timeout) as resp: + return json.loads(resp.read()) + + +@pytest.mark.skipif( + not (RUNTIME_TEMPLATE and RUNTIME_IMAGE), + reason="SUBSTRATE_E2E_RUNTIME_TEMPLATE and SUBSTRATE_E2E_RUNTIME_IMAGE not set; needs a template with the NeMo runtime layer and a task image", +) +def test_runtime_layer_endpoint() -> None: + """A daemon the runtime layer injects is reached through endpoint(): NeMo + Skills' sandbox server answers POST /execute through ate-env-api's port + proxy, with the headers the provider hands out, and keeps an IPython + session between calls.""" + + async def scenario() -> None: + # The layered template is the provider's base (create.template), so the + # task image is derived from it; a provider_options template would + # select the base itself and skip the image. + provider = SubstrateSandboxProvider( + {"connection": {"api_url": API_URL}, "create": {"template": RUNTIME_TEMPLATE}} + ) + handle = None + try: + handle = await provider.create( + SandboxSpec(image=RUNTIME_IMAGE, ports=(RUNTIME_PORT,), ready_timeout_s=240) + ) + ep = await provider.endpoint(handle, RUNTIME_PORT) + url = ep.endpoint.rstrip("/") + "/execute" + # The layer's own interpreter runs the code: the task image needs no python. + out = _post_json(url, ep.headers, {"generated_code": "print(6 * 7)", "language": "python", "timeout": 30}) + assert out.get("process_status") == "completed", out + assert out.get("stdout", "").strip() == "42", out + session = {"X-Session-ID": "e2e", **ep.headers} + _post_json(url, session, {"generated_code": "x = 40", "language": "ipython", "timeout": 30}) + out = _post_json(url, session, {"generated_code": "print(x + 2)", "language": "ipython", "timeout": 30}) + assert out.get("stdout", "").strip() == "42", out + # An undeclared port is refused before anything reaches the cluster. + with pytest.raises(ValueError): + await provider.endpoint(handle, RUNTIME_PORT + 1) + finally: + if handle is not None: + await provider.close(handle) + await provider.aclose() + + asyncio.run(scenario())