Sync upstream snapshot and sandbox runtime updates

.github/workflows/helm-e2e.yaml

@@ -25,7 +25,6 @@ jobs:
     env:
       VERSION: helm-e2e
       E2E_ATENET_DATAPLANE: agentgateway
-      E2E_INGRESS_AUTH_MODE: deprecated-insecure
       E2E_CREDENTIAL_PROVIDER: "1"
       # Retain external-volume coverage after the preview gate is introduced.
       ATE_PREVIEW: ALL
@@ -75,12 +74,12 @@ jobs:
       run: helm upgrade --install substrate-crds charts/substrate-crds
     - name: Bootstrap mTLS authorities
       run: |
-        hack/install-ate-kind.sh --create-podcertificate-controller-cas
-        hack/install-ate-kind.sh --create-jwt-authority-pool-secret
-        hack/install-ate-kind.sh --create-actor-id-ca-pool-secret
-        hack/install-ate-kind.sh --create-actor-id-ca-certs-secret
-        hack/install-ate-kind.sh --create-api-authentication-config
-        hack/install-ate-kind.sh --create-egress-mitm-ca-pool-secret
+        hack/install-ate-kind.sh create podcertificate-controller-cas
+        hack/install-ate-kind.sh create jwt-authority-pool
+        hack/install-ate-kind.sh create actor-id-ca-pool
+        hack/install-ate-kind.sh create actor-id-ca-certs
+        hack/install-ate-kind.sh create api-authentication-config
+        hack/install-ate-kind.sh create egress-mitm-ca-pool
     - name: Install certificate controller
       run: |
         helm upgrade --install substrate-podcert charts/substrate-podcert \
@@ -109,16 +108,16 @@ jobs:
         sudo modprobe nfs || true
         sudo modprobe nfsd || true
     - name: Install CSI NFS driver
-      run: hack/install-ate-kind.sh --setup-csi=nfs
+      run: hack/install-ate-kind.sh setup csi nfs
     - name: Deploy micro-VM counter demo
       # The deploy creates the substrate ActorTemplate and waits for its golden
       # snapshot internally; the ActorTemplate CRD (and its Ready condition)
       # no longer exists to wait on.
       run: hack/run-microvm-demo-kind.sh --skip-control-plane
     - name: Deploy gVisor counter demo
-      run: hack/install-ate-kind.sh --deploy-demo-counter
+      run: hack/install-ate-kind.sh deploy demo counter
     - name: Deploy egress demo
-      run: hack/install-ate-kind.sh --deploy-demo-egress
+      run: hack/install-ate-kind.sh deploy demo egress
     - name: Trust local E2E TLS origins
       run: hack/setup-e2e-egress-tls-kind.sh
     - name: Run E2E tests (gVisor)

.github/workflows/pr-workflow.yaml

@@ -48,7 +48,7 @@ jobs:
       with:
         fetch-depth: 0
     - name: Setup Go
-      uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
+      uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
       with:
         go-version-file: 'go.mod'
     # Unconditional: a released migration becoming permanent is exactly what
@@ -173,11 +173,6 @@ jobs:
       # --preview, so tests of a disabled gate skip themselves (see
       # internal/e2e/preview.go).
       ATE_PREVIEW: ${{ matrix.preview == 'on' && 'ALL' || '' }}
-      # Job-wide for the same reason, and E2E_INGRESS_AUTH_MODE has to match
-      # it so the suites dial the router's mTLS listeners. The agentgateway
-      # dataplane does not support static-mtls.
-      ATE_INGRESS_AUTH_MODE: ${{ matrix.dataplane == 'agentgateway' && 'deprecated-insecure' || 'static-mtls' }}
-      E2E_INGRESS_AUTH_MODE: ${{ matrix.dataplane == 'agentgateway' && 'deprecated-insecure' || 'static-mtls' }}
       # The suites' router client (internal/e2e/ingressauth.go). Only e2e
       # installs allowlist it, so the suites can reach actors through the
       # router's mTLS listeners.
@@ -198,7 +193,7 @@ jobs:
     - name: Checkout
       uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0
     - name: Setup Go
-      uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
+      uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
       with:
         go-version-file: 'go.mod'
         # Use the e2e cache below instead. setup-go's cache key is shared with
@@ -257,28 +252,55 @@ jobs:
       # gateway. ATE_PREVIEW above selects the preview features.
       env:
         ATE_ATENET_DATAPLANE: ${{ matrix.dataplane }}
-      run: hack/install-ate-kind.sh --deploy-ate-system --rollout-timeout=300s
+        NO_DEV_ENV: "true"
+        KO_DOCKER_REPO: "localhost:5001"
+        ATE_INSTALL_KIND: "true"
+        BUCKET_NAME: "ate-snapshot"
+        KIND_CLUSTER_NAME: "kind"
+        KUBECTL_CONTEXT: "kind-kind"
+      run: go run ./cmd/ate-setup deploy ate-system --rollout-timeout=300s
     - name: Enable NFS
       # Load NFS kernel modules so in-cluster NFS server and CSI driver can run.
       run: |
         sudo modprobe nfs || true
         sudo modprobe nfsd || true
     - name: Install CSI NFS driver
-      run: hack/install-ate-kind.sh --setup-csi=nfs
+      env:
+        NO_DEV_ENV: "true"
+        KO_DOCKER_REPO: "localhost:5001"
+        ATE_INSTALL_KIND: "true"
+        BUCKET_NAME: "ate-snapshot"
+        KIND_CLUSTER_NAME: "kind"
+        KUBECTL_CONTEXT: "kind-kind"
+      run: go run ./cmd/ate-setup setup csi nfs
     - name: Deploy micro-VM counter demo
       # Stages the (cached) assets into the cluster's rustfs and deploys the
       # counter-microvm demo onto the control plane installed above.
       run: hack/run-microvm-demo-kind.sh --skip-control-plane
     - name: Deploy gVisor counter demo
-      run: hack/install-ate-kind.sh --deploy-demo-counter
+      env:
+        NO_DEV_ENV: "true"
+        KO_DOCKER_REPO: "localhost:5001"
+        ATE_INSTALL_KIND: "true"
+        BUCKET_NAME: "ate-snapshot"
+        KIND_CLUSTER_NAME: "kind"
+        KUBECTL_CONTEXT: "kind-kind"
+      run: go run ./cmd/ate-setup deploy demo counter
     - name: Deploy egress demos
       # TestActorEgress in the networking suite builds its Actor from the egress
       # ActorTemplate for the class under test, so both fixtures have to exist
       # before their respective lane runs. The demo's actors trust the CA the
       # gateway mints its leaves from.
+      env:
+        NO_DEV_ENV: "true"
+        KO_DOCKER_REPO: "localhost:5001"
+        ATE_INSTALL_KIND: "true"
+        BUCKET_NAME: "ate-snapshot"
+        KIND_CLUSTER_NAME: "kind"
+        KUBECTL_CONTEXT: "kind-kind"
       run: |
-        hack/install-ate-kind.sh --deploy-demo-egress
-        hack/install-ate-kind.sh --deploy-demo-egress-microvm
+        go run ./cmd/ate-setup deploy demo egress
+        go run ./cmd/ate-setup deploy demo egress-microvm
     - name: Trust local TLS origins in egress gateway
       run: hack/setup-e2e-egress-tls-kind.sh
     # scale the idle sandbox class's demo pools to zero so each lane has the node

.golangci-kal.yaml

@@ -83,7 +83,7 @@ linters:
       - path: 'pkg/api/v1alpha1/sandboxconfig_types\.go'
         text: '^(optionalfields|requiredfields): .*\bSandboxConfigSpec\.SandboxClass\b'
       - path: 'pkg/api/v1alpha1/sandboxconfig_types\.go'
-        text: '^(nomaps|optionalfields): .*\bSandboxVersionConfig\.(Assets|PauseImage)\b'
+        text: '^(nomaps|optionalfields): .*\bSandboxVersionConfig\.Assets\b'
       - path: 'pkg/api/v1alpha1/sandboxconfig_types\.go'
         text: '^(nonpointerstructs|requiredfields): .*\bSandboxConfig\.Spec\b'
       - path: 'pkg/api/v1alpha1/csidriverconfig_types\.go'

AGENTS.md

@@ -45,6 +45,7 @@
 - Keep atelet and the worker runtimes opaque to each other. The only knowledge they may share is `internal/proto/ateworkerpb` and the paths in `internal/nodepath`. Using the same library code is fine; depending on how the other side is built is not. We want atelet to roll ahead of workers eventually, so don't widen that interface casually. Until v1.0.0 nothing enforces this; see Backward Compatibility.
 - Actor features work on both gVisor and micro-VM, with e2e coverage for both, unless the issue or PR description scopes the change to one runtime.
 - Don't assume kind, GKE, amd64 or a newer kernel than the repo already needs. Dev scripts must work on arm64 macOS. Don't change host-wide state.
+- Put shell code in standalone `.sh` scripts so shellcheck can lint them. Place each script near what uses it, such as `hack/` for dev scripts or a directory under `.github/` for a workflow. Don't inline it in YAML, such as a GitHub workflow `run:` block, in JSON, or in another language's source, such as Go or Python. Have those call the script. Get a reviewer's agreement before making an exception.
 - Read the `CI` environment variable only inside a named fail-closed predicate, never inline. See Testing Instructions 5.
 - Before changing an API, read `docs/api-style-guide.md` and `docs/api-validation.md`.
 - Pin image references you add or update by digest, and look the digest up yourself: `go run github.com/google/go-containerregistry/cmd/crane@latest digest <image>`.
@@ -152,7 +153,7 @@ See the [metric registry](docs/observability.md#the-metric-registry) section of
 
 Agent Substrate is built for zero-trust, multi-tenant execution of untrusted actors. Preserve and extend these invariants when modifying the codebase (see `docs/security.md` and `docs/threat-model.md` for the full architecture):
 
-- **Workload & Storage Isolation**: Actors run strictly inside `gVisor` (`cmd/ateworker-gvisor`) or `Cloud Hypervisor` microVM (`cmd/ateworker-microvm`) sandboxes; plain Linux containers are not supported. `atelet` scrubs actor directories across lifecycle transitions (`resetActorDirs`), and snapshots are stamped with `SandboxRuntime` metadata and transferred via `cmd/objectstore-plugin`.
+- **Workload & Storage Isolation**: Actors run strictly inside `gVisor` (`cmd/ateworker-gvisor`) or `Cloud Hypervisor` microVM (`cmd/ateworker-microvm`) sandboxes; plain Linux containers are not supported. `atelet` scrubs actor directories across lifecycle transitions (`resetActorDirs`), and snapshots are stamped with `ActorSandboxRuntime` metadata and transferred via `cmd/objectstore-plugin`.
 - **Component & Actor Identity (mTLS, JWTs, and JWKs)**: Inter-component communication uses mTLS backed by `cmd/podcertcontroller` (`podid`, `servicedns`, and `postgres` signers). Actor JWT signing (`internal/localjwtauthority`), verification (`internal/oidcjwt`), and key discovery (served via `cmd/ate-idp-server`) are built on the standard library's crypto packages. Extend those packages rather than adding `go-jose` or another JOSE library, so the code that decides whether a token is valid stays small and supports only the algorithms Substrate uses.
-- **Control-Plane Authentication & Authorization**: `ate-api-server` checks every Control RPC against an embedded OpenFGA model (`cmd/ateapi/internal/authz/model.fga`), on by default. Every served unary RPC needs an entry in `defaultRPCPermissions` (`registry.go`): with enforcement on, an RPC without one is denied, and a test fails the build. Access is granted per global scope and atespace through AccessPolicy (`kubectl ate access-policy`); the install's bootstrap owner is the shared `ate-client` ServiceAccount that `kubectl ate` uses. See `docs/authorization.md` for roles and known gaps. Callers are authenticated via `SPIFFE` mTLS or configured OIDC `JWTProvider`s (`cmd/ateapi/internal/apiauthn`).
-- **Network Isolation, Ingress, & Egress**: `manifests/ate-install/ate-system-networkpolicies.yaml` enforces default-deny ingress across `ate-system`, and `cmd/atecontroller` reconciles per-`WorkerPool` `NetworkPolicy` objects. Actors run in private `/30` point-to-point veth network namespaces (`internal/ateworker/actornet`) with link-local blocking, policy-enforced DNS filtering (`internal/atunnel/dns`), and zero ambient credentials. `atenet-router --ingress-auth-mode=static-mtls` requires clients to present a `SPIFFE` certificate signed by `--ingress-client-ca-file` whose ID is in `--ingress-allowed-spiffe-ids` (`manifests/ate-install/atenet-router.yaml`). Outbound traffic traverses `atunnel` to `atenet-egress`, which enforces per-actor `EgressPolicy` and injects credentials out-of-band via `cmd/credential-provider/kubernetes-secrets` (`k8s-credential-provider`) or actor JWT minting.
+- **Control-Plane Authentication & Authorization**: `ate-api-server` checks every Control RPC against an embedded OpenFGA model (`cmd/ateapi/internal/authz/model.fga`), always enforced. Every served unary RPC needs an entry in `defaultRPCPermissions` (`registry.go`): an RPC without one is denied, and a test fails the build. Access is granted per global scope and atespace through AccessPolicy (`kubectl ate access-policy`); the install's bootstrap owner is the shared `ate-client` ServiceAccount that `kubectl ate` uses. See `docs/authorization.md` for roles and known gaps. Callers are authenticated via `SPIFFE` mTLS or configured OIDC `JWTProvider`s (`cmd/ateapi/internal/apiauthn`).
+- **Network Isolation, Ingress, & Egress**: `manifests/ate-install/ate-system-networkpolicies.yaml` enforces default-deny ingress across `ate-system`, and `cmd/atecontroller` reconciles per-`WorkerPool` `NetworkPolicy` objects. Actors run in private `/30` point-to-point veth network namespaces (`internal/ateworker/actornet`) with link-local blocking, policy-enforced DNS filtering (`internal/atunnel/dns`), and zero ambient credentials. Outbound traffic traverses `atunnel` to `atenet-egress`, which enforces per-actor `EgressPolicy` and injects credentials out-of-band via `cmd/credential-provider/kubernetes-secrets` (`k8s-credential-provider`) or actor JWT minting.

README.md

@@ -119,10 +119,10 @@ To quickly set up the complete environment:
 hack/create-kind-cluster.sh
 
 # install ate, PostgreSQL, rustfs
-hack/install-ate-kind.sh --deploy-ate-system --credential-provider='{"name":"k8s.io"}'
+hack/install-ate-kind.sh deploy ate-system --credential-provider='{"name":"k8s.io"}'
 
 # install counter demo
-hack/install-ate-kind.sh --deploy-demo-counter
+hack/install-ate-kind.sh deploy demo-counter
 
 # install kubectl-ate
 go install ./cmd/kubectl-ate
@@ -168,16 +168,16 @@ curl -X POST \
    go run ./tools/setup-gcp bootstrap
    ```
 
-   On a fresh project this step also creates the atelet Workload Identity IAM
-   grants that snapshots depend on — see
+   On a fresh project this step also creates the Workload Identity IAM grants
+   that ate-api-server and atelet depend on. See
    [what `create iam` actually grants](tools/setup-gcp/README.md#what-create-iam-actually-grants)
    to audit them or apply them manually. If you bring your own cluster instead,
    note the required Kubernetes beta APIs can only be enabled **at cluster
    creation** — see the [Create Cluster warning](tools/setup-gcp/README.md#2-create-cluster).
 
 4. Deploy the Agent Substrate system to your cluster:
    ```bash
-   ./hack/install-ate.sh --deploy-ate-system --credential-provider='{"name":"k8s.io"}'
+   go run ./cmd/ate-setup deploy ate-system --credential-provider='{"name":"k8s.io"}'
    ```
 
    `--credential-provider` is required. `{"name":"k8s.io"}` also deploys the
@@ -193,7 +193,7 @@ curl -X POST \
 
 5. You can then deploy the sample applications. See [demos/counter/README.md](demos/counter/README.md) or [demos/sandbox/README.md](demos/sandbox/README.md) for detailed walkthroughs.
    ```bash
-   ./hack/install-ate.sh --deploy-demo-counter
+   go run ./cmd/ate-setup deploy demo demo-counter
    ```
 
 #### Custom Setup and Deployment
@@ -207,13 +207,14 @@ go run ./tools/setup-gcp create bucket
 To run the PostgreSQL store backend on Cloud SQL — with IAM database
 authentication and no passwords — see [tools/setup-gcp/cloud-sql.md](tools/setup-gcp/cloud-sql.md).
 
-Similarly, you can deploy or cleanup specific Agent Substrate components using the installation script. See `./hack/install-ate.sh --help` for all options.
+Similarly, you can deploy or cleanup specific Agent Substrate components using
+the installation script. See `go run ./cmd/ate-setup` for all options.
 ```bash
 # Re-deploy only ate-apiserver of the ATE system
-./hack/install-ate.sh --deploy-ate-apiserver
+go run ./cmd/ate-setup deploy apiserver
 
 # Delete everything (core system and all demos)
-./hack/install-ate.sh --delete-all
+go run ./cmd/ate-setup delete all
 ```
 
 #### Tearing down resources (GCP)
@@ -243,7 +244,6 @@ We provide several sample applications demonstrating Agent Substrate's capabilit
 3. **[Claude Code Multiplex](demos/claude-code-multiplex/README.md)**: Demonstrates oversubscribing physical hardware by multiplexing multiple Claude Code agents onto a limited pool of workers.
 4. **[Multi-Template](demos/multi-template/README.md)**: Two `ActorTemplate`s running different binaries share one `WorkerPool`, even though the templates live in different atespaces.
 5. **[Request Parking](demos/parking/README.md)**: An oversubscribed pool where the router holds inbound requests until a worker frees up, instead of returning `503`.
-6. **[Autoscaled WorkerPool](demos/autoscaled-workerpool/README.md)**: Scales a `WorkerPool` on its assigned-worker count with an HPA fed by prometheus-adapter.
 
 ### Documentation & Guides
 * [Architecture](docs/architecture.md): How the control plane, node supervisor, and networking stack fit together.
@@ -258,6 +258,7 @@ We provide several sample applications demonstrating Agent Substrate's capabilit
 * [Request Parking](docs/request-parking.md): How the router parks requests through transient worker-pool saturation.
 * [Sandbox Asset Cache](docs/sandbox-asset-cache.md): How a node caches the sandbox binaries actors run on, decides which runtimes it needs, and evicts to a budget.
 * [Upgrade Runbook](docs/upgrade.md): Upgrade a running install to a newer supported release, and roll back if needed.
+* [Node Storage](docs/node-storage.md): How to give snapshot I/O its own disk.
 * [Security Guide](docs/security.md): Security overview, security principles, and threat modeling.
 * [Roadmap](docs/roadmap.md): Current limitations and what is planned next.
 * [Benchmarking Guide](benchmarking/README.md): Locust-based load tests, monitoring stack, and the orchestrated benchmark harness.